From 1df3d470f02b982756afb02c19b3d7d71105f904 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:22:03 +0000 Subject: [PATCH 01/30] refactor: define every stage's interfaces from #1207 and delete the ones they replace (#1207 PR 0) --- Cargo.lock | 11 - crates/accuracy/src/estimators/cms.rs | 18 +- .../accuracy/src/estimators/count_sketch.rs | 9 +- crates/accuracy/src/estimators/hll.rs | 10 +- crates/accuracy/src/estimators/kll.rs | 9 +- crates/accuracy/src/estimators/mod.rs | 27 +- crates/accuracy/src/estimators/univmon.rs | 2 + crates/accuracy/src/propagation.rs | 130 +- crates/accuracy/src/sizing.rs | 182 +- crates/cost/Cargo.toml | 2 +- crates/cost/src/execution_demand.rs | 48 +- crates/cost/src/lib.rs | 6 +- crates/cost/src/state_bytes.rs | 168 +- crates/cost/src/usage.rs | 94 +- crates/cost/tests/held_memory.rs | 168 - crates/cost/tests/ingestion_refresh_runs.rs | 289 - crates/devtools/Cargo.toml | 11 +- crates/devtools/src/bin/asap_planner.rs | 222 +- crates/devtools/src/bin/planner_fuzz.rs | 7 +- crates/devtools/src/bin/sketch_coverage.rs | 30 +- crates/devtools/src/candidate_labels.rs | 160 +- crates/devtools/src/lib.rs | 1 - crates/devtools/src/memo_pricing.rs | 401 -- crates/devtools/src/planner_fuzz/README.md | 2 +- crates/devtools/src/planner_fuzz/mod.rs | 187 +- crates/devtools/src/planning_stages.rs | 135 +- crates/devtools/src/planning_stages_text.rs | 405 +- crates/devtools/tests/asap_planner.rs | 353 - .../devtools/tests/asap_planner_acceptance.rs | 7 +- .../corpus_snapshots/o11y_bench/q02.json | 16 +- .../corpus_snapshots/o11y_bench/q05.json | 36 +- .../corpus_snapshots/o11y_bench/q06.json | 22 +- .../corpus_snapshots/o11y_bench/q11.json | 14 +- .../corpus_snapshots/o11y_bench/q12.json | 4 +- .../corpus_snapshots/o11y_bench/q18.json | 34 +- .../corpus_snapshots/o11y_bench/q19.json | 30 +- .../corpus_snapshots/o11y_bench/q20.json | 8 +- .../corpus_snapshots/o11y_bench/q21.json | 16 +- .../corpus_snapshots/o11y_bench/q22.json | 14 +- .../corpus_snapshots/o11y_bench/q23.json | 14 +- .../corpus_snapshots/o11y_bench/q25.json | 30 +- .../corpus_snapshots/o11y_bench/q26.json | 14 +- .../corpus_snapshots/o11y_bench/q27.json | 8 +- crates/devtools/tests/main.rs | 1 - crates/devtools/tests/memo_pricing.rs | 385 -- .../devtools/tests/sketch_plan_snapshots.rs | 2 +- .../fuzz-issue903-raw-retained.json | 14 +- .../nested-windows-15m-1h-mvp.json | 18 +- .../quantile-grouped-mvp.json | 26 +- .../quantile-single-mvp.json | 12 +- ...random-mixed-seed-862-long-w01-q3-mvp.json | 26 +- .../random-mixed-seed-862-w01-q3-q8-mvp.json | 42 +- .../random-mixed-seed-862-w02-q2-q7-mvp.json | 42 +- .../ratio-of-quantiles-1h-mvp.json | 20 +- .../shared-quantiles-1h-mvp.json | 18 +- .../topk-rate-guarantees-ignored-mvp.json | 16 +- crates/executor/src/capability.rs | 136 +- crates/executor/src/operators/summary/mod.rs | 85 +- .../executor/src/physical_planner/logical.rs | 2 - crates/executor/src/physical_planner/mod.rs | 129 +- .../executor/src/summary_kernels/factory.rs | 142 +- .../summary_kernels/state_column_updater.rs | 29 +- crates/executor/src/supported_summaries.rs | 72 +- crates/executor/src/values/batch.rs | 27 +- crates/executor/src/values/mod.rs | 109 +- crates/executor/tests/blocking_resources.rs | 14 +- crates/executor/tests/common/mod.rs | 27 +- crates/executor/tests/current_series_heap.rs | 9 +- crates/executor/tests/current_timestamp.rs | 3 +- crates/executor/tests/deployment.rs | 53 +- .../executor/tests/deployment_computation.rs | 539 +- crates/executor/tests/hydra_cms_execution.rs | 22 +- .../tests/keyed_summary_row_weight.rs | 23 +- crates/executor/tests/main.rs | 1 - crates/executor/tests/node_ids.rs | 5 +- crates/executor/tests/physical_dag.rs | 52 +- crates/executor/tests/physical_semantics.rs | 49 +- crates/executor/tests/plan_properties.rs | 95 +- .../tests/planspace_series_identity_heap.rs | 4 +- .../executor/tests/precompute_candidates.rs | 363 - .../executor/tests/precompute_population.rs | 24 +- crates/executor/tests/promql_binary.rs | 6 +- crates/executor/tests/promql_values.rs | 2 + crates/executor/tests/raw_scan.rs | 9 +- crates/executor/tests/state_column_updater.rs | 290 +- .../stored_tumbling_window_memory_limit.rs | 18 +- .../tests/summary_build_memory_limit.rs | 15 +- crates/executor/tests/summary_projection.rs | 13 +- crates/executor/tests/time_series_window.rs | 13 +- crates/executor/tests/univmon_execution.rs | 21 +- .../executor/tests/weighted_topk_binding.rs | 284 +- .../frontend-promql/tests/count_planning.rs | 14 +- crates/frontend-promql/tests/support.rs | 23 +- .../tests/univmon_candidates.rs | 106 +- crates/frontend-sql/tests/frequency_l2.rs | 9 +- crates/frontend-sql/tests/pearson_corr.rs | 49 +- crates/integration-tests/Cargo.toml | 10 +- crates/integration-tests/src/lib.rs | 274 +- .../tests/declared_group_counts.rs | 118 - .../tests/deployment_capabilities.rs | 271 - .../tests/exact_composition.rs | 215 +- .../tests/filtered_aggregates.rs | 7 +- .../tests/finer_aggregate_reuse.rs | 262 +- .../tests/frequency_execution.rs | 148 - .../tests/having_shared_aggregate.rs | 115 - .../tests/identical_exact_queries.rs | 1 - .../tests/independent_plan_seed.rs | 1 - .../tests/kll_tumbling_window_execution.rs | 47 +- crates/integration-tests/tests/main.rs | 21 - .../tests/materialization_e2e.rs | 356 - .../plan_wire_version_10.json | 2 +- .../stored_outputs_on_main.json | 16 +- .../tests/materialized_views.rs | 148 +- .../tests/memo_enumeration.rs | 481 -- .../tests/operator_design_examples.rs | 153 +- .../tests/per_computation_candidates_sql.rs | 21 +- .../tests/per_series_exact_execution.rs | 16 +- .../tests/physical_common/mod.rs | 36 +- .../tests/planner_layering_common/mod.rs | 514 +- .../tests/planner_layering_example1.rs | 951 --- .../tests/planner_layering_example2.rs | 1164 ---- .../tests/planner_layering_example3.rs | 635 -- .../tests/planner_layering_example4.rs | 509 -- .../tests/precompute_raw_samples.rs | 641 -- .../tests/production_rollup_workloads.rs | 90 - .../tests/promql_numeric_regressions.rs | 6 +- .../tests/promql_to_post_asap.rs | 371 +- .../tests/range_evaluation.rs | 423 -- crates/integration-tests/tests/reuse_rules.rs | 589 -- .../tests/rollup_controlled.rs | 142 +- .../tests/runtime_handoff.rs | 468 -- .../tests/sharing_issue_regressions.rs | 430 +- .../integration-tests/tests/sketch_rollup.rs | 5 +- .../tests/sketch_rollup_planning.rs | 176 - .../tests/sql_group_bound.rs | 244 - .../tests/sql_to_physical.rs | 142 - .../tests/sql_to_post_asap.rs | 37 +- .../tests/sql_trailing_window.rs | 246 - .../tests/sql_two_percentiles.rs | 555 -- .../tests/stage1_logical_asap_optimization.rs | 102 +- .../stage2_physical_asap_optimization_sql.rs | 202 - .../tests/summary_coverage_e2e.rs | 2 +- .../tests/tumbling_window_merge.rs | 887 --- crates/logical-asap-optimization/Cargo.toml | 2 +- .../src/deployment_capabilities.rs | 6 +- crates/logical-asap-optimization/src/lib.rs | 32 +- crates/logical-asap-optimization/src/pass1.rs | 14 +- .../src/pass1/logical_memo_candidates.rs | 293 +- .../src/pass1/memo_population.rs | 1231 ---- .../src/pass1/per_computation_candidates.rs | 1101 +--- .../pass1/per_computation_candidates_tests.rs | 328 +- .../src/pass1/realization.rs | 17 +- crates/logical-asap-optimization/src/pass2.rs | 9 +- .../src/pass2/asap_aware_cse.rs | 3557 ---------- .../src/pass2/finer_aggregate.rs | 146 - .../src/pass2/logical_memo_candidates.rs | 68 +- .../src/pass2/memo.rs | 12 +- .../src/pass2/memo/enumeration.rs | 236 - .../src/pass2/memo/rules.rs | 20 +- .../src/pass2/memo/rules/finer_aggregate.rs | 177 +- .../src/pass2/memo/rules/reuse.rs | 227 - .../src/pass2/memo/rules/rollup.rs | 366 +- .../src/pass2/memo/rules/scan_labels.rs | 238 +- .../rules/shared_configuration_readers.rs | 637 -- .../src/pass2/memo/rules/shared_producer.rs | 62 - .../src/pass2/memo/rules/summary_inputs.rs | 100 +- .../src/pass2/memo/rules/summary_results.rs | 82 - .../src/pass2/memo/rules/topk.rs | 268 +- .../src/pass2/memo/shared_configurations.rs | 326 - .../src/pass2/memo/state_groups.rs | 1615 ----- .../src/pass2/memo/window_candidates.rs | 325 +- .../logical-asap-optimization/src/stage0.rs | 24 +- .../src/stage0/cse.rs | 31 +- .../src/test_support.rs | 79 +- crates/physical-asap-optimization/Cargo.toml | 24 +- .../src/catalog/configurations.rs | 425 -- .../src/catalog/factoring.rs | 249 - .../src/catalog/inventory.rs | 346 - .../src/catalog/mod.rs | 150 +- .../src/catalog/pruning.rs | 195 - .../src/catalog/realize.rs | 307 +- .../src/catalog/reconstruction.rs | 231 - crates/physical-asap-optimization/src/lib.rs | 2441 +------ .../src/split_by_timing.rs | 375 -- crates/plan-selection/Cargo.toml | 21 - crates/plan-selection/src/accuracy_binding.rs | 306 - crates/plan-selection/src/checks.rs | 305 - crates/plan-selection/src/groups.rs | 2163 ------ .../src/groups/accuracy_positions.rs | 777 --- crates/plan-selection/src/groups/tests.rs | 1085 --- crates/plan-selection/src/lib.rs | 5865 +---------------- crates/plan-selection/src/milp/completion.rs | 306 - crates/plan-selection/src/milp/cost_terms.rs | 1204 ---- crates/plan-selection/src/milp/decode.rs | 162 - .../plan-selection/src/milp/lp_diagnosis.rs | 302 - crates/plan-selection/src/milp/mod.rs | 27 +- crates/plan-selection/src/milp/model.rs | 2327 +------ crates/plan-selection/src/milp/model/tests.rs | 1229 ---- crates/plan-selection/src/milp/warm_start.rs | 780 --- crates/plan-selection/src/models.rs | 58 +- crates/plan-selection/src/report.rs | 28 - crates/plan-selection/src/select.rs | 369 +- crates/plan-selection/src/select/tests.rs | 173 - crates/plan-selection/src/selector.rs | 471 -- crates/planner/Cargo.toml | 12 +- crates/planner/src/installable.rs | 354 +- crates/planner/src/lib.rs | 30 +- crates/planner/src/pass.rs | 53 - crates/planner/src/run.rs | 131 +- .../tests/deployment_accuracy_model.rs | 374 -- crates/planner/tests/main.rs | 3 - crates/planner/tests/per_sample_updater.rs | 405 -- crates/planner/tests/runtime_install.rs | 3334 +--------- crates/planner/tests/sql_per_group_topk.rs | 405 -- crates/planner/tests/sql_table_scans.rs | 357 +- crates/planner/tests/summary_sharing.rs | 505 +- crates/types/src/ir/flat.rs | 7 - crates/types/src/ir/materialized_view.rs | 32 +- crates/types/src/ir/mod.rs | 6 - crates/types/src/ir/operator/node.rs | 55 +- .../src/ir/operator/tumbling_window_chain.rs | 134 +- crates/types/src/ir/physical_export.rs | 346 +- .../src/ir/properties/execution_data_state.rs | 126 +- crates/types/src/ir/properties/mod.rs | 7 +- .../src/ir/properties/summary_coverage.rs | 2 +- crates/types/src/ir/properties/timing.rs | 1640 ----- .../types/src/ir/schema/family_capability.rs | 21 +- crates/types/src/ir/schema/mod.rs | 10 +- crates/types/src/ir/schema/schema_support.rs | 2 - crates/types/src/ir/schema/sketch.rs | 58 +- crates/types/src/lib.rs | 4 +- crates/types/src/physical/materialization.rs | 57 - crates/types/src/physical/mod.rs | 14 +- crates/types/src/physical/properties.rs | 228 + crates/types/src/physical/workload_plan.rs | 27 +- crates/types/src/stages/candidates.rs | 286 +- .../src/stages/logical_memo_candidates.rs | 26 +- crates/types/src/stages/memo.rs | 59 +- crates/types/src/stages/mod.rs | 29 +- ...per_computation_logical_memo_candidates.rs | 28 +- crates/types/src/stages/physical/graph.rs | 183 +- .../src/stages/physical/memo_candidates.rs | 294 +- crates/types/src/stages/physical/mod.rs | 17 +- crates/types/src/stages/physical/producers.rs | 206 - crates/types/src/stages/pruning.rs | 63 +- crates/types/src/stages/realizer.rs | 4 +- crates/types/src/test_support.rs | 54 +- crates/types/src/workload/bounds.rs | 4 +- crates/types/tests/main.rs | 1 - crates/types/tests/materialized_view.rs | 512 -- crates/types/tests/physical_export.rs | 364 +- crates/types/tests/schema_rebuilding.rs | 127 +- crates/types/tests/structure_contract.rs | 45 +- crates/types/tests/summary_coverage.rs | 61 +- crates/types/tests/summary_merge_structure.rs | 102 +- crates/types/tests/tumbling_window_chain.rs | 148 +- .../architecture/input-output-workflow.md | 2 +- .../architecture/physical-plan-integration.md | 8 +- .../architecture/planner-runtime-contract.md | 2 +- ...d_interface_with_pluggable_optimization.md | 4 +- docs/design_docs/concepts/glossary.md | 2 +- docs/design_docs/concepts/post-asap-ir.md | 108 +- .../physical-planning-and-deployment.md | 27 +- .../maintained-populations.md | 4 +- .../proposals/asap-primitive-schema.md | 5 +- .../joint-planning-implementation-plan.md | 4 +- .../proposals/joint-planning-search.md | 16 +- .../design_docs/proposals/operator-sharing.md | 3 + .../planner-layering-example1-acceptance.md | 36 +- .../planner-layering-example2-acceptance.md | 14 +- .../planner-layering-example3-acceptance.md | 4 +- .../planner-layering-example4-acceptance.md | 22 +- .../design_docs/proposals/planner-layering.md | 69 +- .../window-materialization-choices.md | 2 +- .../asap-aware-mapping-architecture.md | 16 +- .../asap-aware-mapping-contracts.md | 17 +- .../develop_docs/extend-asap-aware-mapping.md | 11 +- docs/develop_docs/library-api.md | 8 +- .../develop_docs/physical-compile-coverage.md | 2 +- docs/develop_docs/pre-asap-ir.md | 13 +- .../target-candidate-api-migration.md | 2 +- ...ng-window-search-compression-evaluation.md | 2 +- tools/dag-viewer/README.md | 4 +- tools/dag-viewer/app.js | 6 +- tools/dag-viewer/memo.js | 4 +- tools/dag-viewer/stages.js | 7 +- .../test-fixtures/memo-enumeration-small.json | 2 +- .../dag-viewer/test-fixtures/memo-small.json | 2 +- 288 files changed, 2764 insertions(+), 62458 deletions(-) delete mode 100644 crates/cost/tests/held_memory.rs delete mode 100644 crates/cost/tests/ingestion_refresh_runs.rs delete mode 100644 crates/devtools/src/memo_pricing.rs delete mode 100644 crates/devtools/tests/memo_pricing.rs delete mode 100644 crates/executor/tests/precompute_candidates.rs delete mode 100644 crates/integration-tests/tests/declared_group_counts.rs delete mode 100644 crates/integration-tests/tests/deployment_capabilities.rs delete mode 100644 crates/integration-tests/tests/frequency_execution.rs delete mode 100644 crates/integration-tests/tests/having_shared_aggregate.rs delete mode 100644 crates/integration-tests/tests/materialization_e2e.rs delete mode 100644 crates/integration-tests/tests/memo_enumeration.rs delete mode 100644 crates/integration-tests/tests/planner_layering_example1.rs delete mode 100644 crates/integration-tests/tests/planner_layering_example2.rs delete mode 100644 crates/integration-tests/tests/planner_layering_example3.rs delete mode 100644 crates/integration-tests/tests/planner_layering_example4.rs delete mode 100644 crates/integration-tests/tests/precompute_raw_samples.rs delete mode 100644 crates/integration-tests/tests/range_evaluation.rs delete mode 100644 crates/integration-tests/tests/reuse_rules.rs delete mode 100644 crates/integration-tests/tests/runtime_handoff.rs delete mode 100644 crates/integration-tests/tests/sketch_rollup_planning.rs delete mode 100644 crates/integration-tests/tests/sql_group_bound.rs delete mode 100644 crates/integration-tests/tests/sql_to_physical.rs delete mode 100644 crates/integration-tests/tests/sql_trailing_window.rs delete mode 100644 crates/integration-tests/tests/sql_two_percentiles.rs delete mode 100644 crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs delete mode 100644 crates/integration-tests/tests/tumbling_window_merge.rs delete mode 100644 crates/logical-asap-optimization/src/pass1/memo_population.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/finer_aggregate.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/enumeration.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs delete mode 100644 crates/logical-asap-optimization/src/pass2/memo/state_groups.rs delete mode 100644 crates/physical-asap-optimization/src/catalog/configurations.rs delete mode 100644 crates/physical-asap-optimization/src/catalog/factoring.rs delete mode 100644 crates/physical-asap-optimization/src/catalog/inventory.rs delete mode 100644 crates/physical-asap-optimization/src/catalog/pruning.rs delete mode 100644 crates/physical-asap-optimization/src/catalog/reconstruction.rs delete mode 100644 crates/physical-asap-optimization/src/split_by_timing.rs delete mode 100644 crates/plan-selection/src/accuracy_binding.rs delete mode 100644 crates/plan-selection/src/checks.rs delete mode 100644 crates/plan-selection/src/groups.rs delete mode 100644 crates/plan-selection/src/groups/accuracy_positions.rs delete mode 100644 crates/plan-selection/src/groups/tests.rs delete mode 100644 crates/plan-selection/src/milp/completion.rs delete mode 100644 crates/plan-selection/src/milp/cost_terms.rs delete mode 100644 crates/plan-selection/src/milp/decode.rs delete mode 100644 crates/plan-selection/src/milp/lp_diagnosis.rs delete mode 100644 crates/plan-selection/src/milp/model/tests.rs delete mode 100644 crates/plan-selection/src/milp/warm_start.rs delete mode 100644 crates/plan-selection/src/select/tests.rs delete mode 100644 crates/plan-selection/src/selector.rs delete mode 100644 crates/planner/tests/deployment_accuracy_model.rs delete mode 100644 crates/planner/tests/per_sample_updater.rs delete mode 100644 crates/planner/tests/sql_per_group_topk.rs delete mode 100644 crates/types/src/ir/properties/timing.rs delete mode 100644 crates/types/src/physical/materialization.rs create mode 100644 crates/types/src/physical/properties.rs delete mode 100644 crates/types/src/stages/physical/producers.rs delete mode 100644 crates/types/tests/materialized_view.rs diff --git a/Cargo.lock b/Cargo.lock index bb23328cc..933b01efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,12 +434,7 @@ dependencies = [ name = "asap-physical-asap-optimization" version = "0.1.0" dependencies = [ - "asap-accuracy", - "asap-cost", - "asap-logical-asap-optimization", "asap-types", - "rmp-serde", - "serde_json", "thiserror 2.0.18", ] @@ -449,14 +444,9 @@ version = "0.1.0" dependencies = [ "asap-accuracy", "asap-cost", - "asap-frontend-promql", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", "asap-types", - "asap_sketchlib", "good_lp", "highs", - "rmp-serde", "serde", "serde_json", "thiserror 2.0.18", @@ -476,7 +466,6 @@ dependencies = [ "asap-logical-asap-optimization", "asap-physical-asap-optimization", "asap-plan-selection", - "asap-planner", "asap-types", "futures", "serde", diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/estimators/cms.rs index 663749928..ebce49b97 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/estimators/cms.rs @@ -42,14 +42,11 @@ mod tests { #[test] fn local_guarantee_inverts_frequency_sizing() { use crate::estimators::size_params; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = asap_types::ir::operator::agg_intent::default_cardinality(); let params = size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Cms, + ¶ms, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: None, @@ -68,17 +65,14 @@ mod tests { #[test] fn heap_evaluation_retains_frequency_metric() { - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let cms_heap = SketchParams::CmsWithHeap { width: 272, depth: 5, heap_size: 10, }; - let topk_frequency = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CmsWithHeap, cms_heap), - GroupingStrategy::default(), - ), + let topk_frequency = sketch_guarantee( + &SketchAlgorithm::CmsWithHeap, + &cms_heap, &SketchStatistic::TopK { k: 10 }, ) .expect("heap sketch still provides per-key frequency intervals"); diff --git a/crates/accuracy/src/estimators/count_sketch.rs b/crates/accuracy/src/estimators/count_sketch.rs index dac391fe0..433803db3 100644 --- a/crates/accuracy/src/estimators/count_sketch.rs +++ b/crates/accuracy/src/estimators/count_sketch.rs @@ -157,14 +157,11 @@ mod tests { fn count_sketch_uses_an_l2_guarantee() { use crate::estimators::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let intent = default_cardinality(); let count_sketch = size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); - let guarantee = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::CountSketch, count_sketch), - GroupingStrategy::default(), - ), + let guarantee = sketch_guarantee( + &SketchAlgorithm::CountSketch, + &count_sketch, &SketchStatistic::PointCount { key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, value: Some("1".into()), diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/estimators/hll.rs index 18d45db02..65ff8512f 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/estimators/hll.rs @@ -81,6 +81,7 @@ pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str /// distinct bound ([`classic_failure_statistic`]). The failure probability /// is per output row: each group's estimate fails with at most it. `None` /// for a precision outside the contract's. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn classic_guarantee( precision: u8, relative_error: Option, @@ -479,14 +480,11 @@ mod tests { fn generic_rse_sizing_does_not_certify_confidence() { use crate::estimators::size_params; use asap_types::ir::operator::agg_intent::default_cardinality; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let c = default_cardinality(); let params = size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Hll, + ¶ms, &SketchStatistic::Cardinality, ) .unwrap(); diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index 92bee9d84..068a49121 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -216,14 +216,11 @@ mod tests { fn local_guarantee_inverts_rank_sizing() { use crate::estimators::size_params; use asap_types::ir::operator::agg_intent::default_quantile; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; let q = default_quantile(0.99); let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = local_guarantee( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ), + let g = sketch_guarantee( + &SketchAlgorithm::Kll, + ¶ms, &SketchStatistic::Quantile { q: 0.99 }, ) .unwrap(); diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 6f5dd4f08..27b6197c6 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -11,6 +11,7 @@ pub mod hll; pub(crate) mod kll; pub mod univmon; +#[allow(dead_code)] // Read again for the error bound of each listed θ (`parameter_list`), in #1207 PRs 3–4. pub(super) fn sketch_guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, @@ -28,6 +29,9 @@ pub(super) fn sketch_guarantee( } SketchParams::Kmv { .. } => cardinality::guarantee(algorithm, params, query), SketchParams::UnivMon { .. } => univmon::guarantee(algorithm, params, query), + SketchParams::Hydra(hydra) => { + sketch_guarantee(algorithm, &hydra.per_subpopulation(), query) + } } } @@ -71,8 +75,10 @@ pub(crate) fn local_guarantee( FieldDataType::ExactAggregate(kind, _) => Some(AccuracyGuarantee::exact(format!( "ExactAggregate({kind:?})" ))), - FieldDataType::Sketch(kind, _) if FamilyCapability::of(family).answers(query) => { - sketch_guarantee(kind.algorithm(), kind.params(), query) + FieldDataType::Sketch(..) if FamilyCapability::of(family).answers(query) => { + todo!( + "#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters" + ) } FieldDataType::Sketch(..) => None, // No error model is registered for these families. @@ -136,6 +142,7 @@ pub fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchPara } } +#[allow(dead_code)] // Read again for the predefined θ lists (`parameter_list`), in #1207 PR 1. pub(crate) fn size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -701,25 +708,13 @@ mod tests { #[test] fn local_guarantee_needs_a_statistic_the_family_answers() { use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, GroupingStrategy, SketchKind, - }; - let hll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ), - GroupingStrategy::default(), - ); + use asap_types::ir::schema::{DataType, ExactKind, ExactParams, GroupingStrategy}; + let hll = FieldDataType::Sketch(SketchAlgorithm::Hll, GroupingStrategy::default()); let total = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }; assert!(local_guarantee(&hll, &total).is_none()); - assert!(local_guarantee(&hll, &SketchStatistic::Cardinality).is_some()); for exact in [ FieldDataType::Plain(DataType::Float64), FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index c9378c1d0..899ced4c6 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -213,6 +213,7 @@ fn distinct_heap(eps: f64, delta: f64) -> u32 { /// The statistic a UnivMon distinct count over `source` leaves its /// top-layer failure probability as. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: &str) -> String { format!("{INCOMPLETE_TOP_LAYER}[k={heap_size},layers={layers}]({MAX_DISTINCT}[{source}])") } @@ -257,6 +258,7 @@ pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct /// `δ_c` plus the chance of an incomplete top layer, left symbolic in the /// source's distinct bound. `None` for a shape that was not sized for a /// distinct count, a heap under 8, or a shape the library would not build. +#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. pub(crate) fn distinct_guarantee(params: &SketchParams, source: &str) -> Option { let SketchParams::UnivMon { heap_size, diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index ed33978df..be95ce58e 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -18,8 +18,8 @@ use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, Scala use asap_types::ir::schema::{DataType, Schema}; use asap_types::ir::schema::{ ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergedError, - NonNegativeWeightProof, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - SummaryInputExpr, SummaryUpdate, WeightDomain, + NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, + WeightDomain, }; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, @@ -31,7 +31,7 @@ use crate::composition::{ PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; use crate::estimators::local_guarantee; -use crate::estimators::{hll, univmon}; + use crate::function_rules::approximate_input_composition; use crate::PropagationStats; @@ -122,7 +122,7 @@ pub fn root_guarantee(root: &QueryRoot) -> Option { /// Bind known statistics in existing contracts, then recover compositions /// that previously required constant child bounds. Physical state contracts, -/// schemas, timing, materialization, and sharing are preserved. This does not +/// schemas and sharing are preserved. This does not /// infer facts: `bind` must leave unsupported statistics unknown. pub fn bind_guarantees( root: &QueryRoot, @@ -137,9 +137,7 @@ pub fn bind_guarantees( return done.clone(); } let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization); + let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); rebuilt.guarantee = node .guarantee .as_ref() @@ -159,90 +157,6 @@ pub fn bind_guarantees( root_guarantee(&bound) } -/// `roots` with every sketch summary state rebuilt with the parameters -/// `resize` gives it (same algorithm; `None` keeps them), and every node -/// above one rebuilt over its new inputs with its guarantee composed again -/// ([`composed_guarantee`]; a node that had none keeps none). Timing, -/// materialization and sharing are kept. Pass 1 sizes summaries -/// for a failure probability this way. -pub fn with_resized_summaries( - roots: &[QueryRoot], - resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, -) -> Result, SchemaDerivationError> { - fn walk( - node: &Rc, - resize: &dyn Fn(&OperatorNode, &SketchParams) -> Option, - cache: &mut NodeToRewrittenNodeCache, - ) -> Result, SchemaDerivationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - walk(child, resize, cache)?; - } - let mut changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); - let mut operator = node.operator.clone(); - if let Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) = &mut operator - { - if let Some(params) = resize(node, kind.params()).filter(|p| p != kind.params()) { - *kind = SketchKind::new(*kind.algorithm(), params); - changed = true; - } - } - let rebuilt = if changed { - let template = OperatorNode { - operator, - ..(**node).clone() - }; - let mut rebuilt = - template.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?; - rebuilt.guarantee = node - .guarantee - .as_ref() - .and_then(|_| composed_guarantee(&rebuilt)); - Rc::new( - rebuilt - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - node.clone() - }; - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) - } - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, resize, &mut cache)?), - QueryRoot::Scalar(expr) => { - let mut rebuilt = HashMap::new(); - for node in expr.operator_refs() { - rebuilt.insert(Rc::as_ptr(node), walk(node, resize, &mut cache)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| rebuilt[&Rc::as_ptr(node)].clone()), - ) - } - }) - }) - .collect() -} - /// The one source a summary state counts the distinct items of its own /// rows of, as `source_distinct_guarantee` reads it: the source whose /// distinct bound gives its failure probability. `None` for a state over @@ -653,10 +567,10 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option, ) -> Option { - let FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) = family else { + let FieldDataType::Sketch(algorithm, GroupingStrategy::PerSubpopulationInstance) = family + else { return None; }; if *query != SketchStatistic::Cardinality { return None; } source_scan(input)?; - match kind.params() { - SketchParams::Hll { - precision, - relative_error, - } => hll::classic_guarantee(*precision, *relative_error, &source(input)?), - params @ SketchParams::UnivMon { .. } => { - univmon::distinct_guarantee(params, &source(input)?) + match algorithm { + SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { + todo!( + "#1207 PR 4: the distinct-count guarantee of θ from LogicalProperties::parameters" + ) } _ => None, } @@ -1219,10 +1132,10 @@ pub(crate) fn stream_norm(node: &Rc) -> Option { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + family: FieldDataType::Sketch(algorithm, _), .. }) if matches!( - kind.algorithm(), + algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch @@ -1598,8 +1511,8 @@ pub(crate) fn estimate_has_bound( ) -> bool { let count_min = matches!( family, - FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) + FieldDataType::Sketch(algorithm, _) + if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) ); !matches!(query, SketchStatistic::TopK { .. }) && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) @@ -1616,8 +1529,7 @@ fn interpolation_guarantee( input: &OperatorNode, mut guarantee: AccuracyGuarantee, ) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(kind, _) - if *kind.algorithm() == SketchAlgorithm::DDSketch) + let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch, _)) && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { return Some(guarantee); diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index 4960f3d14..cdda5845c 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -3,21 +3,11 @@ //! family sized for one equal share of that target, and the failure //! probability each approximate input needs over the values the target //! covers. -use std::rc::Rc; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, -}; -use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; -use asap_types::workload::bounds::{series_read, values_per_result}; -use asap_types::workload::DataWorkload; - -use crate::estimators::{hll, local_guarantee, with_failure_probability}; -use crate::per_input_target; -use crate::propagation::{approximate_inputs, distinct_bound_source, with_resized_summaries}; +use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; /// Confidence δ assumed when the target carries only an ε /// (`AccuracyTarget::epsilon`): the (ε, δ)-parameterised sketches (CMS) need @@ -99,6 +89,7 @@ pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { /// smallest parameter satisfying the target, clamped to the family's sane /// range. A non-positive ε saturates to the clamp maximum (tightest /// allowed). +#[allow(dead_code)] // Read by `test_support` until #1207 PR 1 replaces it with `parameter_list`. pub(crate) fn default_size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -108,175 +99,6 @@ pub(crate) fn default_size_params( crate::estimators::size_params(kind, intent, eps, delta) } -/// `algorithm` sized for one of `share` equal shares of `intent`'s accuracy -/// requirement ([`per_input_target`]), split in the metric of the summary's own -/// bound: a relative error splits so products and quotients of the shares -/// meet it. -pub fn sized(algorithm: &SketchAlgorithm, intent: &AggIntent, share: usize) -> SketchKind { - let target = sizing_target(intent).unwrap_or_else(AccuracyTarget::exact); - let size = |target: &AccuracyTarget| { - let (epsilon, delta) = accuracy_budget(target); - SketchKind::new( - *algorithm, - default_size_params(*algorithm, intent, epsilon, delta), - ) - }; - let whole = size(&target); - if share <= 1 { - return whole; - } - let metric = sketch_statistic(intent) - .and_then(|query| { - local_guarantee( - &FieldDataType::Sketch(whole.clone(), GroupingStrategy::default()), - &query, - ) - }) - .map(|guarantee| guarantee.spec.accuracy_metric) - .or(target.accuracy_metric); - // The split needs a concrete metric. A summary without an error model - // whose intent's target sets none is not split (never a guessed - // metric): it stays sized for the whole target, and Stage 3 rejects the - // candidate if its composed output then misses the query's target. - let Some(metric) = metric else { - debug_assert!( - target.is_exact(), - "sizing {algorithm:?} for a share of an approximate target without a metric" - ); - return whole; - }; - per_input_target(&target, metric, share).map_or(whole, |share| size(&share)) -} - -/// The failure probability each of the `budget_parts` approximate inputs -/// of query `root`'s value is sized for, so that the value meets `target` -/// over every value the target covers: `δ / (n · V)`, the query's `δ` -/// shared equally by `n = budget_parts` inputs and union-bounded over the -/// `V` values its scope covers -/// ([`AccuracyScope::max_values_covered`](asap_types::ir::properties::AccuracyScope::max_values_covered): -/// 1 for a per-value target, at most `⌊1 / share⌋` for one over the groups -/// holding a record share, and `G` ([`values_per_result`]) for the whole -/// result). `None` for an exact target, one without a positive failure -/// probability, and a whole-result target whose `G` is unbound: its target -/// then stays unmet, and plan selection rejects the plan. -pub fn per_input_failure_probability( - target: &AccuracyTarget, - budget_parts: usize, - root: &QueryRoot, - data: &DataWorkload, -) -> Option { - let delta = target - .failure_probability - .filter(|delta| !target.is_exact() && *delta > 0.0)?; - let values = target.accuracy_scope.max_values_covered(values_per_result( - root, - series_read(root, data), - data, - ))?; - Some(delta / (budget_parts.max(1) as f64 * values as f64)) -} - -/// Sketch summary `state`'s `params` sized for failure probability -/// `failure_probability`: the parameters that depend on it enlarged -/// ([`with_failure_probability`]), and classic HLL's precision from the -/// distinct bound of the source it counts (keeping the relative error it -/// was sized for). Never smaller than `params`. `None` keeps `params`: an -/// HLL whose source declares no distinct bound. -fn sized_for_failure_probability( - state: &OperatorNode, - params: &SketchParams, - failure_probability: f64, - data: &DataWorkload, -) -> Option { - Some(match params { - SketchParams::Hll { - precision, - relative_error, - } => { - let source = distinct_bound_source(state)?; - let bound = data.sources.get(&source)?.max_distinct_per_evaluation?; - let relative_error = hll::classic_relative_error(*precision, *relative_error); - SketchParams::Hll { - precision: hll::min_precision( - relative_error, - failure_probability, - bound, - *precision, - ), - relative_error: Some(relative_error), - } - } - params => with_failure_probability(params, failure_probability), - }) -} - -/// `roots` with every approximate input of their values -/// ([`approximate_inputs`]) sized for `failure_probability` -/// (`sized_for_failure_probability`) and the guarantees above composed -/// again ([`with_resized_summaries`]). A summary whose output carries no -/// guarantee (e.g. a top-k's heap, which bounds item frequencies, not -/// membership) is no approximate input and keeps its size: its failure -/// probability bounds nothing the target checks. The same `Rc` where -/// nothing changes. -pub fn with_summaries_sized_for_failure_probability( - roots: &[QueryRoot], - failure_probability: f64, - data: &DataWorkload, -) -> Result, SchemaDerivationError> { - let inputs: std::collections::HashSet<*const OperatorNode> = roots - .iter() - .flat_map(approximate_inputs) - .map(|state| Rc::as_ptr(&state)) - .collect(); - if inputs.is_empty() { - return Ok(roots.to_vec()); - } - with_resized_summaries(roots, &|state, params| { - inputs - .contains(&std::ptr::from_ref(state)) - .then(|| sized_for_failure_probability(state, params, failure_probability, data)) - .flatten() - }) -} - -/// Whole-workload `roots` (query `i`'s root `roots[i]` with target -/// `targets[i]`), composed outside the memo, with their summaries sized as -/// Pass 1 sizes a memo candidate: each approximate input of query `i`'s -/// value, `n_i` in all ([`approximate_inputs`]), for -/// [`per_input_failure_probability`] `δ_i / (n_i · V_i)`, a summary several -/// roots read for the smallest among them, and the guarantees above -/// composed again. For plans built by hand (tests, the fuzzer's witnesses); -/// the planner sizes its candidates in Pass 1. -pub fn with_summaries_sized_for_targets( - roots: &[QueryRoot], - targets: &[AccuracyTarget], - data: &DataWorkload, -) -> Result, SchemaDerivationError> { - let mut required: std::collections::HashMap<*const OperatorNode, f64> = - std::collections::HashMap::new(); - for (root, target) in roots.iter().zip(targets) { - let inputs = approximate_inputs(root); - let Some(failure_probability) = - per_input_failure_probability(target, inputs.len(), root, data) - else { - continue; - }; - for state in &inputs { - let entry = required - .entry(Rc::as_ptr(state)) - .or_insert(failure_probability); - *entry = entry.min(failure_probability); - } - } - if required.is_empty() { - return Ok(roots.to_vec()); - } - with_resized_summaries(roots, &|state, params| { - let failure_probability = *required.get(&std::ptr::from_ref(state))?; - sized_for_failure_probability(state, params, failure_probability, data) - }) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/cost/Cargo.toml b/crates/cost/Cargo.toml index 1436c8114..8a8e35fa9 100644 --- a/crates/cost/Cargo.toml +++ b/crates/cost/Cargo.toml @@ -22,4 +22,4 @@ asap-executor = { path = "../executor" } # by the stages above this crate; `tests/held_memory.rs` builds them with # the reference pricer's `optimize_physical_asap`. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } diff --git a/crates/cost/src/execution_demand.rs b/crates/cost/src/execution_demand.rs index 6a7e380bd..817dbf84b 100644 --- a/crates/cost/src/execution_demand.rs +++ b/crates/cost/src/execution_demand.rs @@ -9,7 +9,8 @@ use super::analytical::AnalyticalCostError; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::physical::Materialization; +use asap_types::ir::properties::ExecutionTiming; +use asap_types::physical::Lifetime; use asap_types::workload::{QueryRecurrence, RangeEvaluation, RepeatedDemand}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -280,7 +281,7 @@ pub fn query_starts(dag: &PhysicalASAPDAG) -> Vec> { .collect() } -/// What computes a `NotMaterialized` node once per its own execution +/// What computes a not-materialized node once per its own execution /// ([`consumers`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Consumer { @@ -290,19 +291,19 @@ pub enum Consumer { Materialized(PhysicalASAPNodeId), } -/// The consumers of each `NotMaterialized` node of `dag` that any consumer +/// The consumers of each not-materialized node of `dag` that any consumer /// reaches ([`computations`]): the queries and the materialized nodes whose /// walk down their inputs reaches it before meeting a materialized node, /// each once; the queries first, in order, then the materialized nodes in /// `dag`'s node order. pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap> { - let materialized: Vec = dag + let materialized_nodes: Vec = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) + .filter(|n| n.lifetime != Lifetime::None) .map(|n| n.id) .collect(); - let is_materialized: HashSet = materialized.iter().copied().collect(); + let is_materialized: HashSet = materialized_nodes.iter().copied().collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges .iter() @@ -314,7 +315,7 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap HashMap, ) -> Result, AnalyticalCostError> { let range_of = |query: usize| range_evaluations.get(query).copied().flatten(); - let materialized: HashMap<_, _> = dag + // Each materialized node's timing and lifetime. + let materialized_nodes: HashMap = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .map(|n| (n.id, n.materialization)) + .filter(|node| node.lifetime != Lifetime::None) + .map(|node| (node.id, (node.output_state.timing, node.lifetime))) .collect(); let inputs = |id: PhysicalASAPNodeId| { dag.edges @@ -385,14 +387,12 @@ pub fn computations( } } let mut rate_of_materialized = HashMap::new(); - for (&id, materialization) in &materialized { - // A chain kept across evaluations is built once per request. - let per_request = matches!( - materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ); - let rate = match materialization { - Materialization::MatAtIngestion(_) => 1.0, + for (&id, &(timing, lifetime)) in &materialized_nodes { + let at_ingestion = timing == ExecutionTiming::IngestionTime; + // A query-time chain materialized across evaluations is built once per request. + let per_request = !at_ingestion && matches!(lifetime, Lifetime::AcrossEvaluations(_)); + let rate = match at_ingestion { + true => 1.0, _ => evaluation_rate( reached.get(&id).into_iter().flatten().filter_map(|&q| { let range = if per_request { None } else { range_of(q) }; @@ -416,9 +416,9 @@ pub fn computations( } // The refresh window of each node refreshed at ingestion. let refresh_windows = dag.refresh_windows( - materialized + materialized_nodes .iter() - .filter(|(_, m)| matches!(m, Materialization::MatAtIngestion(_))) + .filter(|(_, (timing, _))| *timing == ExecutionTiming::IngestionTime) .map(|(&id, _)| id), ); let mut count: HashMap<_, (u64, f64)> = HashMap::new(); diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index d381c59bb..f921d3bb7 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -41,7 +41,7 @@ pub use measured::{ pub use operator_statistics::EdgeStatistics; pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; pub use usage::{ - family_name, kept_tumbling_window_chains, maintained_window_ms, node_map, node_usage, + family_name, maintained_window_ms, materialized_tumbling_window_chains, node_map, node_usage, plan_cost, query_reach, retained_tumbling_windows, trailing_window, tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, @@ -80,7 +80,7 @@ pub struct ResourceUsage { pub average_memory_bytes: f64, /// The most memory live at once, assuming every evaluation overlaps: /// in-memory kept state plus each computation's working memory, once - /// per consumer that recomputes it, with the `MatAtIngestion` consumers + /// per consumer that recomputes it, with the consumers materialized at ingestion time /// of one refresh window counted once. pub peak_memory_bytes: u64, /// Bytes kept on disk: state materialized on `Medium::Disk`, plus the @@ -375,7 +375,7 @@ pub struct NodeCost { /// node the query reads first. pub cpu_core_seconds_per_second: f64, /// The number of consumers it is computed for, with the - /// `MatAtIngestion` consumers of one refresh window counted once (1 + /// consumers materialized at ingestion time of one refresh window counted once (1 /// when materialized, [`execution_demand::computations`]). pub multiplier: u64, /// How it was measured, e.g. "scan 1000 samples; x 1.0000/s; keeps diff --git a/crates/cost/src/state_bytes.rs b/crates/cost/src/state_bytes.rs index a8d4c8fba..4b7d297e6 100644 --- a/crates/cost/src/state_bytes.rs +++ b/crates/cost/src/state_bytes.rs @@ -60,7 +60,10 @@ pub fn keeps_every_value(measure: &AggIntent) -> bool { /// 1 for every other sketch. pub fn summary_shape(family: &FieldDataType, input_rows: StateInputRows) -> (f64, u64) { match family { - FieldDataType::Sketch(kind, _) => sketch_shape(kind.params(), input_rows), + FieldDataType::Sketch(..) => { + let _ = input_rows; + todo!("#1207 PR 6: the state shape of θ from the candidate's LogicalProperties::parameters") + } FieldDataType::ExactAggregate(..) => (1.0, EXACT_ACCUMULATOR_STATE_BYTES), _ => (1.0, 8), } @@ -104,6 +107,7 @@ pub fn sketch_shape(params: &SketchParams, input_rows: StateInputRows) -> (f64, usize::try_from(rows.max(1)).unwrap_or(usize::MAX) }); match params { + SketchParams::Hydra(hydra) => sketch_shape(&hydra.per_subpopulation(), input_rows), SketchParams::Cms { width, depth } => ( f64::from(*depth), CountMinF64::estimated_memory_bytes(*depth as usize, *width as usize) as u64, @@ -270,7 +274,7 @@ const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; #[cfg(test)] mod tests { use super::*; - use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind}; + use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm}; /// An exact accumulator's state is priced at the executor's per-group /// state, 56 bytes for every kind (a counter's state alone is 48), so a @@ -353,121 +357,6 @@ mod tests { assert_eq!(priced(Some(10_000), false), priced(None, false)); } - /// DDSketch state grows as its relative accuracy α tightens. - #[test] - fn ddsketch_state_grows_as_alpha_shrinks() { - let bytes = |alpha| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert!( - bytes(0.1) < bytes(0.01), - "{} vs {}", - bytes(0.1), - bytes(0.01) - ); - assert!( - bytes(0.01) < bytes(0.001), - "{} vs {}", - bytes(0.01), - bytes(0.001) - ); - } - - /// Every sketch family's state is sized from its parameters, as - /// `asap_sketchlib` allocates it: an HLL's registers, a KMV's `k` - /// hashes once it has seen `k` distinct items, and a UnivMon's layers - /// once keys of the assumed size have churned through every heap. - #[test] - fn summary_state_is_sized_from_sketch_params() { - use asap_sketchlib::DataInput; - let bytes = |algorithm, params| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert_eq!( - bytes( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 14, - relative_error: None - } - ), - asap_sketchlib::HyperLogLog::::new().memory_bytes() as u64 - ); - let mut kmv: asap_sketchlib::Kmv = asap_sketchlib::Kmv::new(1_000); - for i in 0..2_000u64 { - kmv.update(&DataInput::U64(i)); - } - assert_eq!( - bytes( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 1_000, - relative_error: None - } - ), - kmv.memory_bytes() as u64 - ); - let mut univmon = asap_sketchlib::UnivMon::new(8, 3, 64, 2); - for i in 0..10_000u64 { - let mut key = [0; ASSUMED_VALUE_KEY_BYTES]; - key[..8].copy_from_slice(&i.to_le_bytes()); - univmon.update_weighted(&DataInput::Bytes(&key), 1); - } - assert_eq!( - bytes( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 8, - sketch_rows: 3, - sketch_cols: 64, - layers: 2, - distinct_failure_probability: None, - } - ), - univmon.memory_bytes() as u64 - ); - } - - /// KLL state whose samples are unknown is charged its fixed 744 bytes - /// and the item array at the capacity `asap_sketchlib` grows it to: - /// `compute_max_capacity(k, 8)` 8-byte items, with `k` capped at 26 602. - #[test] - fn kll_state_is_sized_as_asap_sketchlib_allocates_it() { - let bytes = |k| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .1 - }; - assert_eq!(bytes(269), 744 + 8 * 1_206); - assert_eq!(bytes(26_602), 744 + 8 * 80_118); - assert_eq!(bytes(30_000), 744 + 8 * 80_118); - } - /// A KLL state is charged what `asap_sketchlib` allocates for the /// samples it sees, not its full capacity: its fixed struct and level /// array, plus the item array the library starts empty and doubles from @@ -727,18 +616,19 @@ mod tests { assert_eq!(key.to_bytes().unwrap().len(), ASSUMED_HEAP_KEY_BYTES); for (algorithm, params, rows, priced_as, fields, full) in &families { for n in [1, 100, *full] { - let family = FieldDataType::Sketch( - SketchKind::new(*algorithm, params.clone()), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(*algorithm, GroupingStrategy::default()); let input = match rows { Rows::Values => &value, Rows::Labels => &labeled, Rows::Keyed => &keyed, }; - let mut updater = - create_planner_accumulator(&family, input, &GroupingStrategy::default()) - .unwrap(); + let mut updater = create_planner_accumulator( + &family, + Some(params), + input, + &GroupingStrategy::default(), + ) + .unwrap(); for i in 0..n { match rows { // Distinct magnitudes over the assumed range, in a @@ -802,34 +692,4 @@ mod tests { } } } - - /// A UnivMon row updates layers `0..=b`, where its bottom layer `b` is - /// geometric (`asap_sketchlib`'s `bottom_layer_for_hash`), so - /// `2 - 2^(1 - layers)` layers in expectation; each updates one counter - /// per sketch row and its heap. - #[test] - fn univmon_update_cost_is_expected_layers_times_rows_and_heap() { - let ops = |layers| { - summary_shape( - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1_024, - layers, - distinct_failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - StateInputRows::UNKNOWN, - ) - .0 - }; - assert_eq!(ops(1), 6.0); - assert_eq!(ops(2), 1.5 * 6.0); - assert_eq!(ops(16), (2.0 - 2f64.powi(-15)) * 6.0); - } } diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index f811145d4..1932529a1 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -30,7 +30,7 @@ use asap_types::ir::schema::{ FieldDataType, GroupingStrategy, Schema, SketchParams, SketchStatistic, }; use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; -use asap_types::physical::{Materialization, Medium}; +use asap_types::physical::{Lifetime, Medium}; use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; use asap_types::workload::bounds::{ @@ -46,15 +46,15 @@ use asap_types::workload::bounds::{ /// ([`computations`]); an ingestion-time node over one second of arriving /// rows. A materialized node's kept state is counted in byte-seconds per /// second; a tumbling-window chain kept beyond one evaluation keeps the tumbling windows its -/// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `kept_chains` are -/// `dag`'s ([`node_map`], [`kept_tumbling_window_chains`]); `recurrence[q]` and +/// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `materialized_chains` are +/// `dag`'s ([`node_map`], [`materialized_tumbling_window_chains`]); `recurrence[q]` and /// `range_evaluations[q]` say how often query `q` runs and how many steps /// each run evaluates (one when missing). #[allow(clippy::too_many_arguments)] pub fn plan_cost( dag: &PhysicalASAPDAG, nodes: &NodeMap<'_>, - kept_chains: &HashSet, + materialized_chains: &HashSet, recurrence: &[QueryRecurrence], range_evaluations: &[Option], data: &DataWorkload, @@ -77,7 +77,7 @@ pub fn plan_cost( } = each_node_usage( dag, nodes, - kept_chains, + materialized_chains, recurrence, range_evaluations, data, @@ -94,8 +94,8 @@ pub fn plan_cost( // Each node's work in ms on one core: one computation at query time, // one second of arriving rows at ingestion time. let mut work_ms: HashMap = HashMap::new(); - // Query-time memory held only while one evaluation runs, a MatAtQuery - // state in memory or a NotMaterialized node's working memory: (bytes, + // Query-time memory held only while one evaluation runs, a OneEvaluation + // state in memory or a not-materialized node's working memory: (bytes, // per second, whether each consumer computes its own copy), // counted once the evaluations' latencies are known, in node order so // the sum is the same in every run. @@ -246,13 +246,13 @@ pub fn plan_cost( for (latency, evaluation_ms) in latency.iter_mut().zip(evaluation_ms_per_query) { latency.query_ms += evaluation_ms; } - // Query-time memory is held for one evaluation: a MatAtQuery state from + // Query-time memory is held for one evaluation: a `OneEvaluation` state from // its build until the longest query reading it finishes, about that - // query's latency, once per execution. A NotMaterialized node's working + // query's latency, once per execution. A not-materialized node's working // memory is computed once per consumer ([`consumers`]), and each copy is // held while its consumer's evaluation runs: a query's for that query's // latency, at its rate; a materialized node's until the longest query - // reading that node finishes, at the node's rate, as its MatAtQuery + // reading that node finishes, at the node's rate, as its `OneEvaluation` // state is. let longest_reader_ms = |id: PhysicalASAPNodeId| { reach @@ -379,7 +379,7 @@ struct EachNodeUsage<'a> { fn each_node_usage<'a>( dag: &'a PhysicalASAPDAG, nodes: &NodeMap<'a>, - kept_chains: &HashSet, + materialized_chains: &HashSet, recurrence: &[QueryRecurrence], range_evaluations: &[Option], data: &DataWorkload, @@ -423,11 +423,11 @@ fn each_node_usage<'a>( .collect(); let scan_window = match (&node.payload, node.output_state.timing) { (Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - match dag.scan_window(node.id, kept_chains) { + match dag.scan_window(node.id, materialized_chains) { // Priced as before #955 decides `@` anchors and // unwindowed readers: the widest window read. ScanWindow::Unbounded => trailing_window( - widest_window_read_ms(dag, nodes, node.id, kept_chains) + widest_window_read_ms(dag, nodes, node.id, materialized_chains) .or(root_scan_lookback_ms), ), window => window, @@ -475,10 +475,10 @@ fn state_family<'a>( family, grouping, .. }, ) => match family { - FieldDataType::Sketch(kind, _) + FieldDataType::Sketch(..) if *grouping == GroupingStrategy::PerSubpopulationInstance => { - StateFamily::Sketch(kind.params()) + todo!("#1207 PR 6: the state family of θ from the candidate's LogicalProperties::parameters") } FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), _ => StateFamily::None, @@ -957,13 +957,13 @@ pub fn node_usage<'a>( // query time (all of them for a chain kept across // evaluations, whose scan Stage 2 bounds to its newest tumbling window, // the only one it builds). - let rows_per_tumbling_window = match (ingestion, node.materialization) { + let rows_per_tumbling_window = match (ingestion, node.lifetime) { (true, _) => u64::try_from( (u128::from(input.rows) * u128::from(*tumbling_window_width_ms)) .div_ceil(1_000), ) .map_err(|_| AnalyticalCostError::Overflow)?, - (false, Materialization::MatAtQueryKeptAcrossEvaluations(_)) => input.rows, + (false, Lifetime::AcrossEvaluations(_)) => input.rows, (false, _) => input .rows .div_ceil(u64::from(*tumbling_window_count).max(1)), @@ -1128,10 +1128,10 @@ pub fn node_usage<'a>( // consumer that recomputes it; at query time, that is held for the // evaluation too (counted in [`plan_cost`], per consumer: a query's // latency, or a materialized consumer's longest reader's, as a - // MatAtQuery state is). + // OneEvaluation state is). let state = out.bytes.max(estimate.peak_memory_bytes()); - let (kept, medium) = match (node.materialization, &node.payload) { - (Materialization::NotMaterialized, _) => { + let (materialized_bytes, medium) = match (ingestion, node.lifetime, &node.payload) { + (_, Lifetime::None, _) => { let working_memory_bytes = estimate.peak_memory_bytes(); peak_memory_bytes = working_memory_bytes.saturating_mul(multiplier); if !ingestion && working_memory_bytes > 0 { @@ -1139,11 +1139,15 @@ pub fn node_usage<'a>( } (0, Medium::Memory) } - // A tumbling-window chain kept across evaluations holds the tumbling windows its + // A tumbling-window chain materialized across evaluations holds the tumbling windows its // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window // width of the readers' range-evaluation span), all but the // newest between evaluations and all while one runs. - (Materialization::MatAtQueryKeptAcrossEvaluations(medium), _) => { + ( + false, + Lifetime::AcrossEvaluations(medium), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), + ) => { let tumbling_windows = tumbling_windows_of(node); let retained = retained_tumbling_window_count; let between = (state / tumbling_windows) @@ -1165,10 +1169,10 @@ pub fn node_usage<'a>( } Medium::Disk => disk_bytes = between, } - detail += &format!("; keeps {between} bytes across evaluations"); + detail += &format!("; materializes {between} bytes across evaluations"); (0, medium) } - (Materialization::MatAtQuery(medium), _) => { + (_, Lifetime::OneEvaluation(medium), _) => { match medium { Medium::Memory => { held = Some((state, per_second, false)); @@ -1180,7 +1184,8 @@ pub fn node_usage<'a>( } // The retained tumbling windows and the one being built. ( - Materialization::MatAtIngestion(medium), + true, + Lifetime::AcrossEvaluations(medium), Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), ) => { let tumbling_windows = tumbling_windows_of(node); @@ -1195,23 +1200,24 @@ pub fn node_usage<'a>( medium, ) } - (Materialization::MatAtIngestion(medium), _) => ( + // A whole state materialized across evaluations: it and the one being built. + (_, Lifetime::AcrossEvaluations(medium), _) => ( state.checked_mul(2).ok_or(AnalyticalCostError::Overflow)?, medium, ), }; - if node.materialization == Materialization::MatAtIngestion(Medium::Memory) { + if ingestion && node.lifetime == Lifetime::AcrossEvaluations(Medium::Memory) { maintained = Some(state / tumbling_windows_of(node)); } - if kept > 0 { + if materialized_bytes > 0 { match medium { Medium::Memory => { - average_memory_bytes = kept as f64; - peak_memory_bytes = kept; + average_memory_bytes = materialized_bytes as f64; + peak_memory_bytes = materialized_bytes; } - Medium::Disk => disk_bytes = kept, + Medium::Disk => disk_bytes = materialized_bytes, } - detail += &format!("; keeps {kept} bytes"); + detail += &format!("; keeps {materialized_bytes} bytes"); } if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { if !ingestion { @@ -1399,7 +1405,7 @@ pub fn tumbling_window_retention_ms( let nodes = node_map(dag); dag.nodes .iter() - .filter_map(|node| match (&node.payload, node.materialization) { + .filter_map(|node| match (&node.payload, node.lifetime) { ( Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { child, @@ -1407,8 +1413,7 @@ pub fn tumbling_window_retention_ms( tumbling_window_count, .. }), - Materialization::MatAtIngestion(_) - | Materialization::MatAtQueryKeptAcrossEvaluations(_), + Lifetime::AcrossEvaluations(_), ) => { let span_ms = reach .iter() @@ -1594,21 +1599,22 @@ pub fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { pub type NodeMap<'a> = HashMap; -/// The tumbling-window chains of `dag` kept across evaluations -/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 2 offers only when the +/// The query-time tumbling-window chains of `dag` materialized across evaluations +/// (lifetime `AcrossEvaluations` at query time, which Stage 2 offers only when the /// deployment keeps query-time results and every reader builds one new /// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the /// older ones, which complete the readers' look-back, are kept from /// earlier evaluations. -pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { +pub fn materialized_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { dag.nodes .iter() .filter(|node| { matches!( - (&node.payload, node.materialization), + (&node.payload, node.output_state.timing, node.lifetime), ( Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtQueryKeptAcrossEvaluations(_), + ExecutionTiming::QueryTime, + Lifetime::AcrossEvaluations(_), ) ) }) @@ -1619,14 +1625,14 @@ pub fn kept_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet, scan: PhysicalASAPNodeId, - kept_chains: &HashSet, + materialized_chains: &HashSet, ) -> Option { let mut widest = None; let mut stack = vec![scan]; @@ -1641,7 +1647,7 @@ fn widest_window_read_ms( Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - }) if kept_chains.contains(&edge.consumer) => *tumbling_window_width_ms, + }) if materialized_chains.contains(&edge.consumer) => *tumbling_window_width_ms, Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, tumbling_window_count, @@ -1667,7 +1673,7 @@ pub fn family_name(family: &FieldDataType) -> String { FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { format!("{kind:?}") } - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), + FieldDataType::Sketch(algorithm, _) => format!("{algorithm:?}"), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), other => format!("{other:?}"), } diff --git a/crates/cost/tests/held_memory.rs b/crates/cost/tests/held_memory.rs deleted file mode 100644 index 8bc3978c0..000000000 --- a/crates/cost/tests/held_memory.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! Query-time working memory in [`plan_cost`]: how long a `NotMaterialized` -//! node's working memory is held depends on whether a materialized node -//! reads it. - -use asap_cost::execution_demand::evaluation_rate; -use asap_cost::ResourceUnitCost; -use asap_cost::{ - kept_tumbling_window_chains, node_map, plan_cost, CostInputs, CostModel, PlanCost, -}; -use asap_logical_asap_optimization::test_support::lower_promql; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{ - DataArrival, DataWorkload, QueryRecurrence, RepeatedDemand, RepetitionInterval, SourceFacts, -}; - -/// Data arriving at 10 000 rows per second over 10 000 series of `m`. -fn data() -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 10_000.0, - }, - sources: [( - "m".to_string(), - SourceFacts { - series_count: Some(10_000), - ..SourceFacts::default() - }, - )] - .into_iter() - .collect(), - } -} - -/// Two queries repeating every 10 s and every 20 s. -fn recurrence() -> [QueryRecurrence; 2] { - [10_000, 20_000].map(|interval_ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - interval_ms, - ))) - }) -} - -/// `plan_cost` of `dag` for [`recurrence`], at `cost_per_memory_byte_second`. -fn cost(dag: &PhysicalASAPDAG, cost_per_memory_byte_second: f64) -> PlanCost { - let mut unit_cost = ResourceUnitCost::illustrative(); - unit_cost.cost_per_memory_byte_second = cost_per_memory_byte_second; - let settings = CostInputs { - cost_model: CostModel::UsageOverTime(unit_cost), - ..CostInputs::default() - }; - let nodes = node_map(dag); - plan_cost( - dag, - &nodes, - &kept_tumbling_window_chains(dag), - &recurrence(), - &[], - &data(), - &settings, - 1.0, - ) - .unwrap() - .0 -} - -/// Per node holding query-time memory, (its id, its held bytes, its -/// held-memory cost per second at unit memory cost 1). -fn held_memory_costs(dag: &PhysicalASAPDAG) -> Vec<(usize, f64, f64)> { - let (with, without) = (cost(dag, 1.0), cost(dag, 0.0)); - with.per_node - .iter() - .filter_map(|(id, node)| { - let held = node.detail.split("; holds ").nth(1)?; - let bytes: f64 = held.split(' ').next().unwrap().parse().unwrap(); - let term = - node.cost_per_second.unwrap() - without.per_node[id].cost_per_second.unwrap(); - Some((*id, bytes, term)) - }) - .collect() -} - -fn materialization_of(dag: &PhysicalASAPDAG, id: usize) -> Materialization { - dag.nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .materialization -} - -fn close(a: f64, b: f64) -> bool { - (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) -} - -/// Query 0 is `sum(...)` over query 1's output, so it runs longer. Without -/// a materialized node, each query computes the shared nodes and holds -/// their working memory for its own latency: `bytes · Σ rate_q · latency_q`. -/// When query 1's output is `MatAtQuery`, the nodes below it are computed -/// once per execution of it, at its rate, and held until the -/// longest reader, query 0, finishes, as the `MatAtQuery` state itself is: -/// `bytes · rate · latency_0`, not the rate-weighted mean of the two -/// latencies. -#[test] -fn working_memory_read_by_a_materialized_node_is_held_for_the_longest_reader() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let plans = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap(); - let dag_with = |materialization: Materialization| { - plans - .iter() - .map(|plan| &plan.dag) - .find(|dag| materialization_of(dag, dag.roots[1]) == materialization) - .unwrap() - }; - let recomputed = dag_with(Materialization::NotMaterialized); - let shared = dag_with(Materialization::MatAtQuery(Medium::Memory)); - let latency_ms = |dag| { - let cost = cost(dag, 1.0); - [0, 1].map(|q| cost.usage.latency_per_query[q].total_ms()) - }; - - // Recomputed per query: the rate-weighted sum of both latencies. - let [outer_ms, inner_ms] = latency_ms(recomputed); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let below = |dag: &PhysicalASAPDAG| { - let materialized = dag.roots[1]; - held_memory_costs(dag) - .into_iter() - .filter(move |(id, _, _)| *id != materialized && *id != dag.roots[0]) - .collect::>() - }; - let checked = below(recomputed); - assert!(!checked.is_empty(), "the shared nodes hold memory"); - for (id, bytes, term) in checked { - let expected = bytes * (0.1 * outer_ms + 0.05 * inner_ms) / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - } - - // Below a MatAtQuery node: the longest reader's latency at the - // materialized node's rate (its readers' schedules together). - let materialized_rate = evaluation_rate( - recurrence().iter().map(|r| (r, None)), - CostInputs::default().non_repeating_query_horizon_seconds, - ) - .unwrap(); - let [outer_ms, inner_ms] = latency_ms(shared); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let checked = below(shared); - assert!(!checked.is_empty(), "the nodes below it hold memory"); - for (id, bytes, term) in checked { - assert_eq!( - materialization_of(shared, id), - Materialization::NotMaterialized - ); - let expected = bytes * materialized_rate * outer_ms / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - } -} diff --git a/crates/cost/tests/ingestion_refresh_runs.rs b/crates/cost/tests/ingestion_refresh_runs.rs deleted file mode 100644 index ebd33bab5..000000000 --- a/crates/cost/tests/ingestion_refresh_runs.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! How often a `NotMaterialized` node is computed for the views refreshed -//! at ingestion above it ([`computations`]): once per refresh window, since -//! a runtime refreshes the views of one refresh window in one ingestion run -//! (issue #1151). - -use std::rc::Rc; -use std::time::Duration; - -use asap_cost::execution_demand::computations; -use asap_types::ir::operator::operator_properties::{Reduction, Source, TimeShift}; -use asap_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAG}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, NonASAPOp, Operator, - OperatorNode, QueryRoot, TimeRangeKind, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - -/// `m` with columns `ts`, `value` and `job`. -fn scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -/// A quantile sketch of `algorithm` (KLL k = 200 or DDSketch α = 0.01). -fn quantile_sketch(algorithm: SketchAlgorithm) -> FieldDataType { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} - -/// `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of a quantile sketch of -/// `algorithm` per `job` over `input`. -fn chain( - input: &Rc, - algorithm: SketchAlgorithm, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(input), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms, - tumbling_window_count, - })) - .unwrap() -} - -/// One query per summary of `summaries`, merging it, with each summary -/// materialized as `materialization`: the DAG Stage 3 costs, chains -/// unexpanded. -fn dag_of(summaries: &[Rc], materialization: Materialization) -> PhysicalASAPDAG { - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = summaries - .iter() - .map(|summary| { - if materialization != Materialization::NotMaterialized { - assignment.set(summary, materialization); - } - QueryRoot::Operator( - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(summary)], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - compile_physical_asap_workload(&timed).unwrap() -} - -/// `(multiplier, per_second)` of `dag`'s `Scan`, each query every 10 s. -fn scan_computations(dag: &PhysicalASAPDAG) -> (u64, f64) { - computations_of(dag, |op| matches!(op, NonASAPOp::Scan { .. })) -} - -/// `(multiplier, per_second)` of the one node of `dag` whose operator -/// `is_node` picks, each query every 10 s. -fn computations_of( - dag: &PhysicalASAPDAG, - is_node: impl Fn(&NonASAPOp) -> bool, -) -> (u64, f64) { - let every_10_s = - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))); - let recurrence = vec![every_10_s; dag.roots.len()]; - let times = computations(dag, &recurrence, &[], 3600.0, None).unwrap(); - let node = dag - .nodes - .iter() - .find(|node| matches!(&node.payload, Operator::NonASAP(op) if is_node(op))) - .unwrap(); - times[&node.id] -} - -/// `child` evaluated `offset_ms` earlier. -fn shifted(child: &Rc, offset_ms: i64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child: Rc::clone(child), - })) - .unwrap() -} - -const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - -/// A KLL chain and a DDSketch chain of 10 s tumbling windows over one `Scan`, both -/// kept at ingestion, are refreshed in one ingestion run per tumbling window: the -/// `Scan` is computed once per second of arriving data, not twice. -#[test] -fn a_scan_under_two_chains_of_one_tumbling_window_width_is_computed_once() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (1, 1.0)); -} - -/// With 10 s and 60 s tumbling windows the chains need different runs (the 10 s chain -/// needs five tumbling window ends the 60 s chain does not): the `Scan` is computed -/// once for each. -#[test] -fn a_scan_under_chains_of_two_tumbling_window_widths_is_computed_for_each() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 60_000, 5), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// Query consumers are unchanged: with both chains built at query time -/// (`NotMaterialized`), each query computes the `Scan` at its own rate. -#[test] -fn query_consumers_still_count_once_each() { - let scan = scan(); - let range = |width_ms| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap() - }; - let build = |input: Rc, algorithm| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: input, - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let dag = dag_of( - &[ - build(range(60_000), SketchAlgorithm::Kll), - build(range(60_000), SketchAlgorithm::DDSketch), - ], - Materialization::NotMaterialized, - ); - assert_eq!(scan_computations(&dag), (2, 0.2)); -} - -/// Two 10 s chains and a 60 s chain over one `Scan`: two refresh windows, -/// two ingestion runs. -#[test] -fn a_scan_under_chains_of_10_10_and_60_s_is_computed_for_each_width() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain(&scan, SketchAlgorithm::DDSketch, 10_000, 30), - chain(&scan, SketchAlgorithm::Kll, 60_000, 5), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// A 10 s chain over the `Scan` and a 10 s chain over the `Scan` shifted by -/// 5 s end their tumbling windows on different grids (phases 0 and 5 s): two runs. -#[test] -fn chains_of_one_width_and_two_phases_are_computed_apart() { - let scan = scan(); - let dag = dag_of( - &[ - chain(&scan, SketchAlgorithm::Kll, 10_000, 6), - chain( - &shifted(&scan, 5_000), - SketchAlgorithm::DDSketch, - 10_000, - 30, - ), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (2, 2.0)); -} - -/// Two 10 s chains over one `TimeShift(10 s)` of the `Scan` (phase 0 for -/// both): the shift and the `Scan` below it run once per run. -#[test] -fn nodes_below_a_shared_intermediate_node_are_computed_once_per_run() { - let shift = shifted(&scan(), 10_000); - let dag = dag_of( - &[ - chain(&shift, SketchAlgorithm::Kll, 10_000, 6), - chain(&shift, SketchAlgorithm::DDSketch, 10_000, 30), - ], - AT_INGESTION, - ); - assert_eq!(scan_computations(&dag), (1, 1.0)); - assert_eq!( - computations_of(&dag, |op| matches!(op, NonASAPOp::TimeShift { .. })), - (1, 1.0) - ); -} - -/// Two summaries kept at ingestion over rows with no time window -/// (`Values`) have no refresh window: each counts alone. -#[test] -fn consumers_of_unbounded_windows_count_alone() { - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: scan().schema.clone(), - })) - .unwrap(); - let build = |algorithm| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(&rows), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let dag = dag_of( - &[ - build(SketchAlgorithm::Kll), - build(SketchAlgorithm::DDSketch), - ], - AT_INGESTION, - ); - assert_eq!( - computations_of(&dag, |op| matches!(op, NonASAPOp::Values { .. })), - (2, 2.0) - ); -} diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 763e654fe..bb24e585b 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -13,9 +13,9 @@ asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-accuracy = { path = "../accuracy" } asap-logical-asap-optimization = { path = "../logical-asap-optimization" } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } +asap-plan-selection = { path = "../plan-selection" } # The installable plan of the selected candidate (`installable_plan_of`), # whose stored outputs `asap_planner` writes. asap-planner = { path = "../planner" } @@ -32,14 +32,9 @@ serde_json = "1" serde_yaml = "0.9" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } -[features] -# Each Stage 3 selection MILP solve also reports where its LP relaxation -# loses its bound, with `ASAP_STAGE3_LP_DIAGNOSIS=` (measurement only). -lp-diagnosis = ["asap-plan-selection/lp-diagnosis"] - [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 4e231d027..a68d5302c 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -59,7 +59,7 @@ // - stage2_physical_asap_optimization (from `stage2`): `pruned_candidates`, // the physical candidates Stage 2 pruned (each with its memo `group`, // `operator_node`, `operator`, `implementation_root`, `timing`, -// `materialization` and `reason`); with `stage3`, `candidates` holds +// `lifetime` and `reason`); with `stage3`, `candidates` holds // `P1`, the selected plan as Stage 2 built it; // - stage3_plan_selection (from `stage3`): Stage 3's `Selection`: // `selected_dag` (`candidate_of_node`, `edges`, @@ -97,11 +97,11 @@ // A section of a stage after `--run-until` is null. use std::collections::btree_map::Entry; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap}; use std::rc::Rc; use asap_cost::{CostInputs, ResourceUsage}; -use asap_devtools::candidate_labels::{computation_label, node_label}; +use asap_devtools::candidate_labels::node_label; use asap_devtools::planning_stages::{ lower_promql_roots, lower_sql_roots, run_stages_until, sql_catalog, PlanningStagesError, PlanningStagesRun, RunUntil, @@ -109,18 +109,16 @@ use asap_devtools::planning_stages::{ use asap_devtools::planning_stages_text::planning_stages_text; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_logical_asap_optimization::{ChosenCandidatePerMemoGroup, DeploymentCapabilities}; +use asap_logical_asap_optimization::DeploymentCapabilities; use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; -use asap_planner::{ - installable_plan_of, selected_chosen_candidates, PlanningModels, PlanningRun, SelectionError, -}; +use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; use asap_types::ir::flat::{flatten, FlatDag, FlatNode, NodeId}; use asap_types::ir::operator::node::OperatorNode; use asap_types::ir::properties::{default_metric_for_root, AccuracyTarget}; use asap_types::ir::QueryRoot; +use asap_types::stages::PhysicalASAPMemoCandidates; use asap_types::stages::{queries_rooted_at, MemoGroupId}; use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{PerComputationLogicalMemoCandidates, PhysicalASAPMemoCandidates}; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, @@ -348,7 +346,6 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS let mut stage1 = json!({ "combinations": logical.len(), "capped": false, - "pruned_candidates": pass1_pruned_candidates_json(run.run.per_computation_candidates.as_ref()), "candidates": logical, }); // Pass 1 or Pass 2 failed, or the deployment's capabilities @@ -393,21 +390,15 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS /// its materialized views. fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value { let mut stage3 = selection_json(&run.run.selection); - match (&run.run.selection, &run.run.selected_plan) { - (Ok(selection), Some(roots)) => { - let tumbling_window_retention_ms = run - .run - .selected_physical_plan - .as_ref() - .map(|realized| realized.tumbling_window_retention_ms.clone()) - .unwrap_or_default(); - stage3["tumbling_window_retention_ms"] = tumbling_window_retention_ms + match (&run.run.selection, &run.run.selected_physical_plan) { + (Ok(selection), Some(realized)) => { + stage3["tumbling_window_retention_ms"] = realized + .tumbling_window_retention_ms .iter() .map(|(node, ms)| (node.to_string(), json!(ms))) .collect::>() .into(); - match materialized_views_json(roots, selection, &tumbling_window_retention_ms, entries) - { + match materialized_views_json(realized, selection, entries) { Ok(views) => stage3["materialized_views"] = Value::Array(views), Err(error) => { stage3["materialized_views"] = json!([]); @@ -420,16 +411,15 @@ fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value stage3 } -/// The document's `memo` section: the memo after each step `run` ran, and -/// with Stage 3 the selected plan's choice over the last step's memo. +/// The document's `memo` section: the memo after each step `run` ran. fn memo_json(run: &PlanningStagesRun) -> Value { let stage0 = run .stage0_memo_groups .as_ref() .expect("run_stages_until builds Stage 0's memo"); let mut named = vec![("stage0", "Stage 0, the frontends' DAG hash-consed", stage0)]; - if let Some(pass1) = &run.run.per_computation_candidates { - named.push(("pass1", "Stage 1, Pass 1", &pass1.memo)); + if let Some(pass1) = &run.run.pass1_memo { + named.push(("pass1", "Stage 1, Pass 1", pass1)); } if let Some(pass2) = &run.pass2_local_window_composition { named.push(( @@ -452,33 +442,7 @@ fn memo_json(run: &PlanningStagesRun) -> Value { })); previous = Some(memo); } - let mut memo = json!({ "nodes": table.nodes, "steps": json_steps }); - if run.run_until == RunUntil::Stage3 { - match selected_choice(&run.run) { - Ok(choice) => memo["selected_choice"] = choice_json(&choice), - Err(reason) => { - memo["selected_choice"] = Value::Null; - memo["selected_choice_unavailable"] = json!(reason); - } - } - } - memo -} - -/// The selected plan's candidate for each group of the Pass 2 memo -/// (`run.logical_asap_candidates`), or why there is none. -fn selected_choice(run: &PlanningRun) -> Result { - match ( - &run.selection, - &run.logical_asap_candidates, - &run.physical_asap_candidates, - ) { - (Ok(selection), Some(logical), Some(physical)) => { - selected_chosen_candidates(&logical.memo, physical, &selection.selected_dag) - } - (Err(error), _, _) => Err(format!("no plan was selected: {error}")), - _ => Err("no plan was selected".into()), - } + json!({ "nodes": table.nodes, "steps": json_steps }) } /// The [`metric_type_warnings`] of each PromQL query of `described` (the @@ -645,21 +609,12 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option, entries: &[QueryWorkloadEntry], ) -> Result, String> { - let (plan, report) = installable_plan_of( - roots, - Some(selection), - tumbling_window_retention_ms, - entries, - ) - .map_err(|e| e.to_string())?; + let (plan, report) = + installable_plan_of(realized, Some(selection), entries).map_err(|e| e.to_string())?; let views = plan.materialized_views().map_err(|e| e.to_string())?; let mut stage2_of = std::collections::HashMap::new(); for node in report @@ -689,54 +644,15 @@ fn materialized_views_json( .collect() } -/// The ways to compute a measure Pass 1 pruned -/// (`PerComputationLogicalMemoCandidates::pruned_candidates`), in pruning -/// order: each with its aggregate's memo `group`, the `candidate` and the -/// `reason` (`PruneReason`, e.g. `UnsupportedByDeployment`). Empty when -/// Pass 1 failed. -fn pass1_pruned_candidates_json(pass1: Option<&PerComputationLogicalMemoCandidates>) -> Vec { - pass1 - .map_or(&[][..], |pass1| &pass1.pruned_candidates) - .iter() - .map(|pruned| { - json!({ - "group": pruned.group.0, - "candidate": computation_label(&pruned.candidate), - "reason": format!("{:?}", pruned.reason), - }) - }) - .collect() -} - /// The physical candidates Stage 2 pruned -/// (`PhysicalASAPMemoCandidates::pruned_candidates`), in pruning order: -/// each with its memo `group`, its `operator_node` (index into Stage 2's -/// operator nodes), the node's `operator` label, whether it is an -/// `implementation_root`, its `timing` and `materialization`, and the -/// `reason` (`PruneReason`, e.g. -/// `RawDataNotRetained`). Empty without memo steps. +/// (`PhysicalASAPMemoCandidates::pruned_candidates`), in pruning order. +/// Empty without memo steps. fn stage2_pruned_candidates_json(stage2: Option<&PhysicalASAPMemoCandidates>) -> Vec { let Some(stage2) = stage2 else { return Vec::new(); }; - stage2 - .pruned_candidates - .iter() - .map(|pruned| { - let key = &pruned.candidate; - let node = &stage2.operator_nodes_of_candidates_for_memo_groups[key.operator_node.0]; - let flat = export(&[QueryRoot::Operator(node.physical_node.clone())]); - json!({ - "group": pruned.group.0, - "operator_node": key.operator_node.0, - "operator": flat.nodes.last().map(node_label).unwrap_or_default(), - "implementation_root": node.is_implementation_root, - "timing": key.timing, - "materialization": key.materialization, - "reason": format!("{:?}", pruned.reason), - }) - }) - .collect() + let _ = &stage2.pruned_candidates; + todo!("#1207 PR 9 (#1205): each pruned PhysicalCandidateKey with its reason") } /// One plan's cost: per second, its predicted CPU, unit and source, its @@ -816,7 +732,7 @@ fn selection_json(selection: &Result) -> Value { "edges": dag .edges .iter() - .map(|e| json!({ "producer": e.producer.0, "consumer": e.consumer.0, "input": e.input })) + .map(|e| json!({ "producer": e.producer.0, "consumer": e.consumer.0, "input": e.input_index_of_consumer })) .collect::>(), "node_of_output_reference": nodes(&dag.node_of_output_reference), }, @@ -910,16 +826,6 @@ fn with_default_metrics(mut queries: Vec, roots: &[QueryRoot]) -> Vec Value { - Value::Object( - choice - .iter() - .map(|(group, &candidate)| (group.0.to_string(), json!(candidate))) - .collect(), - ) -} - /// The `memo` section's node table (`memo.nodes`): each distinct node once, /// children before parents, its children as indices into the table. Nodes /// are matched by their serialized JSON (children included), since the @@ -961,17 +867,16 @@ impl MemoNodes { } } -/// `memo`'s groups, each with its candidates, their -/// nodes added to `table`; a candidate is `added` when `previous` (the memo -/// at the step before) has no candidate of that group with the same node and -/// children. +/// `memo`'s groups, each with its candidates, their nodes added to +/// `table`; a candidate is `added` when `previous` (the memo at the step +/// before) has no such candidate. fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) -> Vec { let ids = |groups: &[MemoGroupId]| groups.iter().map(|g| g.0).collect::>(); memo.groups .iter() - .zip(&memo.candidates) .enumerate() - .map(|(id, (group, candidates))| { + .map(|(id, group)| { + let candidates = &group.candidates; let node = table.add(std::slice::from_ref(&group.logical_node)).0[0]; let mut by_origin = serde_json::Map::new(); for candidate in candidates { @@ -980,47 +885,14 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .or_insert(json!(0)); *count = json!(count.as_u64().unwrap_or(0) + 1); } - let earlier = previous - .and_then(|memo| memo.candidates.get(id)) - .map_or(&[][..], Vec::as_slice); - let listed: Vec<_> = candidates - .iter() - .enumerate() - .map(|(index, candidate)| { - let (roots, originals) = table.add(std::slice::from_ref(&candidate.root_node)); - let group_of_node: serde_json::Map<_, _> = originals - .iter() - .filter_map(|(original, node)| { - candidate - .input_groups - .iter() - .find(|child| { - Rc::ptr_eq( - &memo.groups[child.0 as usize].logical_node, - original, - ) - }) - .map(|child| (node.to_string(), json!(child.0))) - }) - .collect(); - json!({ - "index": index, - "implements": std::iter::once(id as u32) - .chain(candidate.fused_groups.iter().map(|g| g.0)) - .collect::>(), - "children": ids(&candidate.input_groups), - "origin": origin_name(&candidate.origin), - "accuracy_budget_parts": candidate.accuracy_budget_parts, - "added": !earlier.iter().any(|before| { - Rc::ptr_eq(&before.root_node, &candidate.root_node) - && before.input_groups == candidate.input_groups - }), - "label": candidate_node_label(&table.nodes, roots[0], &group_of_node), - "node": roots[0], - "group_of_node": group_of_node, - }) - }) - .collect(); + let listed: Vec = if candidates.is_empty() { + Vec::new() + } else { + let _ = previous; + todo!( + "#1207 PR 9 (#1205): each candidate's expression, properties and requirements" + ) + }; json!({ "id": id, "label": node_label(&table.nodes[node]), @@ -1042,8 +914,6 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { OriginOfCandidateForMemoGroup::Pass1 => "pass1", - OriginOfCandidateForMemoGroup::SharedBuild => "shared_build", - OriginOfCandidateForMemoGroup::SummaryResult => "summary_result", OriginOfCandidateForMemoGroup::SummaryInput => "summary_input", OriginOfCandidateForMemoGroup::TumblingWindows => "tumbling_windows", OriginOfCandidateForMemoGroup::Rollup => "rollup", @@ -1053,26 +923,6 @@ fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { .into() } -/// A candidate's own nodes (those above its child groups' nodes), root -/// first, as `node_label`s joined by " ← ". -fn candidate_node_label( - nodes: &[FlatNode], - root: NodeId, - group_of_node: &serde_json::Map, -) -> String { - let mut labels = Vec::new(); - let mut pending = vec![root]; - let mut seen = HashSet::new(); - while let Some(id) = pending.pop() { - if group_of_node.contains_key(&id.to_string()) || !seen.insert(id) { - continue; - } - labels.push(node_label(&nodes[id])); - pending.extend(nodes[id].operator.children().into_iter().rev().copied()); - } - labels.join(" ← ") -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/devtools/src/bin/planner_fuzz.rs b/crates/devtools/src/bin/planner_fuzz.rs index ce59b86ba..7f0fc16ec 100644 --- a/crates/devtools/src/bin/planner_fuzz.rs +++ b/crates/devtools/src/bin/planner_fuzz.rs @@ -15,8 +15,8 @@ // is filed. // // `--relations` takes relation names or the letters a–o (default: all); -// `--search` is `candidate-connections` (the default search) or -// `exhaustive`; `--ignore-accuracy-guarantees` accepts every output +// `--search` is `candidate-connections` (the default search); +// `--ignore-accuracy-guarantees` accepts every output // regardless of its accuracy guarantee (guarantees are enforced by // default, as `plan_installable` does). A relation is violated when it misses by more // than max(`--relative-tolerance` (default 0.01) × cost, @@ -33,7 +33,7 @@ use asap_devtools::planner_fuzz::report::{run_campaign, serve, Campaign, WorkerR use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, SearchKind}; const USAGE: &str = "usage: planner_fuzz --seed --cases --time-budget-s \ -[--relations ,...] [--search candidate-connections|exhaustive] \ +[--relations ,...] [--search candidate-connections] \ [--seed-workload ]... [--ignore-accuracy-guarantees] [--relative-tolerance ] \ [--absolute-tolerance ] [--max-shrink-attempts ] [--case-timeout-s ] --out "; @@ -94,7 +94,6 @@ fn run(args: Vec) -> Result<(), String> { "--search" => { settings.search = match value()?.as_str() { "candidate-connections" => SearchKind::CandidateConnections, - "exhaustive" => SearchKind::Exhaustive, other => return Err(format!("unknown search {other}")), } } diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index f511b3497..ebf91c9e0 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -3,7 +3,7 @@ // Lowers every query in every corpus we have (mirrors `variant_coverage`'s // corpus list exactly, so the two reports are directly comparable) with an // *approximate* `AccuracyTarget`, runs Pass 1 of logical ASAP-aware -// optimization (`generate_per_computation_logical_memo_candidates`, every +// optimization (`pass1_memo`, every // summary supported, no accuracy pruning) over each query, and // reports which summary candidates Pass 1 offers for it, plus the fraction of // lowered queries with at least one. @@ -20,11 +20,11 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; +use asap_logical_asap_optimization::memo_of_queries; +use asap_logical_asap_optimization::pass1_memo; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; use asap_types::workload::{DataWorkload, SqlDialect}; use std::collections::BTreeSet; use std::rc::Rc; @@ -60,8 +60,8 @@ fn analyze_corpus( let mut rejected = 0; let mut summaries = Vec::new(); for (id, root) in roots { - let Ok(pass1) = generate_per_computation_logical_memo_candidates( - generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), + let Ok(pass1) = pass1_memo( + memo_of_queries(&[QueryRoot::Operator(root)]), std::slice::from_ref(accuracy), &IgnoreAccuracyGuarantees, &NoAccuracyEvidence, @@ -76,9 +76,9 @@ fn analyze_corpus( // excluded. A way to compute a measure that Pass 1 offers but cannot // compose into a candidate is not counted. let mut offered = BTreeSet::new(); - for candidate in pass1.memo.candidates.iter().flatten() { + for candidate in pass1.groups.iter().flat_map(|group| &group.candidates) { if candidate.fused_groups.is_empty() { - summary_families(&candidate.root_node, &mut offered); + summary_family(&candidate.expression.operator, &mut offered); } } summaries.push((id, offered)); @@ -92,20 +92,16 @@ fn analyze_corpus( } } -/// Adds the family of every summary `node` and its descendants build to -/// `families`. -fn summary_families(node: &OperatorNode, families: &mut BTreeSet) { - if let Some( +/// Adds the family of the summary `operator` builds, if any, to `families`. +fn summary_family(operator: &Operator, families: &mut BTreeSet) { + if let Operator::ASAP( ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) = node.asap() + ) = operator { if let FieldDataType::Sketch(kind, _) = family { - families.insert(format!("{:?}", kind.algorithm())); + families.insert(format!("{:?}", kind)); } } - for child in node.children() { - summary_families(child, families); - } } fn covered(r: &CorpusCoverage) -> usize { diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 87ef5d797..eafb16176 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -2,7 +2,7 @@ //! `asap_planner` viewer document and text //! only. Planning never reads them: they //! are pure functions of the candidates' planning data (the logical roots -//! and the materialization assignment). +//! and the physical properties assignment). use std::collections::BTreeMap; use std::rc::Rc; @@ -10,10 +10,11 @@ use std::rc::Rc; use serde_json::Value; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::NonASAPOp; use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::Materialization; +use asap_types::physical::Lifetime; use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::HowToComputeAggregateMeasure; @@ -144,19 +145,16 @@ fn duration_name(ms: u64) -> String { /// just `logical` when nothing is. pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String { let mut parts: Vec<(&str, Vec)> = Vec::new(); - for (id, materialization) in &candidate.materialization { - let phase = match materialization { - Materialization::NotMaterialized => continue, - Materialization::MatAtIngestion(_) => "ingestion time", - Materialization::MatAtQuery(_) => "query time, shared", - Materialization::MatAtQueryKeptAcrossEvaluations(_) => "query time, kept", + for node in &candidate.dag.nodes { + let id = &node.id; + let phase = match (node.output_state.timing, node.lifetime) { + (_, Lifetime::None) => continue, + (ExecutionTiming::IngestionTime, _) => "ingestion time", + (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => "query time, shared", + (ExecutionTiming::QueryTime, Lifetime::AcrossEvaluations(_)) => { + "query time, materialized across evaluations" + } }; - let node = candidate - .dag - .nodes - .iter() - .find(|n| n.id == *id) - .expect("an assigned node is in the DAG"); let label = match &node.payload { // Name the chain: chains of one family and tumbling-window count differ by // their readers and tumbling-window width. @@ -212,7 +210,7 @@ fn state_label(payload: &Operator) -> String { Operator::ASAP( ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, ) => match family { - FieldDataType::Sketch(kind, _) => sketch_name(kind.algorithm()), + FieldDataType::Sketch(algorithm, _) => sketch_name(algorithm), FieldDataType::ExactAggregate(kind, _) => format!("{kind:?} acc"), _ => payload.kind_name().to_string(), }, @@ -400,12 +398,11 @@ mod tests { #[test] fn connected_labels_describe_selected_states() { let workload = example1(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries( &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); + )) + .unwrap(); let choice = vec![0; inventory.targets.len()]; let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); let label = connected_candidate_label(&roots); @@ -417,12 +414,11 @@ mod tests { #[test] fn connected_labels_name_actual_shared_state() { let workload = example1(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries( &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); + )) + .unwrap(); let roots = compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); let label = connected_candidate_label(&[roots[1].clone(), roots[1].clone()]); @@ -430,120 +426,6 @@ mod tests { assert_eq!(label.matches("shared Sum acc").count(), 1); } - /// Two physical candidates that each maintain a different query's tumbling window - /// chain at ingestion time get different labels, even when both chains - /// have the same family and tumbling-window count (6 × 10s for Q1, 6 × 30s for Q2). - #[test] - fn physical_labels_name_the_tumbling_window_chain_maintained_at_ingestion() { - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_physical_asap_optimization::optimize_physical_asap; - use asap_types::workload::PlanningWorkload; - - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "query_batch": null, "repeating_queries": [ - {"query": "quantile_over_time(0.5, data[1m])", "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}}}, - {"query": "quantile_over_time(0.5, data[3m])", "demand": {"fixed_interval": 30000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}}}"#, - ) - .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&workload).unwrap(), - ), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - *c == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .expect("a DDSketch candidate") - }) - .collect(); - let summary: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let grids: Vec<_> = entries.iter().map(|e| e.evaluation_grid_ms()).collect(); - let tumbling_windows = tumbling_window_candidates(&summary, &grids); - let roots = window_composition_pass2_candidates(&summary, &tumbling_windows) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty()) - .expect("each query's DDSketch in its own tumbling-window chain") - .roots - .unwrap(); - let plans = optimize_physical_asap( - "L1", - &roots, - &entries, - &workload.data_workload.clone().unwrap_or_default(), - false, - ) - .unwrap(); - // The plans that maintain exactly one tumbling-window chain at ingestion time. - let one_chain_at_ingestion: Vec = plans - .iter() - .filter(|plan| { - plan.materialization - .iter() - .filter(|(id, m)| { - matches!(m, Materialization::MatAtIngestion(_)) - && matches!( - plan.dag - .nodes - .iter() - .find(|n| n.id == **id) - .unwrap() - .payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count() - == 1 - }) - .map(|plan| physical_label("L1", plan)) - .collect(); - let ingestion = |label: &String| { - label - .split(" · ") - .find(|p| p.starts_with("ingestion time")) - .map(str::to_string) - }; - assert!( - one_chain_at_ingestion - .iter() - .any(|l| ingestion(l).as_deref() - == Some("ingestion time: DDSketch for Q1, 6 × 10s tumbling windows")), - "{one_chain_at_ingestion:#?}" - ); - assert!( - one_chain_at_ingestion - .iter() - .any(|l| ingestion(l).as_deref() - == Some("ingestion time: DDSketch for Q2, 6 × 30s tumbling windows")), - "{one_chain_at_ingestion:#?}" - ); - let mut distinct = one_chain_at_ingestion.clone(); - distinct.sort(); - distinct.dedup(); - assert_eq!( - distinct.len(), - one_chain_at_ingestion.len(), - "{one_chain_at_ingestion:#?}" - ); - } - /// Raw logical operators have an exact label, rather than an empty choice label. #[test] fn connected_labels_name_exact_results() { diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index f3c546d6e..6fe2ec83f 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -23,7 +23,6 @@ pub mod candidate_labels; pub mod corpora; -pub mod memo_pricing; pub mod planner_fuzz; pub mod planner_layering_examples; pub mod planning_stages; diff --git a/crates/devtools/src/memo_pricing.rs b/crates/devtools/src/memo_pricing.rs deleted file mode 100644 index 128cc4911..000000000 --- a/crates/devtools/src/memo_pricing.rs +++ /dev/null @@ -1,401 +0,0 @@ -//! Price a memo's enumerated choices as the selected plan is priced: each -//! choice's composed DAG (`memo::enumeration::build_logical_asap_dags_from_chosen_candidates`) goes through Stage 2 -//! (`optimize_physical_asap`, every materialization assignment) and Stage 3 -//! (the reference pricer, `asap_plan_selection::reference_pricer`, under the -//! run's models), and the plan it selects for -//! it is installed as `plan_installable` installs one (`installable_plan_of`, -//! which also compiles it with the executor), so a choice cheaper than the -//! selected plan shows that the search missed a plan the memo stands for. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_logical_asap_optimization::{ - build_logical_asap_dags_from_chosen_candidates, ChosenCandidatePerMemoGroup, -}; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_plan_selection::reference_pricer::{select_among_plans, WholePlanSelection}; -use asap_plan_selection::{ErrorSelectingPlan, Selection}; -use asap_planner::{installable_plan_of, InstallablePlanError, PlanningModels, PlanningRun}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -/// One choice's result: valid with its cost, or why it is not valid. -#[derive(Debug, Clone, PartialEq)] -pub struct PricedChoice { - /// Stage 2 implemented it, Stage 3 accepted one of its physical plans, - /// and the plan Stage 3 selected installs (a standalone scalar query, - /// which no choice changes, aside). - pub valid: bool, - /// Why it is not valid: the Stage 3 rejection cause of its first - /// physical plan in snake_case (e.g. `accuracy_target_missed`), or - /// `compose_failed`, `invalid_structure`, `stage2_failed`, - /// `stage3_failed`, `rejected` (no plan and no rejection listed), - /// `not_executable` (the executor cannot compile the selected plan's - /// view refresh or query run), or `not_installable` (installation - /// refuses the selected plan for another reason, e.g. an unsupported - /// raw-data window). - pub rejection: Option, - /// The rejection's one-line reason, with the numbers. - pub rejection_reason: Option, - /// The cheapest cost per second of its valid physical plans under the - /// run's cost model; `None` when not valid or under cost model 3. - pub cost_per_second: Option, - /// Valid and costs less than the selected plan (beyond rounding). - pub cheaper_than_selected: bool, -} - -/// What the choices are compared against: the run's selection. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct SelectedBaseline { - /// The selected plan's cost per second. - pub cost_per_second: Option, - /// The calibration factor the run priced with, frozen for every choice - /// so each is priced on the same scale as the selected plan. - pub calibration_factor: Option, -} - -impl SelectedBaseline { - /// The baseline of `selection`: its selected plan's cost. Stage 3 - /// reports no calibration factor, so a measured cost table without a - /// frozen factor is priced at each choice's own fit. - pub fn of(selection: &Selection) -> Self { - Self { - cost_per_second: selection.cost.cost_per_second, - calibration_factor: None, - } - } -} - -/// Costs within this fraction of each other are the same sum of charges -/// rounded differently (Stage 3's relative tie tolerance). -const TIE_TOLERANCE: f64 = 1e-9; - -/// Each of `choices` of `memo`, composed, implemented and priced. Query `i` -/// is `stage0_roots[i]` with workload entry `entries[i]`; its operator -/// references are replaced by their root groups' composed DAGs. -pub fn price_choices( - memo: &Memo, - choices: &[ChosenCandidatePerMemoGroup], - stage0_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, - selected: Option, -) -> Vec { - let mut settings = models.cost_inputs.clone(); - if settings.frozen_calibration_factor.is_none() { - settings.frozen_calibration_factor = selected.and_then(|s| s.calibration_factor); - } - let models = models.with_cost_inputs(&settings); - let selected_cost = selected.and_then(|s| s.cost_per_second); - // Which root group each query's operator references name is the same - // for every choice. - let root_groups = root_groups_of_queries(memo, stage0_roots); - choices - .iter() - .enumerate() - .map(|(index, choice)| { - let roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .map_err(|error| error.to_string()) - .and_then(|composed| query_roots(&composed, stage0_roots, &root_groups)); - let mut priced = price_choice(roots, index, entries, data, models); - priced.cheaper_than_selected = match (priced.cost_per_second, selected_cost) { - (Some(cost), Some(selected)) => cost < selected - TIE_TOLERANCE * selected.abs(), - _ => false, - }; - priced - }) - .collect() -} - -/// One choice, its query roots composed (or why they do not compose), -/// implemented and priced. -fn price_choice( - roots: Result, String>, - index: usize, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> PricedChoice { - let invalid = |cause: &str, reason: String| PricedChoice { - valid: false, - rejection: Some(cause.into()), - rejection_reason: Some(reason), - cost_per_second: None, - cheaper_than_selected: false, - }; - let roots = match roots { - Ok(roots) => roots, - Err(error) => return invalid("compose_failed", error), - }; - if let Err(error) = roots.iter().try_for_each(QueryRoot::validate_structure) { - return invalid("invalid_structure", error.to_string()); - } - let physical = match physical_plans_of(&roots, index, entries, data, models) { - Ok(physical) => physical, - Err(error) => return invalid("stage2_failed", error), - }; - match reference_select(&physical, entries, data, models) { - Ok(selection) => { - match installable_plan_of( - &selection.plan, - None, - &selection.tumbling_window_retention_ms, - entries, - ) { - // A standalone scalar query, which a runtime cannot answer yet, - // does not depend on the choice. - Ok(_) | Err(InstallablePlanError::UnsupportedQueryRoot { .. }) => PricedChoice { - valid: true, - rejection: None, - rejection_reason: None, - cost_per_second: selection.costs[&selection.selected].cost_per_second, - cheaper_than_selected: false, - }, - Err( - error @ (InstallablePlanError::MaterializedViewNotExecutable { .. } - | InstallablePlanError::QueryRunNotExecutable { .. }), - ) => invalid("not_executable", error.to_string()), - Err(error) => invalid("not_installable", error.to_string()), - } - } - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => match rejected.first() { - Some(first) => invalid( - serde_json::to_value(first.cause) - .ok() - .and_then(|v| v.as_str().map(str::to_owned)) - .as_deref() - .unwrap_or("rejected"), - first.reason.clone(), - ), - None => invalid("rejected", "no valid physical plan".into()), - }, - Err(error) => invalid("stage3_failed", error.to_string()), - } -} - -/// Every physical plan Stage 2 offers for choice `index`'s composed `roots` -/// (each materialization assignment), named `E-`. -pub fn physical_plans_of( - roots: &[QueryRoot], - index: usize, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result, String> { - let id = format!("E{}", index + 1); - let mut physical = optimize_physical_asap( - &id, - roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - ) - .map_err(|error| error.to_string())?; - retain_deployable(&mut physical, models); - if physical.is_empty() { - return Err("no physical plan".into()); - } - for (k, plan) in physical.iter_mut().enumerate() { - plan.id = format!("{id}-{k}"); - plan.from_logical = id.clone(); - } - Ok(physical) -} - -/// Drops the plans of `physical` that Stage 2 would not offer under -/// `models.deployment_capabilities`: without raw data retained, a plan that -/// scans raw data at query time (Stage 2's `PruneReason::RawDataNotRetained`; -/// `optimize_physical_asap` enumerates every assignment and leaves this -/// rule to Stage 2's catalog, and the reference pricer no longer checks it). -pub fn retain_deployable(physical: &mut Vec, models: PlanningModels<'_>) { - if models.deployment_capabilities.raw_data_retained { - return; - } - physical.retain(|plan| { - plan.dag.nodes.iter().all(|node| { - !matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) || node.output_state.timing == ExecutionTiming::IngestionTime - }) - }); -} - -/// The reference pricer over `physical` under `models`: every plan checked, -/// priced and ranked, every candidate kept. -pub fn reference_select( - physical: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, -) -> Result { - select_among_plans( - physical, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - ) -} - -/// One enumerated memo choice of a reference enumeration: its composed -/// logical roots (or why they do not compose), under id `E`. -#[derive(Debug)] -pub struct EnumeratedChoice { - pub id: String, - pub roots: Result, String>, -} - -/// The reference enumeration: the first `max_choices` choices of the -/// Pass 2 memo (`run.logical_asap_candidates`), each composed, -/// implemented by Stage 2 with every materialization assignment, and all -/// their physical plans priced together by the reference pricer. -#[derive(Debug)] -pub struct ReferenceEnumeration { - pub choices: Vec, - pub physical: Vec, - /// Why a composed choice has no physical plan, by choice id. - pub stage2_failed: Vec<(String, String)>, - /// Whether the memo has more choices than `max_choices`. - pub capped: bool, - pub selection: Result, -} - -/// The reference enumeration of `run` ([`ReferenceEnumeration`]); `None` -/// without a memo. -pub fn reference_enumeration( - run: &PlanningRun, - stage0_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: PlanningModels<'_>, - max_choices: usize, -) -> Option { - use asap_logical_asap_optimization::enumerate; - let memo = &run.logical_asap_candidates.as_ref()?.memo; - let enumeration = enumerate(memo, max_choices.saturating_add(1)); - let capped = !enumeration.complete || enumeration.all_chosen_candidates.len() > max_choices; - let root_groups = root_groups_of_queries(memo, stage0_roots); - let mut choices = Vec::new(); - let mut physical = Vec::new(); - let mut stage2_failed = Vec::new(); - for (index, choice) in enumeration - .all_chosen_candidates - .iter() - .take(max_choices) - .enumerate() - { - let id = format!("E{}", index + 1); - let roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .map_err(|error| error.to_string()) - .and_then(|composed| query_roots(&composed, stage0_roots, &root_groups)) - .and_then(|roots| { - roots - .iter() - .try_for_each(QueryRoot::validate_structure) - .map_err(|error| error.to_string())?; - Ok(roots) - }); - if let Ok(roots) = &roots { - match physical_plans_of(roots, index, entries, data, models) { - Ok(plans) => physical.extend(plans), - Err(error) => stage2_failed.push((id.clone(), error)), - } - } - choices.push(EnumeratedChoice { id, roots }); - } - let selection = reference_select(&physical, entries, data, models); - Some(ReferenceEnumeration { - choices, - physical, - stage2_failed, - capped, - selection, - }) -} - -/// For each query, the root group of each of its operator references, in -/// `operator_refs` order: the one group the query's root (in the memo's -/// `query_roots`) reads when its root is one operator, else the group -/// whose node is the reference. -fn root_groups_of_queries( - memo: &Memo, - stage0_roots: &[QueryRoot], -) -> Result>, String> { - stage0_roots - .iter() - .enumerate() - .map(|(query, root)| { - let groups: Vec<_> = memo - .query_roots - .get(query) - .map(|root| { - root.operator_refs() - .into_iter() - .copied() - .collect::>() - }) - .unwrap_or_default() - .into_iter() - .collect(); - match root { - QueryRoot::Operator(_) if groups.len() == 1 => Ok(groups), - QueryRoot::Operator(node) => Ok(vec![group_of(memo, &groups, node, query)?]), - QueryRoot::Scalar(expression) => expression - .operator_refs() - .into_iter() - .map(|node| group_of(memo, &groups, node, query)) - .collect(), - } - }) - .collect() -} - -/// The group among `groups` whose node is `node`: the same node, else an -/// equal one (the memo's group nodes are the frontends' nodes with common -/// sub-DAGs shared). -fn group_of( - memo: &Memo, - groups: &[MemoGroupId], - node: &Rc, - query: usize, -) -> Result { - let node_of = |g: &&MemoGroupId| &memo.groups[g.0 as usize].logical_node; - groups - .iter() - .find(|g| Rc::ptr_eq(node_of(g), node)) - .or_else(|| groups.iter().find(|g| **node_of(g) == **node)) - .copied() - .ok_or_else(|| format!("query {query}'s root has no root group")) -} - -/// Each query's root with every operator reference replaced by the composed -/// DAG of its root group (`root_groups`). -fn query_roots( - composed: &[(MemoGroupId, Rc)], - stage0_roots: &[QueryRoot], - root_groups: &Result>, String>, -) -> Result, String> { - let root_groups = root_groups.as_ref().map_err(String::clone)?; - let composed: HashMap<_, _> = composed.iter().cloned().collect(); - stage0_roots - .iter() - .zip(root_groups) - .map(|(root, groups)| { - let mut next = groups.iter().map(|group| Rc::clone(&composed[group])); - Ok(match root { - QueryRoot::Operator(_) => QueryRoot::Operator(next.next().expect("one group")), - QueryRoot::Scalar(expression) => QueryRoot::Scalar( - expression.map_operator_refs(&mut |_| next.next().expect("a group per ref")), - ), - }) - }) - .collect() -} diff --git a/crates/devtools/src/planner_fuzz/README.md b/crates/devtools/src/planner_fuzz/README.md index 3370fa96f..044dbf232 100644 --- a/crates/devtools/src/planner_fuzz/README.md +++ b/crates/devtools/src/planner_fuzz/README.md @@ -106,7 +106,7 @@ derive_cost = cost(witness) − cost(W's plan) never from a guess: 1. Stage 2 and Stage 3 implement and price the witness the same way they price - any candidate. Each materialization assignment goes through + any candidate. Each physical properties assignment goes through `optimize_physical_asap` and `select_plan`, using the same cost model, deployment capabilities and accuracy checks as the search. The cheapest assignment is kept. 2. `W`'s selected logical plan is priced the same way. This gives the witness diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index e37b85e75..ab359d74f 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -8,8 +8,7 @@ //! target, a filter on a group key, a coarser grouping, ...), it builds a //! *witness*: `W`'s selected logical plan plus `q'` derived from `q`'s //! node by one operator. Stage 2 and Stage 3 implement and price the -//! witness exactly as they price any candidate -//! ([`optimize_physical_asap`], the reference pricer), and the relation +//! witness exactly as they price any candidate, and the relation //! //! ```text //! cost(W ∪ {q'}) − cost(W) ≤ derive_cost + tolerance, @@ -35,7 +34,7 @@ pub mod relations; pub mod report; pub mod shrink; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::HashMap; use std::rc::Rc; use serde::{Deserialize, Serialize}; @@ -43,7 +42,6 @@ use serde_json::{json, Value}; use asap_cost::CostInputs; use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_physical_asap_optimization::optimize_physical_asap; use asap_planner::{PlanSearchSettings, PlanningModels}; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::sketch::SketchAlgorithm; @@ -53,9 +51,7 @@ use asap_types::workload::{ SqlDialect, }; -use crate::planning_stages::{ - lower_promql_roots, lower_sql_roots, run_planning_stages_over, run_search_over, sql_catalog, -}; +use crate::planning_stages::{lower_promql_roots, lower_sql_roots, run_search_over, sql_catalog}; use generate::QuerySpec; /// The language of a fuzz workload: one per workload, as one @@ -219,17 +215,9 @@ impl FuzzWorkload { /// The `asap_planner` invocation (a bash command) that plans this /// workload, saved as `workload_file`, with `settings`' accuracy - /// guarantees. `asap_planner` has no exhaustive search, so under - /// `--search exhaustive` the command starts with a comment saying it - /// may not reproduce. Its output document is `.out.json`. + /// guarantees. Its output document is `.out.json`. pub fn asap_planner_command(&self, workload_file: &str, settings: &FuzzSettings) -> String { let mut command = String::new(); - if matches!(settings.search, SearchKind::Exhaustive) { - command.push_str( - "# asap_planner runs the candidate-connection search, not the \ - exhaustive one; it may not reproduce this.\n", - ); - } command.push_str(&format!( "cargo run -p asap-devtools --bin asap_planner -- --workload {workload_file} \ --run-until stage3 --format json" @@ -314,14 +302,8 @@ pub fn read_seed_file(path: &std::path::Path) -> Result { pub enum SearchKind { /// Joint candidate-connection planning, the facade's search. CandidateConnections, - /// Every generated candidate (`--exhaustive`), up to - /// [`EXHAUSTIVE_MAX_COMBINATIONS`]. - Exhaustive, } -/// `--exhaustive`'s default evaluated-assignment budget. -pub const EXHAUSTIVE_MAX_COMBINATIONS: usize = 10_000; - /// How the tester plans and judges. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct FuzzSettings { @@ -368,7 +350,7 @@ pub struct Planned { } /// A plan priced by Stage 2 and Stage 3 outside the search: the cheapest -/// materialization assignment of given logical roots. +/// physical properties assignment of given logical roots. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct Priced { pub selected: String, @@ -443,70 +425,30 @@ impl Planner { .with_deployment_capabilities(&deployment) .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); let run = match self.settings.search { - SearchKind::Exhaustive => run_planning_stages_over( - roots.clone(), - &entries, - &data, - Some(EXHAUSTIVE_MAX_COMBINATIONS), - models, - ), SearchKind::CandidateConnections => run_search_over( roots.clone(), &entries, &data, models, PlanSearchSettings::default(), - None, ), } .map_err(|e| e.to_string())?; - match &run.reference { - // The reference pricer's selection among the enumerated plans. - Some(reference) => { - let selection = reference - .selection - .as_ref() - .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection.costs[&selection.selected] - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - let from_logical = reference - .physical - .iter() - .find(|p| p.id == selection.selected) - .map(|p| p.from_logical.clone()); - let logical_roots = from_logical.and_then(|id| { - reference - .choices - .iter() - .find(|c| c.id == id) - .and_then(|c| c.roots.as_ref().ok().cloned()) - }); - Ok(Planned { - selected: selection.selected.clone(), - cost_per_second, - stage0_roots: roots, - logical_roots, - }) - } - None => { - let selection = run - .run - .selection - .as_ref() - .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection - .cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - Ok(Planned { - selected: "selected".into(), - cost_per_second, - stage0_roots: roots, - logical_roots: run.run.selected_plan.clone(), - }) - } - } + let selection = run + .run + .selection + .as_ref() + .map_err(|e| format!("selection: {e}"))?; + let cost_per_second = selection + .cost + .cost_per_second + .ok_or("no cost per second (cost model 3)")?; + Ok(Planned { + selected: "selected".into(), + cost_per_second, + stage0_roots: roots, + logical_roots: run.run.selected_plan.as_ref().cloned(), + }) } /// Stage 2 and Stage 3 over the logical plan `roots` of `workload`'s @@ -526,93 +468,8 @@ impl Planner { roots: &[QueryRoot], last_root_is_key_filter: bool, ) -> Result { - for root in roots { - root.validate_structure() - .map_err(|e| format!("witness validation: {e}"))?; - } - let mut roots = asap_accuracy::propagation::recompose_guarantees(roots) - .map_err(|e| format!("witness guarantees: {e}"))?; - let planning = workload.planning_workload(); - let entries: Vec<_> = planning.query_workload.entries().collect(); - // A witness is built outside the memo: size its summaries as Pass 1 - // sizes a candidate for these readers, which Stage 3 only checks. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - roots = asap_accuracy::sizing::with_summaries_sized_for_targets( - &roots, - &targets, - &workload.data, - ) - .map_err(|e| format!("witness sizing: {e}"))?; - if let (true, Some(last)) = (last_root_is_key_filter, roots.last_mut()) { - let filter = last.as_operator().ok_or("a scalar root")?; - let input = filter.operator.children()[0].guarantee.clone(); - *last = QueryRoot::Operator(Rc::new((**filter).clone().with_guarantee(input))); - } - let cost_inputs = self.cost_inputs(); - let deployment = workload.deployment(); - let models = PlanningModels::builtin() - .with_cost_inputs(&cost_inputs) - .with_deployment_capabilities(&deployment) - .with_ignore_accuracy_guarantees(!self.settings.enforce_accuracy_guarantees); - let mut candidates = optimize_physical_asap( - "witness", - &roots, - &entries, - &workload.data, - deployment.query_time_results_retained, - ) - .map_err(|e| format!("witness Stage 2: {e}"))?; - for (index, candidate) in candidates.iter_mut().enumerate() { - candidate.id = format!("witness-m{index}"); - } - let selection = - crate::memo_pricing::reference_select(&candidates, &entries, &workload.data, models) - .map_err(|e| format!("witness Stage 3: {e}"))?; - let cost = &selection.costs[&selection.selected]; - let cost_per_second = cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - let candidate = candidates - .iter() - .find(|c| c.id == selection.selected) - .expect("the selected candidate is offered"); - let dag = &candidate.dag; - let mut inputs: BTreeMap<_, Vec<_>> = BTreeMap::new(); - for edge in &dag.edges { - inputs.entry(edge.consumer).or_default().push(edge.producer); - } - let reached = |root| { - let mut seen = BTreeSet::new(); - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - if seen.insert(node) { - stack.extend(inputs.get(&node).into_iter().flatten().copied()); - } - } - seen - }; - let reached_by_root: Vec> = dag.roots.iter().map(|r| reached(*r)).collect(); - let (last, others) = reached_by_root.split_last().ok_or("no roots")?; - let private: Vec<_> = last - .iter() - .filter(|node| others.iter().all(|set| !set.contains(node))) - .filter_map(|node| cost.per_node.get(node)) - .collect(); - let derived_root = PrivateNodes { - cost_per_second: private.iter().filter_map(|n| n.cost_per_second).sum(), - operators: private - .iter() - .map(|n| format!("{:?}", n.operator)) - .collect(), - }; - Ok(Priced { - selected: selection.selected.clone(), - cost_per_second, - derived_root, - }) + let _ = (workload, roots, last_root_is_key_filter); + todo!("#1207 PR 6: price a witness plan through the new Stage 2 and Stage 3") } /// [`price`](Self::price) of `workload`'s selected logical plan diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index b40f36dea..fff254628 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -4,9 +4,8 @@ use asap_frontend_sql::SqlCatalog; use asap_logical_asap_optimization::{ - generate_dag_of_logical_memo_groups, generate_per_computation_logical_memo_candidates, - pass2_after_local_window_composition, pass2_local_window_composition, - PerComputationLogicalMemoCandidatesError, + memo_of_queries, pass1_memo, pass2_after_local_window_composition, + pass2_local_window_composition, Pass1Error, }; use asap_physical_asap_optimization::{ generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, @@ -17,20 +16,15 @@ use asap_planner::{ }; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::QueryRoot; +use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::{AssignmentRealizer, DagOfLogicalMemoGroups, Memo}; +use asap_types::stages::{AssignmentRealizer, Memo}; use asap_types::workload::bounds::with_confirmed_weights_at; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; use serde_json::Value; use std::collections::HashSet; use std::rc::Rc; -use crate::memo_pricing::{reference_enumeration, ReferenceEnumeration}; - -/// The memo choices the reference enumeration prices when no limit is given. -pub const DEFAULT_REFERENCE_CHOICES: usize = 10_000; - /// The last stage a run runs (`asap_planner --run-until`); the run's /// fields for later stages are left empty. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -87,9 +81,6 @@ pub struct PlanningStagesRun { pub pass2_local_window_composition: Option, /// The stages' outputs: Pass 1, Pass 2, Stage 2 and the selection. pub run: PlanningRun, - /// With a reference enumeration: Pass 2's memo choices, each implemented - /// by Stage 2 and all priced by the reference pricer. - pub reference: Option, } /// A logical candidate the document shows: its id and composed roots, or @@ -102,47 +93,30 @@ pub struct ShownLogicalCandidate { impl PlanningStagesRun { /// The logical candidates the document shows: the selected plan (`L1`; - /// its roots as Stage 2 built them, with Stage 3's weight proofs), then - /// with a reference enumeration, its's choices (`E`). + /// its roots as Stage 2 built them, with Stage 3's weight proofs). pub fn logical_candidates(&self) -> Vec { - let selected = self - .run + self.run .selected_plan .iter() - .map(|roots| ShownLogicalCandidate { + .map(|plan| ShownLogicalCandidate { id: "L1".into(), - roots: Ok(roots.clone()), - }); - let enumerated = self.reference.iter().flat_map(|reference| { - reference - .choices - .iter() - .map(|choice| ShownLogicalCandidate { - id: choice.id.clone(), - roots: choice.roots.clone(), - }) - }); - selected.chain(enumerated).collect() + roots: Ok(plan.clone()), + }) + .collect() } /// The physical candidates the document shows: the selected plan as - /// Stage 2 built it (`P1`, from `L1`), then with a reference enumeration every - /// plan the reference pricer priced (`E-`). + /// Stage 2 built it (`P1`, from `L1`). pub fn physical_candidates(&self) -> Vec { - let selected = - self.run - .selected_physical_plan - .iter() - .map(|realized| PhysicalWorkloadPlan { - id: "P1".into(), - from_logical: "L1".into(), - ..realized.plan.clone() - }); - let enumerated = self - .reference + self.run + .selected_physical_plan .iter() - .flat_map(|reference| reference.physical.iter().cloned()); - selected.chain(enumerated).collect() + .map(|realized| PhysicalWorkloadPlan { + id: "P1".into(), + from_logical: "L1".into(), + ..realized.plan.clone() + }) + .collect() } } @@ -215,23 +189,14 @@ impl From for String { } } -/// The planning stages over `workload`, with the reference enumeration of -/// at most `max_combinations` memo choices (default -/// [`DEFAULT_REFERENCE_CHOICES`]). +/// The planning stages over `workload`. pub fn run_planning_stages( workload: &PlanningWorkload, roots: Vec, - max_combinations: Option, ) -> Result { let entries: Vec<_> = workload.query_workload.entries().collect(); let data = workload.data_workload.clone().unwrap_or_default(); - run_planning_stages_over( - roots, - &entries, - &data, - max_combinations, - PlanningModels::builtin(), - ) + run_planning_stages_over(roots, &entries, &data, PlanningModels::builtin()) } /// [`run_planning_stages`] over roots lowered from any mix of query @@ -244,7 +209,6 @@ pub fn run_planning_stages_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, - max_combinations: Option, models: PlanningModels<'_>, ) -> Result { run_search_over( @@ -252,14 +216,11 @@ pub fn run_planning_stages_over( entries, data, models, - // The reference enumeration reads Pass 2's memo. PlanSearchSettings { inspect: true }, - Some(max_combinations.unwrap_or(DEFAULT_REFERENCE_CHOICES)), ) } -/// The facade's planning, as `plan_installable` runs it, without a -/// reference enumeration. Use the executor's summary capabilities in +/// The facade's planning, as `plan_installable` runs it. Use the executor's summary capabilities in /// `models` when comparing with installation behavior. pub fn run_default_search_over( roots: Vec, @@ -267,53 +228,32 @@ pub fn run_default_search_over( data: &DataWorkload, models: PlanningModels<'_>, ) -> Result { - run_search_over( - roots, - entries, - data, - models, - PlanSearchSettings::default(), - None, - ) + run_search_over(roots, entries, data, models, PlanSearchSettings::default()) } -/// The facade's planning with `settings`, and with `reference_choices` -/// the reference enumeration of that many memo choices. +/// The facade's planning with `settings`. pub fn run_search_over( roots: Vec, entries: &[QueryWorkloadEntry], data: &DataWorkload, models: PlanningModels<'_>, settings: PlanSearchSettings, - reference_choices: Option, ) -> Result { validate(&roots)?; let run = run_planning(roots.clone(), entries, data, models, settings) .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; - if let Some(roots) = &run.selected_plan { - validate(roots)?; + if let Some(plan) = &run.selected_plan { + validate(plan)?; } - let reference = reference_choices - .and_then(|max| reference_enumeration(&run, &roots, entries, data, models, max)); Ok(PlanningStagesRun { run_until: RunUntil::Stage3, stage0_memo_groups: None, pass2_local_window_composition: None, stage0_query_language_frontends: roots, run, - reference, }) } -/// Stage 0's memo groups, `dag`, without candidates. -fn stage0_memo_groups(dag: DagOfLogicalMemoGroups) -> Memo { - Memo { - candidates: vec![Vec::new(); dag.groups.len()], - groups: dag.groups, - query_roots: dag.query_roots, - } -} - /// The stages over `roots` up to and including `until`, as the facade runs /// them. A stage's failure is reported in the run, as the facade does. // ponytail: copies run_planning's stage order, which has no stop point; @@ -326,14 +266,14 @@ pub fn run_stages_until( until: RunUntil, ) -> Result { validate(&roots)?; - let stage0_dag = generate_dag_of_logical_memo_groups(&roots); + let stage0_dag = memo_of_queries(&roots); let mut stages = PlanningStagesRun { run_until: until, - stage0_memo_groups: Some(stage0_memo_groups(stage0_dag.clone())), + stage0_memo_groups: Some(stage0_dag.clone()), pass2_local_window_composition: None, stage0_query_language_frontends: roots.clone(), run: PlanningRun { - per_computation_candidates: None, + pass1_memo: None, logical_asap_candidates: None, physical_asap_candidates: None, selection: Err(SelectionError::NotRun), @@ -341,7 +281,6 @@ pub fn run_stages_until( selected_plan: None, report: PlanningReport::default(), }, - reference: None, }; if until == RunUntil::Stage0 { return Ok(stages); @@ -352,7 +291,7 @@ pub fn run_stages_until( .map(|entry| entry.requirements.accuracy.clone()) .collect(); let accuracy = models.accuracy_check_inputs(); - let pass1 = generate_per_computation_logical_memo_candidates( + let pass1 = pass1_memo( stage0_dag, &accuracy_targets, accuracy.model, @@ -365,7 +304,7 @@ pub fn run_stages_until( ); // As `run_planning`: a query Pass 1 cannot plan is reported before // contradictory deployment capabilities. - if let Err(PerComputationLogicalMemoCandidatesError::PerComputation(error)) = &pass1 { + if let Err(Pass1Error::PerComputation(error)) = &pass1 { return Err(PlanningStagesError::Pass1(error.to_string())); } if let Err(error) = models.deployment_capabilities.validate(data) { @@ -379,11 +318,17 @@ pub fn run_stages_until( return Ok(stages); } }; - run.per_computation_candidates = Some(pass1.clone()); + run.pass1_memo = Some(pass1.clone()); if until == RunUntil::Pass1 { return Ok(stages); } - let composed = pass2_local_window_composition(pass1, entries); + let composed = match pass2_local_window_composition(pass1, entries) { + Ok(composed) => composed, + Err(error) => { + run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); + return Ok(stages); + } + }; stages.pass2_local_window_composition = Some(composed.clone()); let logical = match pass2_after_local_window_composition(composed) { Ok(logical) => logical, @@ -434,7 +379,7 @@ pub fn run_stages_until( Ok(realized) => { // As the facade does: the plan to run carries the weight // proofs Stage 3 made. - let proven: HashSet<*const asap_types::ir::OperatorNode> = selection + let proven: HashSet<*const OperatorNode> = selection .nodes_with_proven_non_negative_weights .iter() .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index f68332018..4742965f0 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -1,6 +1,6 @@ //! The planner stages' output as plain text: `asap_planner --format text`. //! It prints, up to the run's last stage, the frontends' roots and their -//! memo groups (Stage 0), Pass 1's `PerComputationLogicalMemoCandidates`, +//! memo groups (Stage 0), Pass 1's `Memo`, //! the memo after Pass 2's local window composition and Pass 2's //! `LogicalASAPMemoCandidates` (Stage 2's input), Stage 2's //! `PhysicalASAPMemoCandidates` (Stage 3's input), and Stage 3's @@ -20,14 +20,12 @@ //! [1] TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000 inputs=[G0] //! ``` -use std::collections::HashSet; use std::fmt::Write; use std::rc::Rc; -use asap_planner::selected_chosen_candidates; use asap_types::ir::flat::flatten; use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{Memo, MemoGroupId, PhysicalASAPMemoCandidates}; +use asap_types::stages::{Memo, PhysicalASAPMemoCandidates}; use crate::candidate_labels::node_label; use crate::planning_stages::{PlanningStagesRun, RunUntil}; @@ -52,7 +50,7 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { write_dag(&mut out, &stages.stage0_query_language_frontends); section( &mut out, - "Stage 0 → Stage 1: DagOfLogicalMemoGroups (the roots hash-consed)", + "Stage 0 → Stage 1: Memo (the roots hash-consed, without candidates)", ); if let Some(memo) = &stages.stage0_memo_groups { write_memo(&mut out, memo, None, false); @@ -61,32 +59,12 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { return out; } - let Some(pass1) = &run.per_computation_candidates else { + let Some(pass1) = &run.pass1_memo else { failure(&mut out); return out; }; - section( - &mut out, - "Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates", - ); - write_memo(&mut out, &pass1.memo, None, true); - let _ = writeln!( - out, - "pruned_candidates ({}):", - pass1.pruned_candidates.len() - ); - let rows = pass1 - .pruned_candidates - .iter() - .map(|pruned| { - vec![ - format!("G{}:", pruned.group.0), - format!("{:?}", pruned.candidate), - format!("{:?}", pruned.reason), - ] - }) - .collect(); - write_rows(&mut out, " ", rows); + section(&mut out, "Stage 1 Pass 1 output: Memo"); + write_memo(&mut out, pass1, None, true); if until == RunUntil::Pass1 { return out; } @@ -102,7 +80,7 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { &mut out, "Stage 1 Pass 2, after local window composition (tumbling-window candidates)", ); - write_memo(&mut out, pass2, Some(&pass1.memo), true); + write_memo(&mut out, pass2, Some(pass1), true); section( &mut out, "Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates", @@ -126,20 +104,6 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { } section(&mut out, "Stage 3 output: Selection"); - let chosen = match &run.selection { - Ok(selection) => { - selected_chosen_candidates(&logical.memo, physical, &selection.selected_dag) - } - Err(_) => Err("no plan was selected".into()), - }; - match &chosen { - Ok(choice) => { - let _ = writeln!(out, "selected choice: {}", choice_text(choice)); - } - Err(reason) => { - let _ = writeln!(out, "selected choice unavailable: {reason}"); - } - } write_selection(&mut out, stages); out } @@ -215,29 +179,9 @@ fn reads(inputs: &[String]) -> String { } } -/// `G0, G1`. -fn group_list(groups: &[MemoGroupId]) -> String { - groups - .iter() - .map(|group| format!("G{}", group.0)) - .collect::>() - .join(", ") -} - -/// `G2:c0 G3:c0`: each chosen group's candidate. -fn choice_text(choice: &asap_logical_asap_optimization::ChosenCandidatePerMemoGroup) -> String { - choice - .iter() - .map(|(group, candidate)| format!("G{}:c{candidate}", group.0)) - .collect::>() - .join(" ") -} - /// `memo`'s groups, one line each with the child groups it reads, and with -/// `with_candidates` each group's candidates below it: index, origin, -/// `new` when `previous` (the memo at the step before) has no candidate of -/// that group with the same node and inputs, chain and input groups (fused -/// groups, and accuracy budget parts when more than one). +/// `with_candidates` each group's candidates below it, marked `new` when +/// `previous` (the memo at the step before) has no such candidate. fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candidates: bool) { // The group lines share one set of column widths, with each group's // candidates printed under its line. @@ -265,43 +209,8 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi if !with_candidates { continue; } - let earlier = previous - .and_then(|memo| memo.candidates.get(id)) - .map_or(&[][..], Vec::as_slice); - let rows = memo.candidates[id] - .iter() - .enumerate() - .map(|(index, candidate)| { - let added = previous.is_some() - && !earlier.iter().any(|before| { - Rc::ptr_eq(&before.root_node, &candidate.root_node) - && before.input_groups == candidate.input_groups - }); - let mut inputs = format!("inputs=[{}]", group_list(&candidate.input_groups)); - if !candidate.fused_groups.is_empty() { - let _ = write!(inputs, " fuses=[{}]", group_list(&candidate.fused_groups)); - } - if candidate.accuracy_budget_parts != 1 { - let _ = write!( - inputs, - " accuracy_budget_parts={}", - candidate.accuracy_budget_parts - ); - } - vec![ - format!("[{index}]"), - format!("{:?}", candidate.origin), - if added { - "new".to_owned() - } else { - String::new() - }, - chain_label(memo, &candidate.root_node, &candidate.input_groups), - inputs, - ] - }) - .collect(); - write_rows(out, " ", rows); + let _ = (previous, &memo.groups[id].candidates); + todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo .query_roots @@ -315,112 +224,8 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi } fn write_physical(out: &mut String, physical: &PhysicalASAPMemoCandidates) { - let nodes = &physical.operator_nodes_of_candidates_for_memo_groups; - let _ = writeln!( - out, - "operator_nodes_of_candidates_for_memo_groups ({}):", - nodes.len() - ); - let rows = nodes - .iter() - .enumerate() - .map(|(id, node)| { - let implements: Vec = node - .candidates_for_memo_group - .iter() - .map(|candidate| format!("G{}[{}]", candidate.group.0, candidate.index)) - .collect(); - let mut role = if node.is_implementation_root { - "implementation root".to_owned() - } else { - "private".to_owned() - }; - if node.accuracy_budget_parts != 1 { - let _ = write!( - role, - " accuracy_budget_parts={}", - node.accuracy_budget_parts - ); - } - vec![ - format!("O{id}:"), - root_label(&node.physical_node), - format!("implements=[{}]", implements.join(", ")), - role, - ] - }) - .collect(); - write_rows(out, " ", rows); - - let _ = writeln!(out, "candidates ({}):", physical.candidates.len()); - let rows = physical - .candidates - .iter() - .enumerate() - .map(|(id, candidate)| { - // One `{...}` per input: the candidates that may produce it. - let producers: Vec = candidate - .producer_candidates_per_input - .iter() - .map(|producers| { - let ids: Vec = producers.iter().map(|p| format!("C{}", p.0)).collect(); - format!("{{{}}}", ids.join(", ")) - }) - .collect(); - vec![ - format!("C{id}:"), - format!("O{}", candidate.operator_node.0), - format!("{:?}", candidate.timing), - format!("{:?}", candidate.materialization), - // It may produce a query's output. - if candidate.may_produce_query_output { - "output".to_owned() - } else { - String::new() - }, - reads(&producers), - ] - }) - .collect(); - write_rows(out, " ", rows); - - let output_groups: Vec = physical - .query_roots - .iter() - .map(|root| match root { - QueryRoot::Operator(group) => format!("G{}", group.0), - scalar => format!("{scalar:?}"), - }) - .collect(); - let _ = writeln!(out, "query_roots: [{}]", output_groups.join(", ")); - let _ = writeln!( - out, - "reader_constraints ({}):", - physical.reader_constraints.len() - ); - for constraint in &physical.reader_constraints { - let _ = writeln!(out, " {constraint:?}"); - } - let _ = writeln!( - out, - "pruned_candidates ({}):", - physical.pruned_candidates.len() - ); - let rows = physical - .pruned_candidates - .iter() - .map(|pruned| { - let key = &pruned.candidate; - vec![ - format!("G{}", pruned.group.0), - format!("O{}", key.operator_node.0), - format!("{:?}", key.timing), - format!("{:?}", key.materialization), - format!("{:?}", pruned.reason), - ] - }) - .collect(); - write_rows(out, " ", rows); + let _ = (out, physical); + todo!("#1207 PR 9: print each physical candidate's properties and its edges with α") } /// Stage 3's selection: the search's outcome, the selected DAG (each node's @@ -428,90 +233,8 @@ fn write_physical(out: &mut String, physical: &PhysicalASAPMemoCandidates) { /// node's charge, and each query's accepted accuracy; or why none was /// selected. fn write_selection(out: &mut String, stages: &PlanningStagesRun) { - let run = &stages.run; - let selection = match &run.selection { - Ok(selection) => selection, - Err(error) => { - let _ = writeln!(out, "error: {error}"); - return; - } - }; - let report = &selection.search_report; - let _ = writeln!( - out, - "search: {:?}, {} MILP variables, {} constraints, {} ms solving", - report.status, report.milp_variables, report.milp_constraints, report.solver_ms - ); - let dag = &selection.selected_dag; - let rows = dag - .candidate_of_node - .iter() - .enumerate() - .map(|(node, candidate)| { - let mut inputs: Vec<_> = dag - .edges - .iter() - .filter(|edge| edge.consumer.0 == node) - .collect(); - inputs.sort_by_key(|edge| edge.input); - let producers: Vec = inputs - .iter() - .map(|edge| format!("node{}", edge.producer.0)) - .collect(); - // The operator the candidate runs. - let operator = - run.physical_asap_candidates - .as_ref() - .map_or(String::new(), |physical| { - let node = physical.candidates[candidate.0].operator_node; - root_label( - &physical.operator_nodes_of_candidates_for_memo_groups[node.0] - .physical_node, - ) - }); - vec![ - format!("node{node}:"), - format!("C{}", candidate.0), - operator, - reads(&producers), - ] - }) - .collect(); - write_rows(out, "", rows); - let outputs: Vec = dag - .node_of_output_reference - .iter() - .map(|node| format!("node{}", node.0)) - .collect(); - let _ = writeln!(out, "node_of_output_reference: [{}]", outputs.join(", ")); - let cost = &selection.cost; - let _ = writeln!( - out, - "cost_per_second: {}", - cost.cost_per_second - .map_or("none".to_owned(), |cost| format!("{cost:.6}")) - ); - // Keyed by the selected plan's node ids, as Stage 2 built it. - let rows = cost - .per_node - .iter() - .map(|(node, charge)| { - vec![ - format!("#{node}"), - format!("{:?}", charge.operator), - format!("{:?}", charge.timing), - charge - .cost_per_second - .map_or("none".to_owned(), |cost| format!("{cost:.6}")), - charge.detail.clone(), - ] - }) - .collect(); - write_rows(out, " ", rows); - let _ = writeln!(out, "accepted accuracy:"); - for (query, accepted) in selection.accepted_accuracy_per_query.iter().enumerate() { - let _ = writeln!(out, " q{}: {accepted}", query + 1); - } + let _ = (out, stages); + todo!("#1207 PR 9: print the selected DAG over the new physical candidates") } /// The label of `node`'s own operator, without its children. @@ -520,101 +243,5 @@ fn root_label(node: &Rc) -> String { flat.nodes.last().map(node_label).unwrap_or_default() } -/// A candidate's own nodes, from `root` down to its `input_groups`' nodes -/// (excluded), root first, joined by " ← ". -fn chain_label(memo: &Memo, root: &Rc, input_groups: &[MemoGroupId]) -> String { - let (flat, originals) = flatten(&[QueryRoot::Operator(root.clone())]); - let is_input = |id: usize| { - input_groups - .iter() - .any(|group| Rc::ptr_eq(&memo.groups[group.0 as usize].logical_node, &originals[id])) - }; - let mut labels = Vec::new(); - let mut pending = vec![flat.nodes.len() - 1]; - let mut seen = HashSet::new(); - while let Some(id) = pending.pop() { - if is_input(id) || !seen.insert(id) { - continue; - } - labels.push(node_label(&flat.nodes[id])); - pending.extend( - flat.nodes[id] - .operator - .children() - .into_iter() - .rev() - .copied(), - ); - } - labels.join(" ← ") -} - #[cfg(test)] -mod tests { - use super::*; - use crate::planner_layering_examples::example1; - use crate::planning_stages::{lower_promql_roots, run_stages_until}; - use asap_planner::PlanningModels; - use asap_types::workload::Query; - - /// One repeating `sum(rate(...[1m]))` query: every section is printed, - /// Pass 2 adds tumbling-window candidates with their layout, Stage 2 offers - /// the tumbling-window chain at ingestion time, and Stage 3 names the selected choice. - #[test] - fn prints_every_stage_of_a_repeating_rate_query() { - let mut workload = example1(); - let queries = workload.query_workload.repeating_queries.as_mut().unwrap(); - queries.truncate(1); - queries[0].query = Query("sum by (job) (rate(http_requests_total[1m]))".into()); - let roots = lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let stages = run_stages_until( - roots, - &entries, - &workload.data_workload.clone().unwrap(), - PlanningModels::builtin(), - RunUntil::Stage3, - ) - .unwrap(); - - let text = planning_stages_text(&stages); - - for heading in [ - "Stage 0 output", - "Stage 0 → Stage 1: DagOfLogicalMemoGroups", - "Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates", - "Stage 1 Pass 2, after local window composition", - "Stage 2 input: LogicalASAPMemoCandidates", - "Stage 3 input: PhysicalASAPMemoCandidates", - "Stage 3 output: Selection", - ] { - assert!(text.contains(heading), "missing {heading}:\n{text}"); - } - // A group line names its operator and the groups it reads. - assert!(text.contains("G1: TimeRange 60s <- G0\n"), "{text}"); - // The line containing every one of `parts`, columns aside. - let has_line = |parts: &[&str]| { - text.lines() - .any(|line| parts.iter().all(|part| line.contains(part))) - }; - // Pass 2 adds a tumbling-window candidate reading the scan, its layout in its chain. - assert!( - has_line(&[ - "TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000", - "inputs=[G0]", - ]), - "{text}" - ); - // Stage 2 offers the tumbling-window chain at ingestion time, read from the - // ingestion-time scan. - assert!( - has_line(&["IngestionTime MatAtIngestion(Memory)", "<- {C1}"]), - "{text}" - ); - assert!( - text.contains("selected choice: G0:c0 G1:c0 G2:c0 G3:c0\n"), - "{text}" - ); - assert!(text.contains("accepted accuracy:\n q1: "), "{text}"); - } -} +mod tests {} diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 5f655d3b2..dfa0b9512 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -5,7 +5,6 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; -use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; use asap_types::ir::flat::FlatDag; use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::QueryRoot; @@ -1058,134 +1057,6 @@ fn milp_lower_bound_is_consistent_with_the_selected_cost() { } } -/// A dashboard panel (`max_over_time(m[5m])` every minute over 1M series -/// sampled every 1.5 s) answered as the last hour at 1-minute steps -/// (`range_evaluation`) is priced as 61 evaluations per request: the -/// selected plan costs no more than the tumbling windows kept across evaluations, and -/// the document reports their retention, 1 h + 5 min. The workload section -/// shows the range. An invalid range is refused. -#[test] -fn workload_file_with_a_range_evaluation_is_planned() { - let workload = json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [{ - "query": "max_over_time(m[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}, - }], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666666.67}}, - "sources": {"m": {"series_count": 1000000, "sample_interval": 1500}}, - }, - }); - let capabilities = TempFile::new(r#"{"query_time_results_retained": true}"#); - let document = document( - &workload, - &[ - "--run-until", - "stage3", - "--ignore-accuracy-guarantees", - "--deployment-capabilities", - capabilities.path(), - ], - ); - assert_eq!( - document["workload"]["queries"][0]["requirements"]["range_evaluation"], - json!({"step_ms": 60000, "span_ms": 3600000}) - ); - let selection = &document["stage3_plan_selection"]; - let cost = selection["cost"]["cost_per_second"].as_f64().unwrap(); - // The reference: the window merged from tumbling windows kept across evaluations, - // priced under the same models. Its absolute cost is pinned in - // integration-tests/tests/range_evaluation.rs. - let kept_tumbling_windows = { - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; - use asap_types::ir::ASAPOp; - use asap_types::physical::{Materialization, Medium}; - let parsed: PlanningWorkload = serde_json::from_value(workload.clone()).unwrap(); - let entries: Vec<_> = parsed.query_workload.entries().collect(); - let data = parsed.data_workload.clone().unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &lower_promql_roots(&parsed).unwrap(), - ), - ) - .unwrap(); - let exact: Vec<_> = compose_logical_candidate(&inventory, &[0]).unwrap(); - let grids: Vec<_> = entries.iter().map(|e| e.evaluation_grid_ms()).collect(); - let tumbling_windows = tumbling_window_candidates(&exact, &grids); - let windowed = window_composition_pass2_candidates(&exact, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let deployment = asap_planner::DeploymentCapabilities { - query_time_results_retained: true, - ..Default::default() - }; - let kept = asap_physical_asap_optimization::optimize_physical_asap( - "kept-tumbling-windows", - &recompose_guarantees(&windowed).unwrap(), - &entries, - &data, - deployment.query_time_results_retained, - ) - .unwrap() - .into_iter() - .find(|plan| { - plan.dag.nodes.iter().any(|n| { - matches!( - n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && n.materialization - == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) - }) - }) - .expect("a plan keeps its tumbling windows across evaluations"); - let settings = asap_cost::CostInputs::default(); - let priced = asap_devtools::memo_pricing::reference_select( - std::slice::from_ref(&kept), - &entries, - &data, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(&deployment), - ) - .unwrap(); - priced.costs[&priced.selected].cost_per_second.unwrap() - }; - assert!( - cost <= kept_tumbling_windows * (1.0 + 1e-9), - "selected {SELECTED} costs {cost}, more than the kept tumbling windows' {kept_tumbling_windows}" - ); - assert!(selection["tumbling_window_retention_ms"] - .as_object() - .unwrap() - .values() - .any(|retention| retention.as_u64() == Some(3_900_000))); - - let mut invalid = workload.clone(); - invalid["query_workload"]["repeating_queries"][0]["range_evaluation"]["step_ms"] = json!(0); - let invalid = TempFile::json(&invalid); - let stderr = refused(&["--workload", invalid.path(), "--run-until", "stage0"]); - assert!( - stderr.contains("step_ms must be greater than zero"), - "{stderr}" - ); -} - /// The default is to enforce accuracy guarantees: Pass 1 prunes every /// sketch candidate of an average of p99s, which has no guarantee /// (`NoErrorBoundForQueryTarget`). With `--ignore-accuracy-guarantees`, the @@ -1459,230 +1330,6 @@ fn a_scan_two_queries_share_is_offered_built_once_per_evaluation() { assert!(cost <= 0.143, "{cost}"); } -/// A scan and a time range that already have every label of their metric -/// are not widened, so no memo group's candidate reads its own group, and -/// Stage 2's candidates, whose producers are its input groups' candidates, -/// read no candidate that reads them back. -#[test] -fn widened_scans_keep_memo_groups_and_stage2_candidates_acyclic() { - use asap_logical_asap_optimization::DeploymentCapabilities; - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s_within_1_percent("sum by (job, region) (rate(data[5m]))"), - every_10s_within_1_percent("sum by (job) (rate(data[5m]))"), - every_10s_within_1_percent("sum by (job, region) (data)"), - every_10s_within_1_percent("sum by (job) (data)"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "counter", "sample_interval": 15000}}, - }, - })) - .unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let deployment = DeploymentCapabilities { - raw_data_retained: true, - ..DeploymentCapabilities::default() - }; - let run = run_stages_until( - lower_promql_roots(&workload).unwrap(), - &entries, - workload.data_workload.as_ref().unwrap(), - asap_planner::PlanningModels::builtin().with_deployment_capabilities(&deployment), - RunUntil::Stage2, - ) - .unwrap() - .run; - let memo = &run.logical_asap_candidates.as_ref().unwrap().memo; - assert!(memo - .candidates - .iter() - .flatten() - .any(|candidate| candidate.origin - == asap_types::stages::OriginOfCandidateForMemoGroup::WidenedScans)); - // Whether `to` is reachable from `from` along `edges`. - fn reaches(edges: &dyn Fn(usize) -> Vec, from: usize, to: usize) -> bool { - let mut seen = HashSet::new(); - let mut pending = edges(from); - while let Some(next) = pending.pop() { - if next == to { - return true; - } - if seen.insert(next) { - pending.extend(edges(next)); - } - } - false - } - let group_reads = |group: usize| -> Vec { - memo.candidates[group] - .iter() - .flat_map(|candidate| candidate.input_groups.iter().map(|g| g.0 as usize)) - .collect() - }; - for group in 0..memo.groups.len() { - assert!( - !reaches(&group_reads, group, group), - "memo group {group} reads itself" - ); - } - let physical = run.physical_asap_candidates.as_ref().unwrap(); - let candidate_reads = |candidate: usize| -> Vec { - physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .flat_map(|producers| producers.iter().map(|id| id.0)) - .collect() - }; - for candidate in 0..physical.candidates.len() { - assert!( - !reaches(&candidate_reads, candidate, candidate), - "Stage 2 candidate {candidate} reads itself" - ); - } -} - -/// #840's workload: Q1 and Q2 window `data` (1 m and 5 m), Q3 and Q4 read -/// its latest samples by (host, job) and by job, so the four scans project -/// different labels. The widening rule's one scan containing every label -/// installs, compiles and is charged once per evaluation, and the selected -/// plan costs no more than it or than the partial-sharing plan (#994). -#[test] -fn scans_projecting_different_labels_are_shared() { - let workload = json!({ - "query_workload": { - "language": "promql", - "query_batch": null, - "repeating_queries": [ - every_10s_within_1_percent("quantile_over_time(0.5, data[1m])"), - every_10s_within_1_percent("quantile_over_time(0.9, data[5m])"), - every_10s_within_1_percent("quantile by (host, job) (0.5, data)"), - every_10s_within_1_percent("quantile by (job) (0.5, data)"), - ], - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", "sample_interval": 15000}}, - }, - }); - let document = document( - &workload, - &["--run-until", "stage3", "--ignore-accuracy-guarantees"], - ); - let workload: PlanningWorkload = serde_json::from_value(workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let roots = lower_promql_roots(&workload).unwrap(); - use asap_logical_asap_optimization::test_support::with_union_scan_labels; - use asap_types::physical::{Materialization, Medium}; - // Plans `roots` with identical sub-DAGs made one node, as the memo's - // groups are, and a node read by several roots kept for the evaluation. - let plan = |name: &str, roots: &[QueryRoot]| { - let shared: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, std::rc::Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let physical = asap_physical_asap_optimization::optimize_physical_asap_assigned( - name, - &shared, - &entries, - data, - false, - &mut |readers| { - Some(if readers.len() > 1 { - Materialization::MatAtQuery(Medium::Memory) - } else { - Materialization::NotMaterialized - }) - }, - ) - .unwrap() - .expect("a node read by several roots can be kept for the evaluation"); - let offered = asap_devtools::memo_pricing::reference_select( - std::slice::from_ref(&physical), - &entries, - data, - asap_planner::PlanningModels::builtin(), - ) - .unwrap(); - (physical, offered) - }; - // The widened scan: one scan with every label, shared by all four. - let widened = with_union_scan_labels(&roots).unwrap().unwrap(); - let (physical, offered) = plan("widened-scan-rule", &widened); - // The partial-sharing reference: the two windows share a narrow scan, and - // the two instant selectors a scan widened to their labels. - let mut partial = roots[..2].to_vec(); - partial.extend(with_union_scan_labels(&roots[2..]).unwrap().unwrap()); - let (_, reference) = plan("partial-sharing", &partial); - let (installed, _) = asap_planner::installable_plan_of( - &physical.roots, - None, - &offered.tumbling_window_retention_ms, - &entries, - ) - .unwrap(); - installed.validate().unwrap(); - installed.check_compiles().unwrap(); - let dag = &physical.dag; - let scans: Vec<_> = dag - .nodes - .iter() - .filter(|n| n.payload.kind_name() == "Scan") - .collect(); - let [scan] = &scans[..] else { - panic!("one scan: {scans:?}") - }; - let field_names: Vec<&str> = scan - .output_schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - assert_eq!( - field_names, - ["ts", "value", "host", "job", "$promql_series_identity"] - ); - let cost = &offered.costs[&offered.selected]; - let scan_cost = &cost.per_node[&scan.id]; - assert_eq!(scan_cost.multiplier, 1, "{scan_cost:?}"); - // 10,000 series × 20 samples in 5 m, once per 10 s evaluation. - let detail = &scan_cost.detail; - assert!( - detail.starts_with("scan 200000 samples; x 0.1000/s"), - "{detail}" - ); - let shared_cost = cost.cost_per_second.unwrap(); - assert!(shared_cost.is_finite() && shared_cost > 0.0); - let selected_cost = document["stage3_plan_selection"]["cost"]["cost_per_second"] - .as_f64() - .unwrap(); - let reference_cost = reference.costs[&reference.selected] - .cost_per_second - .unwrap(); - assert!( - selected_cost <= shared_cost, - "{selected_cost} > {shared_cost}" - ); - assert!( - selected_cost <= reference_cost * (1.0 + 1e-9), - "{selected_cost} > partial sharing {reference_cost}" - ); -} - /// The search is the candidate-connection search; its report has no /// legacy route. #[test] diff --git a/crates/devtools/tests/asap_planner_acceptance.rs b/crates/devtools/tests/asap_planner_acceptance.rs index 7043d0767..7567cbde3 100644 --- a/crates/devtools/tests/asap_planner_acceptance.rs +++ b/crates/devtools/tests/asap_planner_acceptance.rs @@ -73,13 +73,10 @@ const SECTIONS: [(&str, &[&str]); 5] = [ "stage0", &[ "Stage 0 output: the frontends' roots", - "Stage 0 → Stage 1: DagOfLogicalMemoGroups", + "Stage 0 → Stage 1: Memo (the roots hash-consed, without candidates)", ], ), - ( - "stage1_pass1", - &["Stage 1 Pass 1 output: PerComputationLogicalMemoCandidates"], - ), + ("stage1_pass1", &["Stage 1 Pass 1 output: Memo"]), ( "stage1_pass2", &[ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json index d803d90f7..5cb411f72 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q02.json @@ -721,7 +721,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -861,8 +861,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -954,7 +954,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1044,7 +1044,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1112,7 +1112,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1197,7 +1197,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1257,7 +1257,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json index b986b993f..4d42935a1 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q05.json @@ -880,7 +880,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1034,8 +1034,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1127,7 +1127,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1217,7 +1217,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1285,7 +1285,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1356,7 +1356,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1404,7 +1404,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1528,8 +1528,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1613,7 +1613,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1695,7 +1695,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1755,7 +1755,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1826,7 +1826,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1874,7 +1874,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1932,7 +1932,7 @@ } }, "id": 13, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1971,7 +1971,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2040,7 +2040,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json index 99affb00a..8cca94ae9 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q06.json @@ -569,7 +569,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -709,8 +709,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -802,7 +802,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -892,7 +892,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -960,7 +960,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1045,7 +1045,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1105,7 +1105,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1172,7 +1172,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1211,7 +1211,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ @@ -1280,7 +1280,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": false, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json index 16f3c5c73..33b444675 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q11.json @@ -328,7 +328,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -438,7 +438,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -495,7 +495,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -566,7 +566,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -614,7 +614,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -653,7 +653,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -722,7 +722,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json index 2a4d99a83..8f1b22d92 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q12.json @@ -133,7 +133,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -274,7 +274,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json index cc49899a7..a491b2f63 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q18.json @@ -7148,7 +7148,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7319,7 +7319,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7391,8 +7391,8 @@ { "guarantee": null, "id": 2, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7484,7 +7484,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7574,7 +7574,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7642,7 +7642,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7713,7 +7713,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7761,7 +7761,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7902,7 +7902,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7966,8 +7966,8 @@ { "guarantee": null, "id": 9, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -8051,7 +8051,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8133,7 +8133,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8193,7 +8193,7 @@ { "guarantee": null, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8264,7 +8264,7 @@ } }, "id": 13, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8312,7 +8312,7 @@ } }, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json index 2d5150c66..2e30b8d58 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q19.json @@ -7012,7 +7012,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7166,8 +7166,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7259,7 +7259,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7349,7 +7349,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7417,7 +7417,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7488,7 +7488,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7536,7 +7536,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7660,8 +7660,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7745,7 +7745,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7827,7 +7827,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7887,7 +7887,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7958,7 +7958,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8006,7 +8006,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json index 6b44c713b..3adc6ca62 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q20.json @@ -219,7 +219,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -329,7 +329,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -386,7 +386,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -457,7 +457,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json index 8546dd2b2..3ffc88b31 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q21.json @@ -694,7 +694,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -835,7 +835,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -899,8 +899,8 @@ { "guarantee": null, "id": 2, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -984,7 +984,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1066,7 +1066,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1126,7 +1126,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1197,7 +1197,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json index b7c91f320..e0a83a8d0 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q22.json @@ -630,7 +630,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -754,8 +754,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -839,7 +839,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -921,7 +921,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -981,7 +981,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1052,7 +1052,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json index 2ad030eed..5156b5315 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q23.json @@ -560,7 +560,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -684,8 +684,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -769,7 +769,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -851,7 +851,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -911,7 +911,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -982,7 +982,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json index 9f4988b12..c69ea0939 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q25.json @@ -5332,7 +5332,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5486,8 +5486,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -5579,7 +5579,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5669,7 +5669,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5737,7 +5737,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5808,7 +5808,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5856,7 +5856,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -5980,8 +5980,8 @@ { "guarantee": null, "id": 7, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -6065,7 +6065,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6147,7 +6147,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6207,7 +6207,7 @@ { "guarantee": null, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6278,7 +6278,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6326,7 +6326,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json index e33bf4553..fbdeba67f 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q26.json @@ -2618,7 +2618,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2711,8 +2711,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2788,7 +2788,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2862,7 +2862,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2914,7 +2914,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2985,7 +2985,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json index e119b2b13..e87f425a7 100644 --- a/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json +++ b/crates/devtools/tests/corpus_snapshots/o11y_bench/q27.json @@ -219,7 +219,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -329,7 +329,7 @@ } }, "id": 1, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -386,7 +386,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -457,7 +457,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/main.rs b/crates/devtools/tests/main.rs index 4952ae75e..ed12546d0 100644 --- a/crates/devtools/tests/main.rs +++ b/crates/devtools/tests/main.rs @@ -5,6 +5,5 @@ mod asap_planner; mod asap_planner_acceptance; mod cross_language; -mod memo_pricing; mod planner_fuzz; mod viewer_contract; diff --git a/crates/devtools/tests/memo_pricing.rs b/crates/devtools/tests/memo_pricing.rs deleted file mode 100644 index 1bdc9e850..000000000 --- a/crates/devtools/tests/memo_pricing.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! Pricing a memo's enumerated choices: on small workloads every choice is -//! priced, and none that is valid is cheaper than the plan the search -//! selected. -use asap_cost::CostInputs; -use asap_devtools::memo_pricing::{price_choices, PricedChoice, SelectedBaseline}; -use asap_devtools::planner_layering_examples::example1; -use asap_devtools::planning_stages::{lower_promql_roots, run_stages_until, RunUntil}; -use asap_logical_asap_optimization::enumerate; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; -use asap_logical_asap_optimization::test_support::try_populate_pass1; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_planner::PlanningModels; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::QueryRoot; -use asap_types::stages::Memo; -use asap_types::workload::{ - DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, QueryWorkloadEntry, RepeatedDemand, RepeatingEntry, - RepetitionInterval, -}; - -/// The issue's workload file: the query every minute, ε = 0.01, δ = 0.001. -const RATIO_OF_QUANTILES_WORKLOAD: &str = - "../../tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json"; - -const RATIO_OF_QUANTILES: &str = "avg by (job) (quantile_over_time(0.5, data[5m]) \ - / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))"; - -/// One query evaluated every minute at max_error 0.01, failure probability -/// 0.001, over 1,000 series of `data`, each sampled every 15 s. -fn ratio_of_quantiles() -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(RATIO_OF_QUANTILES.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).failure_probability(0.001), - max_latency_ms: None, - }, - predictability: Predictability::Unknown, - range_evaluation: None, - }]), - }, - data_workload: Some({ - let mut data = DataWorkload::sampled_every(["data".to_string()], DurationMs(15_000)); - for facts in data.sources.values_mut() { - facts.series_count = Some(1_000); - } - data - }), - } -} - -/// The workload's roots, entries and data, and its Pass 1 memo. -struct Planned { - roots: Vec, - entries: Vec, - data: DataWorkload, - memo: Memo, -} - -/// `asap_planner` lowers each series' identity as a column -/// (`lower_promql_roots`); without it (`series_identity` false, as the -/// memo-enumeration tests lower), the ratio's two `quantile 0.5` have no -/// provable unique key, but they are still one group. -fn planned(workload: &PlanningWorkload, series_identity: bool) -> Planned { - let roots = if series_identity { - lower_promql_roots(workload).unwrap() - } else { - asap_frontend_promql::lower_promql_query_workload(workload) - .unwrap() - .into_iter() - .collect() - }; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - Planned { - roots, - entries: workload.query_workload.entries().collect(), - data: workload.data_workload.clone().unwrap(), - memo: try_populate_pass1(initial, &inventory).unwrap(), - } -} - -/// `asap_planner`'s models: the executor's summary families, -/// `settings` (accuracy guarantees ignored unless enforced). -fn models<'a>( - settings: &'a Settings, - deployment: &'a DeploymentCapabilities, -) -> PlanningModels<'a> { - PlanningModels::builtin() - .with_cost_inputs(&settings.cost_inputs) - .with_deployment_capabilities(deployment) - .with_ignore_accuracy_guarantees(settings.ignore_accuracy_guarantees) -} - -fn deployment() -> DeploymentCapabilities { - DeploymentCapabilities { - supported_summaries: Some(asap_executor::supported_summary_families()), - ..DeploymentCapabilities::default() - } -} - -/// `asap_planner`'s cost inputs, and whether it ignores accuracy -/// guarantees. -struct Settings { - cost_inputs: CostInputs, - ignore_accuracy_guarantees: bool, -} - -fn settings(enforce_accuracy_guarantees: bool) -> Settings { - Settings { - cost_inputs: CostInputs::default(), - ignore_accuracy_guarantees: !enforce_accuracy_guarantees, - } -} - -/// `asap_planner`'s selection of `planned`'s workload. -fn selected(planned: &Planned, models: PlanningModels<'_>) -> SelectedBaseline { - let run = run_stages_until( - planned.roots.clone(), - &planned.entries, - &planned.data, - models, - RunUntil::Stage3, - ) - .unwrap(); - SelectedBaseline::of(run.run.selection.as_ref().expect("a selection")) -} - -/// Every choice of `planned`'s Pass 1 memo (the enumeration is complete), -/// priced against `baseline`. -fn price_every_choice( - planned: &Planned, - models: PlanningModels<'_>, - baseline: SelectedBaseline, -) -> Vec { - let enumeration = enumerate(&planned.memo, 100_000); - assert!(enumeration.complete); - price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models, - Some(baseline), - ) -} - -/// Every choice of `workload`'s Pass 1 memo, as `asap_planner` builds it -/// (the enumeration is complete), priced against the plan `asap_planner` -/// selects: none is cheaper, every valid one costs at least as much, and -/// the others say why they were rejected. Returns the priced choices. -fn assert_no_pass1_choice_is_cheaper( - workload: &PlanningWorkload, - enforce_accuracy_guarantees: bool, -) -> Vec { - let planned = planned(workload, true); - let (settings, deployment) = (settings(enforce_accuracy_guarantees), deployment()); - let models = models(&settings, &deployment); - let run = run_stages_until( - planned.roots.clone(), - &planned.entries, - &planned.data, - models, - RunUntil::Stage3, - ) - .unwrap(); - let selected = SelectedBaseline::of(run.run.selection.as_ref().expect("a selection")); - let memo = &run.run.per_computation_candidates.as_ref().unwrap().memo; - let enumeration = enumerate(memo, 100_000); - assert!(enumeration.complete); - let priced = price_choices( - memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models, - Some(selected), - ); - let selected = selected.cost_per_second.unwrap(); - for choice in &priced { - assert!(!choice.cheaper_than_selected, "{choice:?}"); - if choice.valid { - assert!(choice.cost_per_second.unwrap() >= selected, "{choice:?}"); - assert_eq!(choice.rejection, None); - } else { - assert!(choice.rejection.is_some(), "{choice:?}"); - } - } - assert!(priced.iter().any(|choice| choice.valid)); - priced -} - -/// #509 Example 1: its Pass 1 memo stands for 3 DAGs -/// (Q2's top-k group: exact, Count-Min with a heap and Count Sketch with a -/// heap); -/// Q2's summary reads miss its 100 ms latency bound, and no valid DAG is -/// cheaper than the selection. -#[test] -fn example1_no_pass1_choice_is_cheaper_than_the_selected_plan() { - let choices = assert_no_pass1_choice_is_cheaper(&example1(), false); - assert_eq!(choices.len(), 3); - assert!(choices - .iter() - .any(|c| c.rejection.as_deref() == Some("latency_bound_exceeded"))); -} - -/// The issue's ratio of quantiles as `asap_planner` lowers it (series -/// identity: the two `quantile 0.5` share one group): no Pass 1 DAG is -/// cheaper than the selection, with accuracy guarantees ignored or -/// enforced (the default). Enforced, Pass 1 prunes every summary: no query -/// has a guarantee through `/` and `+` over a KLL rank error or a DDSketch -/// relative one, so every listed DAG is exact and valid. -#[test] -fn ratio_of_quantiles_no_pass1_choice_is_cheaper_than_the_selected_plan() { - let workload: PlanningWorkload = - serde_json::from_str(&std::fs::read_to_string(RATIO_OF_QUANTILES_WORKLOAD).unwrap()) - .unwrap(); - let ignored = assert_no_pass1_choice_is_cheaper(&workload, false); - assert!(ignored.len() >= 25); - let enforced = assert_no_pass1_choice_is_cheaper(&workload, true); - assert!(!enforced.is_empty()); - assert!(enforced.iter().all(|choice| choice.valid)); -} - -/// Lowered without series identity, the two `quantile 0.5` have no unique -/// key but return the same rows, so they are still one group: 5 × 5 = 25 -/// DAGs (two distinct approximate inputs), 9 of them sized for the whole -/// budget. The executor compiles none of their per-series summaries without -/// the series identity, so only the exact DAG compiles and is valid. Its price is not -/// compared with the plan selected for the series-identity lowering: its -/// rows carry no identity column, so it prices lower than the same exact -/// DAG with the identity (#1035). -#[test] -fn ratio_of_quantiles_25_pass1_choices_without_identity_only_exact_is_valid() { - let without_identity = planned(&ratio_of_quantiles(), false); - let enumeration = enumerate(&without_identity.memo, 100_000); - let whole_budget = enumeration - .all_chosen_candidates - .iter() - .filter(|choice| { - choice.iter().all(|(group, &index)| { - without_identity.memo.candidates[group.0 as usize][index].accuracy_budget_parts == 1 - }) - }) - .count(); - assert_eq!( - (enumeration.all_chosen_candidates.len(), whole_budget), - (25, 9) - ); - let with_identity = planned(&ratio_of_quantiles(), true); - let deployment = deployment(); - for enforce in [false, true] { - let settings = settings(enforce); - let models = models(&settings, &deployment); - let baseline = selected(&with_identity, models); - let priced = price_every_choice(&without_identity, models, baseline); - assert_eq!(priced.iter().filter(|p| p.valid).count(), 1); - } -} - -/// A selection one step worse than the cheapest valid choice (the -/// next-cheapest cost) makes exactly the cheaper choices -/// `cheaper_than_selected`. -#[test] -fn a_worse_selection_is_caught_by_cheaper_than_selected() { - let planned = planned(&example1(), true); - let (settings, deployment) = (settings(false), deployment()); - let models = models(&settings, &deployment); - let choices = enumerate(&planned.memo, 100).all_chosen_candidates; - let price = |selected| { - price_choices( - &planned.memo, - &choices, - &planned.roots, - &planned.entries, - &planned.data, - models, - selected, - ) - }; - let unpriced = price(None); - assert!(unpriced.iter().all(|p| !p.cheaper_than_selected)); - let mut costs: Vec = unpriced.iter().filter_map(|p| p.cost_per_second).collect(); - costs.sort_by(f64::total_cmp); - costs.dedup(); - assert!(costs.len() >= 2, "two valid choices of different cost"); - let worse = SelectedBaseline { - cost_per_second: Some(costs[1]), - calibration_factor: None, - }; - let priced = price(Some(worse)); - for (before, after) in unpriced.iter().zip(&priced) { - assert_eq!(before.cost_per_second, after.cost_per_second); - assert_eq!( - after.cheaper_than_selected, - after.cost_per_second == Some(costs[0]) - ); - } - assert!(priced.iter().any(|p| p.cheaper_than_selected)); -} - -/// A query whose root is a scalar expression reading two operator DAGs: -/// each reference is replaced by its own root group's composed DAG, so -/// every choice composes and is priced. -#[test] -fn scalar_root_choices_are_composed_and_priced() { - let mut workload = ratio_of_quantiles(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = Query( - "scalar(sum(quantile_over_time(0.5, data[5m]))) + scalar(max(rate(data[1m])))".into(), - ); - let planned = planned(&workload, true); - assert!(matches!(planned.roots[0], QueryRoot::Scalar(_))); - let (settings, deployment) = (settings(false), deployment()); - let enumeration = enumerate(&planned.memo, 100_000); - assert!(enumeration.complete && enumeration.all_chosen_candidates.len() > 1); - let priced = price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models(&settings, &deployment), - None, - ); - for p in &priced { - assert_ne!(p.rejection.as_deref(), Some("compose_failed"), "{p:?}"); - } - assert!(priced.iter().any(|p| p.valid)); -} - -/// A choice whose selected plan installation refuses for a reason other -/// than the executor (here a raw-data window under a subquery, which no -/// runtime can place) is `not_installable`, never valid, so it is never -/// flagged cheaper than the selected plan. -#[test] -fn a_choice_installation_refuses_is_not_installable() { - let mut workload = ratio_of_quantiles(); - workload.query_workload.repeating_queries.as_mut().unwrap()[0].query = - Query("max_over_time(rate(data[5m])[1h:1m])".into()); - workload - .query_workload - .repeating_queries - .as_mut() - .unwrap() - .truncate(1); - let planned = planned(&workload, true); - let (settings, deployment) = (settings(false), deployment()); - let enumeration = enumerate(&planned.memo, 100_000); - let priced = price_choices( - &planned.memo, - &enumeration.all_chosen_candidates, - &planned.roots, - &planned.entries, - &planned.data, - models(&settings, &deployment), - None, - ); - assert!(!priced.is_empty()); - for p in &priced { - assert!(!p.valid && !p.cheaper_than_selected, "{p:?}"); - } - let refused: Vec<_> = priced - .iter() - .filter(|p| p.rejection.as_deref() == Some("not_installable")) - .collect(); - assert!(!refused.is_empty(), "{priced:?}"); - for p in refused { - let reason = p.rejection_reason.as_deref().unwrap(); - assert!( - reason.contains("a subquery or a TimeShift above the range"), - "{reason}" - ); - } -} diff --git a/crates/devtools/tests/sketch_plan_snapshots.rs b/crates/devtools/tests/sketch_plan_snapshots.rs index e3c9d1be5..1ab0097d1 100644 --- a/crates/devtools/tests/sketch_plan_snapshots.rs +++ b/crates/devtools/tests/sketch_plan_snapshots.rs @@ -6,7 +6,7 @@ //! queries or nested windows, answer a ratio of two quantiles from one tumbling window //! chain, and materialize tumbling windows, under the MVP deployment (raw data not //! retained) and raw data retained. A change to sketch kinds, parameters, -//! sizes, sharing, timing, materialization or cost shows up as a snapshot +//! sizes, sharing, timing, lifetime or cost shows up as a snapshot //! diff. //! //! `tests/sketch_plan_snapshots/manifest.json` lists each workload by id: diff --git a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json index 225a81a8a..f27b75631 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json +++ b/crates/devtools/tests/sketch_plan_snapshots/fuzz-issue903-raw-retained.json @@ -378,7 +378,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -488,8 +488,8 @@ } }, "id": 1, - "materialization": { - "MatAtQuery": "Memory" + "lifetime": { + "OneEvaluation": "Memory" }, "output_schema": { "closed": true, @@ -564,7 +564,7 @@ } }, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -634,7 +634,7 @@ { "guarantee": null, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -750,7 +750,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -834,7 +834,7 @@ } }, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json index 9b4fb2c61..54a2d6f95 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/nested-windows-15m-1h-mvp.json @@ -2802,7 +2802,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2895,8 +2895,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2986,7 +2986,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3085,7 +3085,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3176,7 +3176,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3249,7 +3249,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3347,7 +3347,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3438,7 +3438,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json index 3baaa111e..37361b98d 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-grouped-mvp.json @@ -1898,7 +1898,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2007,8 +2007,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2106,7 +2106,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2212,7 +2212,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2311,7 +2311,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2398,7 +2398,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2573,7 +2573,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2616,7 +2616,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2701,7 +2701,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2761,7 +2761,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2932,7 +2932,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3092,7 +3092,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json index 4f357790a..6583082f6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/quantile-single-mvp.json @@ -1461,7 +1461,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1554,8 +1554,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1647,7 +1647,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1739,7 +1739,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1823,7 +1823,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json index 04ea77719..d569fff3a 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-long-w01-q3-mvp.json @@ -12158,7 +12158,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12267,8 +12267,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -12366,7 +12366,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12472,7 +12472,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12571,7 +12571,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12658,7 +12658,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12833,7 +12833,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12876,7 +12876,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -12961,7 +12961,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13021,7 +13021,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13192,7 +13192,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -13352,7 +13352,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json index f9db2cbc5..95d124a1c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w01-q3-q8-mvp.json @@ -6490,7 +6490,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6599,8 +6599,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -6698,7 +6698,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6804,7 +6804,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6903,7 +6903,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -6990,7 +6990,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7165,7 +7165,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7208,7 +7208,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7293,7 +7293,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7353,7 +7353,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7524,7 +7524,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7684,7 +7684,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7759,7 +7759,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7868,8 +7868,8 @@ { "guarantee": null, "id": 13, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7953,7 +7953,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8035,7 +8035,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8112,7 +8112,7 @@ } }, "id": 16, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8201,7 +8201,7 @@ } }, "id": 17, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8283,7 +8283,7 @@ } }, "id": 18, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json index 6de034914..94d046d27 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/random-mixed-seed-862-w02-q2-q7-mvp.json @@ -7250,7 +7250,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7359,8 +7359,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -7458,7 +7458,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7564,7 +7564,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7663,7 +7663,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7750,7 +7750,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7925,7 +7925,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -7968,7 +7968,7 @@ { "guarantee": null, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8053,7 +8053,7 @@ } }, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8113,7 +8113,7 @@ } }, "id": 9, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8284,7 +8284,7 @@ } }, "id": 10, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8444,7 +8444,7 @@ } }, "id": 11, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8519,7 +8519,7 @@ } }, "id": 12, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8628,8 +8628,8 @@ { "guarantee": null, "id": 13, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -8713,7 +8713,7 @@ { "guarantee": null, "id": 14, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8795,7 +8795,7 @@ } }, "id": 15, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8872,7 +8872,7 @@ } }, "id": 16, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -8961,7 +8961,7 @@ } }, "id": 17, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -9043,7 +9043,7 @@ } }, "id": 18, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json index 0d170be65..266118af6 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/ratio-of-quantiles-1h-mvp.json @@ -2890,7 +2890,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2983,8 +2983,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -3074,7 +3074,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3172,7 +3172,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3263,7 +3263,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3336,7 +3336,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3434,7 +3434,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3525,7 +3525,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3598,7 +3598,7 @@ { "guarantee": null, "id": 8, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json index 227c9d39c..fa094b72a 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/shared-quantiles-1h-mvp.json @@ -2802,7 +2802,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2895,8 +2895,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -2986,7 +2986,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3084,7 +3084,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3175,7 +3175,7 @@ } }, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3248,7 +3248,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3346,7 +3346,7 @@ } }, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -3437,7 +3437,7 @@ } }, "id": 7, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json index b49a890c2..3d269fb2c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json +++ b/crates/devtools/tests/sketch_plan_snapshots/topk-rate-guarantees-ignored-mvp.json @@ -1571,7 +1571,7 @@ } }, "id": 0, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1680,8 +1680,8 @@ { "guarantee": null, "id": 1, - "materialization": { - "MatAtIngestion": "Memory" + "lifetime": { + "AcrossEvaluations": "Memory" }, "output_schema": { "closed": true, @@ -1765,7 +1765,7 @@ { "guarantee": null, "id": 2, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1847,7 +1847,7 @@ } }, "id": 3, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1907,7 +1907,7 @@ { "guarantee": null, "id": 4, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -1984,7 +1984,7 @@ { "guarantee": null, "id": 5, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ @@ -2032,7 +2032,7 @@ { "guarantee": null, "id": 6, - "materialization": "NotMaterialized", + "lifetime": "None", "output_schema": { "closed": true, "fields": [ diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index f7a2e3afd..00aa39b12 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -8,12 +8,11 @@ //! //! Stored-state encodings belong to deployments. Full plan acceptance is //! owned by `binding`, which also validates schemas, expressions and inputs. -use crate::summary_kernels::hll_sketch::HLL_PRECISIONS; -use crate::summary_kernels::weighted_frequency::{self, FrequencyAlgorithm}; + use crate::Error; use planner_types::ir::schema::{ ExactKind, ExactParams, FamilyCapability, FieldDataType as SummaryFamilyType, GroupingStrategy, - SketchAlgorithm, SketchParams, SketchStatistic, StatisticKind, SummaryUpdate, + SketchAlgorithm, SketchStatistic, StatisticKind, SummaryUpdate, }; /// Check the same contract used by `create_planner_accumulator` before a plan @@ -47,76 +46,8 @@ pub fn validate_summary_kernel( if layout != grouping { return Err("Planner family and operator grouping disagree".into()); } - use SketchAlgorithm as A; - use SketchParams as P; - match (kind.algorithm(), kind.params()) { - // `asap_sketchlib` silently caps a larger `k`. - (A::Kll, P::Kll { k, .. }) - if (8..=planner_types::ir::schema::sketch::KLL_MAX_K).contains(k) => - { - false - } - (A::DDSketch, P::DDSketch { alpha }) - if alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 => - { - false - } - (A::Hll, P::Hll { precision, .. }) if HLL_PRECISIONS.contains(precision) => false, - (A::Kmv, P::Kmv { k, .. }) if *k >= crate::summary_kernels::kmv::KMV_MIN_K => false, - (A::Cms, P::Cms { width, depth }) if valid_matrix(*width, *depth) => true, - // The kernel's shape rules (an odd depth, for the median). - (A::CountSketch, P::CountSketch { width, depth }) - if valid_matrix(*width, *depth) - && weighted_frequency::check_shape( - FrequencyAlgorithm::CountSketch, - *width as usize, - *depth as usize, - ) - .is_ok() => - { - true - } - ( - A::CmsWithHeap, - P::CmsWithHeap { - width, - depth, - heap_size, - }, - ) - | ( - A::CountSketchWithHeap, - P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ) if valid_matrix(*width, *depth) && *heap_size > 0 => true, - ( - A::UnivMon, - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - }, - ) if crate::summary_kernels::univmon::check_shape( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - ) - .is_ok() => - { - false - } - _ => { - return Err(format!( - "unsupported kernel or invalid parameters: {kind:?}" - )) - } - } + let _ = kind; + todo!("#1207 PR 6: check θ, which the exported node carries") } _ => return Err(format!("unsupported summary kernel {family:?}")), }; @@ -188,41 +119,20 @@ pub(crate) fn constant_row_weight(weight: f64) -> Result Result<(usize, usize, usize, usize), String> { - use planner_types::ir::schema::{HydraKind, HydraParams}; + use planner_types::ir::schema::HydraKind; let SummaryFamilyType::Sketch( kind, GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: - HydraParams::HydraCms { - width, - depth, - shared_rows, - shared_columns, - }, }, ) = family else { return Err("a shared-grouping (Hydra) family has no registered kernel".into()); }; - if kind.algorithm() != &SketchAlgorithm::Cms - || kind.params() - != &(SketchParams::Cms { - width: *width, - depth: *depth, - }) - { + if kind != &SketchAlgorithm::Cms { return Err("HydraCms family must be Count-Min with the Hydra width and depth".into()); } - if !valid_matrix(*width, *depth) || !valid_matrix(*shared_columns, *shared_rows) { - return Err("invalid HydraCms dimensions".into()); - } - Ok(( - *shared_rows as usize, - *shared_columns as usize, - *width as usize, - *depth as usize, - )) + todo!("#1207 PR 6: the Hydra grid's sizes are θ, which the exported node carries") } /// A filtered `SummaryAgg` never runs at ingestion time. An ingestion-time @@ -230,7 +140,7 @@ pub(crate) fn hydra_cms_shape( /// none of the query's columns for the filter predicate to read, so only a /// query-time build can apply it. The ingestion-time compiler and the /// binding of any ingestion-timed `SummaryAgg` apply this check. The planner -/// applies the same rule: Stage 2 never offers `MatAtIngestion` for a +/// applies the same rule: Stage 2 never offers materializing at ingestion time for a /// filtered build (`IngestionPruneReason::Filtered`), and materialization /// timing rejects an explicit one (`FilteredSummaryAtIngestion`). This check /// guards plans that did not come from the planner. @@ -243,6 +153,7 @@ pub fn validate_ingestion_time_summary( } } +#[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. fn valid_matrix(width: u32, depth: u32) -> bool { // Construction uses the kernel's native row hashing, so no encoded-size // limit applies here. @@ -279,36 +190,19 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { // Plain Count-Min is native as stored state only: it merges and reads // its bare count, but the DAG does not build it from rows. HydraCms // is built by the all-groups summary build. - if let (A::Cms, SketchParams::Cms { width, depth }) = (kind.algorithm(), kind.params()) { + if let A::Cms = kind { if grouping != &GroupingStrategy::PerSubpopulationInstance { return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); } - return if valid_matrix(*width, *depth) { - Ok(()) - } else { - Err(Error::Invalid( - "invalid Count-Min dimensions or grouping strategy".into(), - )) - }; + todo!("#1207 PR 6: check the Count-Min θ, which the exported node carries") } - if matches!(kind.algorithm(), A::CmsWithHeap | A::CountSketchWithHeap) { - let (_, width, depth, _) = weighted_frequency::WeightedFrequency::configuration(kind)?; - return if valid_matrix(width as u32, depth as u32) && grouping == &Default::default() { - Ok(()) - } else { - Err(Error::Invalid( - "invalid weighted frequency dimensions or grouping strategy".into(), - )) - }; + if matches!(kind, A::CmsWithHeap | A::CountSketchWithHeap) { + todo!("#1207 PR 6: check the weighted-frequency θ, which the exported node carries") } } match family { SummaryFamilyType::ExactAggregate(..) => {} - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.algorithm(), - A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon - ) => {} + SummaryFamilyType::Sketch(A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon, _) => {} _ => { return Err(Error::Invalid( "summary family has no native DAG state implementation".into(), @@ -359,7 +253,7 @@ fn evaluates(family: &SummaryFamilyType, readout: StatisticKind) -> bool { SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { EVALUATED_READOUTS .iter() - .any(|(family, readouts)| family == kind.algorithm() && readouts.contains(&readout)) + .any(|(family, readouts)| family == kind && readouts.contains(&readout)) } SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { matches!(kind, HydraKind::HydraCms | HydraKind::HydraKll) diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 273597d46..4f40f2a06 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -40,58 +40,8 @@ impl Operator { row_weight: RowWeight, groups: Vec, ) -> Result { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - crate::values::validate_family(&family)?; - let SummaryFamilyType::Sketch(kind, _) = &family else { - return Err(invalid("keyed sketch required")); - }; - WeightedFrequency::configuration(kind)?; - validate_groups(&input, &groups)?; - if items.is_empty() { - return Err(invalid("keyed summary requires item columns")); - } - if let RowWeight::Column(weight) = row_weight { - if !matches!( - plain(&input, weight)?.0, - DataType::Float64 | DataType::Int64 - ) { - return Err(invalid("keyed summary weights must be Float64 or Int64")); - } - } - for &item in &items { - if !matches!( - plain(&input, item)?.0, - DataType::Utf8 - | DataType::Timestamp - | DataType::Int64 - | DataType::Float64 - | DataType::Bool - | DataType::Null - ) { - return Err(invalid("unsupported keyed summary identity type")); - } - } - let mut fields = groups - .iter() - .map(|&i| input.fields[i].clone()) - .collect::>(); - fields.push(SummaryField { - name: "state".into(), - dtype: family.clone(), - nullable: false, - table: None, - }); - Ok(Self { - kind: Kind::PerGroupKeyedSummaryBuild { - family, - items, - row_weight, - groups, - filter: None, - }, - inputs: vec![input], - output: schema(fields), - }) + let _ = (input, family, items, row_weight, groups); + todo!("#1207 PR 6: the weighted-frequency θ, which the exported node carries") } /// One summary state shared by all groups (a HydraCms grid), emitted as /// one row per group that reads its own part of it. Each row adds its @@ -177,29 +127,8 @@ impl Operator { k: usize, output: SchemaRef, ) -> Result { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - crate::values::validate_family(&field(&input, state)?.dtype)?; - let SummaryFamilyType::Sketch(kind, _) = &field(&input, state)?.dtype else { - return Err(invalid("keyed evaluation requires summary state")); - }; - let (_, _, _, capacity) = WeightedFrequency::configuration(kind)?; - if k > capacity || output.fields.len() <= input.fields.len() { - return Err(invalid("invalid keyed evaluation shape or capacity")); - } - if state + 1 != input.fields.len() - || output.fields[..state] != input.fields[..state] - || output.fields.last().unwrap().dtype != SummaryFamilyType::Plain(DataType::Float64) - { - return Err(invalid( - "keyed evaluation must preserve partitions and return a Float64 score", - )); - } - crate::values::validate_schema(&output)?; - Ok(Self { - kind: Kind::KeyedEvaluation { state, k }, - inputs: vec![input], - output, - }) + let _ = (input, state, k, output); + todo!("#1207 PR 6: the weighted-frequency capacity of θ, which the exported node carries") } /// One summary state per group, updated from the value of one input /// column per row (`time` orders counter samples): an exact aggregate, @@ -366,8 +295,8 @@ fn accepts_update_type(family: &SummaryFamilyType, dtype: &DataType) -> bool { }; match dtype { DataType::Float64 | DataType::Int64 => true, - DataType::Utf8 => matches!(kind.algorithm(), A::Kmv | A::Hll | A::UnivMon), - DataType::Bool => kind.algorithm() == &A::UnivMon, + DataType::Utf8 => matches!(kind, A::Kmv | A::Hll | A::UnivMon), + DataType::Bool => kind == &A::UnivMon, _ => false, } } @@ -401,7 +330,7 @@ impl Operator { /// non-integral total rather than rounding it. fn integral_count(family: &SummaryFamilyType, query: &SummaryEvaluation) -> bool { matches!(family, SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::ir::schema::SketchAlgorithm::Cms) + if kind == &planner_types::ir::schema::SketchAlgorithm::Cms) && matches!( query, SummaryEvaluation::Sketch(planner_types::ir::schema::SketchStatistic::PointCount { diff --git a/crates/executor/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs index fc4f78c91..4b4d58ff1 100644 --- a/crates/executor/src/physical_planner/logical.rs +++ b/crates/executor/src/physical_planner/logical.rs @@ -36,8 +36,6 @@ pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result Result bool { matches!(family, FieldDataType::Sketch(kind, _) if matches!( - kind.algorithm(), + kind, planner_types::ir::schema::SketchAlgorithm::UnivMon | planner_types::ir::schema::SketchAlgorithm::Hll )) @@ -1579,7 +1579,6 @@ fn local_scalar(expr: &ScalarExpr) -> Result PhysicalASAPDAG { PhysicalASAPDAG { @@ -1598,130 +1597,17 @@ mod compilation_tests { Err(Error::Invalid(_)) )); } - - /// `window_count` one-minute windows over one table `Scan`, the `i`-th - /// shifted `i` minutes back, each window a root (the shape of a - /// tumbling-window-expanded DAG), with the roots' node ids. - fn shifted_windows_over_one_scan(window_count: i64) -> (PhysicalASAPDAG, Vec) { - use planner_types::ir::operator::{Source, TimeShift}; - use planner_types::ir::physical_export::compile_physical_asap_workload; - use planner_types::ir::schema::{Field, Schema}; - use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - Operator, QueryRoot, - }; - let minute_ms = 60_000; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let windows: Vec<_> = (0..window_count) - .map(|i| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: i * minute_ms, - at: None, - }, - child: std::rc::Rc::clone(&scan), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(minute_ms as u64), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() - }) - .collect(); - // One timing cache, so every window reads the one `Scan`. The cache is - // keyed by node address, so `windows` stays alive while it runs. - let mut cache = NodeToTimedNodeCache::new(); - let roots: Vec<_> = windows - .iter() - .map(|window| { - let timed = apply_materialization_timings( - window, - &MaterializationAssignment::default(), - &mut cache, - ) - .unwrap(); - QueryRoot::Operator(timed) - }) - .collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); - let roots = dag.roots.iter().map(|&id| id as NodeId).collect(); - (dag, roots) - } - - /// Three one-minute windows shifted 0, 1 and 2 minutes back over one - /// `Scan`: the `Scan` reads the union `(T - 3m, T]`, and each window's - /// `TimeRange` ends its shift before `T` and is one minute wide. - #[test] - fn shifted_windows_over_one_scan_compile_to_their_windows() { - let (dag, roots) = shifted_windows_over_one_scan(3); - let compiled = serde_json::to_value(compile(&dag, &[], &roots).unwrap()).unwrap(); - let nodes = compiled["nodes"].as_object().unwrap(); - let scan_windows: Vec<_> = nodes - .values() - .filter_map(|node| node.get("RawData")) - .map(|slot| slot["window"].clone()) - .collect(); - assert_eq!( - scan_windows, - [serde_json::json!({"Within": {"end_offset_ms": 0, "width_ms": 180_000}})] - ); - let mut time_ranges: Vec<(i64, i64)> = nodes - .values() - .filter_map(|node| node.pointer("/Operator/operator/kind/TimeRange")) - .map(|kind| { - ( - kind["end_offset_ms"].as_i64().unwrap(), - kind["width_ms"].as_i64().unwrap(), - ) - }) - .collect(); - time_ranges.sort(); - assert_eq!( - time_ranges, - [(0, 60_000), (60_000, 60_000), (120_000, 60_000)] - ); - } - - /// 20,000 shifted windows over one `Scan` compile in linear time. A - /// search over every node per window, and over every root per reader of - /// the `Scan`, made the compile quadratic in the number of windows - /// (#993): in a debug build, 10,000 windows compiled in 5.4 s and 20,000 - /// in over 10 s before, 1.4 s and 3.1 s after. - #[test] - #[ignore = "slow: run by hand"] - fn many_windows_over_one_scan_compile_in_linear_time() { - let (dag, roots) = shifted_windows_over_one_scan(20_000); - let started = Instant::now(); - compile(&dag, &[], &roots).unwrap(); - eprintln!("compiled 20,000 windows in {:?}", started.elapsed()); - } } /// Helpers for the Planner's own tests (`test-support`). #[cfg(feature = "test-support")] pub mod test_support { use super::*; - use planner_types::physical::{Materialization, Medium}; + use planner_types::physical::{Lifetime, Medium}; use std::collections::HashMap; /// The materialized views at `nodes`, in that order: each node that is - /// not a view is marked `MatAtIngestion`, and it and every node below it + /// not a view is marked materialized at ingestion time, and it and every node below it /// run at ingestion time (their outputs and outgoing edges); then every /// view's `retention_ms` is set ([`PhysicalASAPDAG::set_retention`]) /// with no tumbling window or range-evaluation span. For tests that bind states at @@ -1743,7 +1629,10 @@ pub mod test_support { .iter() .filter(|node| { nodes.contains(&(node.id as NodeId)) - && !planner_types::test_support::is_materialized_view(node.materialization) + && !matches!( + node.lifetime, + planner_types::physical::Lifetime::AcrossEvaluations(_) + ) }) .map(|node| node.id as NodeId) .collect(); @@ -1757,7 +1646,7 @@ pub mod test_support { for node in &mut dag.nodes { let id = node.id as NodeId; if marked.contains(&id) { - node.materialization = Materialization::MatAtIngestion(Medium::Memory); + node.lifetime = Lifetime::AcrossEvaluations(Medium::Memory); } if at_ingestion.contains(&id) { node.output_state.timing = ExecutionTiming::IngestionTime; diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs index e3345397f..890a16fb8 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -7,7 +7,7 @@ use crate::summary_kernels::{ }; use crate::AggregateCore; use planner_types::ir::schema::{ - FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams, + FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchKind, SketchParams, }; /// Generate the clone-based `AccumulatorUpdater` methods for updaters whose @@ -462,8 +462,12 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { /// Construct the kernel declared by a Planner SummaryAgg. No deployment config /// tags participate in this dispatch and unsupported payloads are errors. +/// +/// `parameters` is θ, which the exported node carries +/// (`PhysicalASAPDAGNode::parameters`); `None` for an exact accumulator. pub fn create_planner_accumulator( family: &SummaryFamilyType, + parameters: Option<&SketchParams>, input: &planner_types::ir::schema::SummaryUpdate, grouping: &planner_types::ir::schema::GroupingStrategy, ) -> Result, String> { @@ -500,7 +504,10 @@ pub fn create_planner_accumulator( if family_grouping != grouping { return Err("Planner family and operator grouping disagree".into()); } - let updater: Box = match (kind.algorithm(), kind.params()) { + let Some(params) = parameters else { + return Err("a sketch summary needs its parameters θ".into()); + }; + let updater: Box = match (kind, params) { (SketchAlgorithm::Kll, SketchParams::Kll { k, .. }) => { if *k > planner_types::ir::schema::sketch::KLL_MAX_K { return Err("KLL k exceeds runtime bound".into()); @@ -525,7 +532,8 @@ pub fn create_planner_accumulator( } (SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, _) => { let (algorithm, width, depth, capacity) = - WeightedFrequency::configuration(kind).map_err(|e| e.to_string())?; + WeightedFrequency::configuration(&SketchKind::new(*kind, params.clone())) + .map_err(|e| e.to_string())?; Box::new( WeightedFrequencyUpdater::new(algorithm, width, depth, capacity) .map_err(|e| e.to_string())?, @@ -557,7 +565,7 @@ pub fn create_planner_accumulator( }), _ => { return Err(format!( - "unsupported Planner algorithm/parameters: {kind:?}" + "unsupported Planner algorithm/parameters: {kind:?} {params:?}" )) } }; @@ -672,132 +680,6 @@ impl AccumulatorUpdater for UnivMonUpdater { } } -#[cfg(test)] -mod planner_parameter_regression { - use super::*; - use planner_types::ir::schema::{SketchKind, SummaryInputExpr, SummaryUpdate}; - - // Planner width is the bucket count; depth is the independent hash-row count. - #[test] - fn planner_sketch_dimensions_are_not_transposed() { - for (algorithm, params) in [ - ( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 128, - depth: 3, - }, - ), - ( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 128, - depth: 3, - }, - ), - ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 128, - depth: 3, - heap_size: 8, - }, - ), - ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 128, - depth: 3, - heap_size: 8, - }, - ), - ] { - let family = - SummaryFamilyType::Sketch(SketchKind::new(algorithm, params), Default::default()); - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - }; - let state = create_planner_accumulator(&family, &update, &Default::default()) - .unwrap() - .snapshot_accumulator(); - let dims = match algorithm { - SketchAlgorithm::Cms => { - let s = state - .as_any() - .downcast_ref::() - .unwrap(); - (s.inner.rows(), s.inner.cols()) - } - SketchAlgorithm::CountSketch - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketchWithHeap => { - let (width, depth, _) = state - .as_any() - .downcast_ref::() - .unwrap() - .shape(); - (depth, width) - } - _ => unreachable!(), - }; - assert_eq!(dims, (3, 128), "{algorithm:?}"); - } - } - - // HydraCms builds one shared grid shaped by its Hydra params; it is never - // a per-group updater, and its family must be the matching Count-Min. - #[test] - fn hydra_cms_builds_one_shared_grid() { - use planner_types::ir::schema::{ - GroupingStrategy, HydraKind, HydraParams, NonNegativeWeightProof, WeightDomain, - }; - let grouping = |width| GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width, - depth: 3, - shared_rows: 5, - shared_columns: 32, - }, - }; - let family = |width| { - SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 128, - depth: 3, - }, - ), - grouping(width), - ) - }; - assert_eq!( - create_hydra_cms(&family(128)).unwrap().shape(), - (5, 32, 128, 3) - ); - assert!(create_hydra_cms(&family(64)).is_err()); - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("host".into()), - )), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - assert!( - crate::capability::validate_summary_kernel(&family(128), &update, &grouping(128)) - .is_ok() - ); - assert!(create_planner_accumulator(&family(128), &update, &grouping(128)).is_err()); - } -} - #[cfg(test)] mod ddsketch_domain { use super::*; diff --git a/crates/executor/src/summary_kernels/state_column_updater.rs b/crates/executor/src/summary_kernels/state_column_updater.rs index e059ab25c..358abed27 100644 --- a/crates/executor/src/summary_kernels/state_column_updater.rs +++ b/crates/executor/src/summary_kernels/state_column_updater.rs @@ -2,13 +2,13 @@ //! group: the code the `SummaryAgg` build runs row by row, exposed so a //! runtime that ingests sample by sample builds the same states. use super::exact::ScalarState; -use super::factory::{create_hydra_cms, create_planner_accumulator, AccumulatorUpdater}; +use super::factory::{create_hydra_cms, AccumulatorUpdater}; use super::hydra_cms::{hydra_group_name, hydra_item, hydra_weight, HydraCms, HydraCmsGroup}; use super::weighted_frequency::WeightedFrequency; use crate::values::{StateColumn, Value}; use crate::{AggregateCore, Error}; -use planner_types::ir::scalar::ColumnRef; -use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy, SummaryUpdate}; + +use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy}; use std::sync::Arc; /// The open summary states of one family, one per group, updated one sample @@ -47,6 +47,7 @@ enum GroupStates { Sketches(Vec>), /// A keyed frequency sketch (Count-Min or Count Sketch with a heap) per /// group. + #[allow(dead_code)] // Used again in #1207 PR 6, with θ from the exported node. Keyed { /// An empty state of the family's shape, cloned for each new group /// (boxed: much larger than the other kinds' fields). @@ -89,15 +90,15 @@ impl StateColumnUpdater { empty: ScalarState::empty(family).map_err(Error::Invalid)?, states: Vec::new(), }, - FieldDataType::Sketch(kind, grouping) + FieldDataType::Sketch(algorithm, grouping) if grouping == &GroupingStrategy::PerSubpopulationInstance - && WeightedFrequency::configuration(kind).is_ok() => + && matches!( + algorithm, + planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap + ) => { - let (algorithm, width, depth, capacity) = WeightedFrequency::configuration(kind)?; - GroupStates::Keyed { - empty: Box::new(WeightedFrequency::new(algorithm, width, depth, capacity)?), - states: Vec::new(), - } + todo!("#1207 PR 6: the keyed frequency state of θ, which the exported node carries") } FieldDataType::Sketch(..) => { // Fails for a family with no per-group value kernel. @@ -125,12 +126,8 @@ impl StateColumnUpdater { } fn sketch_updater(family: &FieldDataType) -> Result, Error> { - create_planner_accumulator( - family, - &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), - ) - .map_err(Error::Invalid) + let _ = family; + todo!("#1207 PR 6: build it with `create_planner_accumulator` and θ from the exported node's `parameters`") } /// The number of groups. diff --git a/crates/executor/src/supported_summaries.rs b/crates/executor/src/supported_summaries.rs index 6ad48c3e1..973ce0cfb 100644 --- a/crates/executor/src/supported_summaries.rs +++ b/crates/executor/src/supported_summaries.rs @@ -24,13 +24,12 @@ mod tests { use super::*; use crate::capability::validate_sketch_evaluation; use crate::factory::create_planner_accumulator; - use crate::summary_kernels::weighted_frequency::WeightedFrequency; + use planner_types::ir::operator::operator_properties::{Reduction, Source}; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ DataType, FamilyCapability, Field, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatisticKind, - SummaryInputExpr, SummaryUpdate, + SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryUpdate, }; use planner_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; @@ -56,57 +55,8 @@ mod tests { StatisticKind::TopK, ]; - /// Valid parameters of `algorithm`, as the Planner sizes them. - fn params(algorithm: &SketchAlgorithm) -> SketchParams { - use SketchAlgorithm as A; - match algorithm { - A::UnivMon => SketchParams::UnivMon { - heap_size: 32, - sketch_rows: 3, - sketch_cols: 1024, - layers: 8, - distinct_failure_probability: None, - }, - A::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - A::Cms => SketchParams::Cms { - width: 64, - depth: 3, - }, - A::Hll => SketchParams::Hll { - precision: 12, - relative_error: None, - }, - A::DDSketch => SketchParams::DDSketch { alpha: 0.01 }, - A::CmsWithHeap => SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 10, - }, - A::Kmv => SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - A::CountSketch => SketchParams::CountSketch { - width: 64, - depth: 3, - }, - A::CountSketchWithHeap => SketchParams::CountSketchWithHeap { - width: 64, - depth: 3, - heap_size: 10, - }, - other => unreachable!("{other:?} is not probed"), - } - } - fn per_instance(algorithm: &SketchAlgorithm) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(*algorithm, params(algorithm)), - GroupingStrategy::PerSubpopulationInstance, - ) + FieldDataType::Sketch(*algorithm, GroupingStrategy::PerSubpopulationInstance) } /// A statistic of kind `readout`. @@ -142,8 +92,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - create_planner_accumulator(family, &value, grouping).is_ok() - || create_planner_accumulator(family, &keyed, grouping).is_ok() + create_planner_accumulator(family, None, &value, grouping).is_ok() + || create_planner_accumulator(family, None, &keyed, grouping).is_ok() } /// Whether the executor answers `readout` from `family`'s state: top-k @@ -151,9 +101,11 @@ mod tests { /// sketch evaluation, whose kernel must then read it from a built state. fn evaluation_answers(family: &FieldDataType, readout: StatisticKind) -> bool { match (readout, family) { - (StatisticKind::TopK, FieldDataType::Sketch(kind, _)) => { - WeightedFrequency::configuration(kind).is_ok() - } + (StatisticKind::TopK, FieldDataType::Sketch(kind, _)) => matches!( + kind, + planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap + ), _ => { let accepted = validate_sketch_evaluation(family, &statistic(readout)).is_ok(); if accepted { @@ -175,8 +127,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - let Ok(mut updater) = create_planner_accumulator(family, &value, grouping) - .or_else(|_| create_planner_accumulator(family, &keyed, grouping)) + let Ok(mut updater) = create_planner_accumulator(family, None, &value, grouping) + .or_else(|_| create_planner_accumulator(family, None, &keyed, grouping)) else { return false; }; diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index 2bc7e0284..12f06cf90 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -609,9 +609,7 @@ impl Batch { mod tests { use super::*; use crate::summary_kernels::DDSketchAccumulator; - use planner_types::ir::schema::{ - DataType, Field, Schema, SketchAlgorithm, SketchKind, SketchParams, - }; + use planner_types::ir::schema::{DataType, Field, Schema, SketchAlgorithm}; fn field(name: &str, dtype: DataType, nullable: bool) -> Field { Field { @@ -774,13 +772,7 @@ mod tests { } fn dd_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - Default::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::DDSketch, Default::default()) } // Rows sharing a state keep sharing one state; filtering rows shares the @@ -1134,22 +1126,11 @@ mod tests { /// exact sums. fn state_batch(count: u32, exact: bool) -> Batch { use crate::StateColumnUpdater; - use planner_types::ir::schema::{ - ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, - }; + use planner_types::ir::schema::{ExactKind, ExactParams, SketchAlgorithm}; let family = if exact { FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) } else { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) }; let mut updater = StateColumnUpdater::new(&family).unwrap(); updater.ensure_groups(count as usize).unwrap(); diff --git a/crates/executor/src/values/mod.rs b/crates/executor/src/values/mod.rs index a28ec8f44..40c273375 100644 --- a/crates/executor/src/values/mod.rs +++ b/crates/executor/src/values/mod.rs @@ -210,29 +210,10 @@ pub(super) fn validate_state( family: &SummaryFamilyType, state: &dyn AggregateCore, ) -> Result<(), Error> { - use crate::summary_kernels::{ - count_min_sketch::CountMinSketchAccumulator, datasketches_kll::DatasketchesKLLAccumulator, - dd_sketch::DDSketchAccumulator, exact::ExactAccumulator, hll_sketch::HllSketchAccumulator, - kmv::KmvAccumulator, univmon::UnivMonAccumulator, - }; - use planner_types::ir::schema::SketchParams; + use crate::summary_kernels::exact::ExactAccumulator; + validate_family(family)?; let valid = match family { - SummaryFamilyType::Sketch(kind, _) - if matches!( - kind.params(), - SketchParams::CmsWithHeap { .. } | SketchParams::CountSketchWithHeap { .. } - ) => - { - use crate::summary_kernels::weighted_frequency::WeightedFrequency; - let (algorithm, width, depth, capacity) = WeightedFrequency::configuration(kind)?; - state - .as_any() - .downcast_ref::() - .is_some_and(|state| { - state.algorithm() == algorithm && state.shape() == (width, depth, capacity) - }) - } SummaryFamilyType::Sketch(_, grouping) if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance => @@ -247,49 +228,9 @@ pub(super) fn validate_state( .as_any() .downcast_ref::() .is_some_and(|s| s.family() == family && !s.is_keyed()), - SummaryFamilyType::Sketch(kind, _) => match kind.params() { - SketchParams::Kll { k, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.k() == *k as usize), - SketchParams::DDSketch { alpha } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.alpha() == *alpha), - SketchParams::Hll { precision, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.precision() == *precision), - SketchParams::Kmv { k, .. } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| s.inner.k == *k as usize), - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| { - s.dimensions() - == ( - *heap_size as usize, - *sketch_rows as usize, - *sketch_cols as usize, - usize::from(*layers), - ) - }), - SketchParams::Cms { width, depth } => state - .as_any() - .downcast_ref::() - .is_some_and(|s| { - (s.inner.rows(), s.inner.cols()) == (*depth as usize, *width as usize) - }), - _ => false, - }, + SummaryFamilyType::Sketch(..) => { + todo!("#1207 PR 6: compare the state with θ, which the exported node carries") + } _ => false, }; if valid { @@ -343,50 +284,22 @@ pub(crate) fn plain(schema: &SchemaRef, column: usize) -> Result<(&DataType, boo mod weighted_state_tests { use super::*; use crate::summary_kernels::weighted_frequency::{FrequencyAlgorithm, WeightedFrequency}; - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; // A state cannot acquire a different family or shape merely by relabeling its batch. #[test] fn weighted_state_family_and_shape_must_match() { - let cms = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 32, - depth: 5, - heap_size: 8, - }, - ), - Default::default(), - ); - let cs = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 32, - depth: 5, - heap_size: 8, - }, - ), - Default::default(), - ); + let cms = SummaryFamilyType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); + let cs = + SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); let state = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 32, 5, 8).unwrap(); assert!(validate_state(&cs, &state).is_ok()); assert!(validate_state(&cms, &state).is_err()); let wrong_shape = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 64, 5, 8).unwrap(); assert!(validate_state(&cs, &wrong_shape).is_err()); - let even_depth = SummaryFamilyType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 32, - depth: 4, - heap_size: 8, - }, - ), - Default::default(), - ); + let even_depth = + SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); assert!(validate_family(&even_depth).is_err()); } } diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index ee3b4435b..e397b15d8 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -186,7 +186,7 @@ fn cooperative_sort_preserves_ties_across_chunks() { // The integrated weighted-summary path obeys the same cooperative cancellation contract. #[test] fn weighted_summary_build_yields_within_a_batch() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; let input = Arc::new(planner_types::ir::schema::Schema { unique_keys: vec![], @@ -222,17 +222,7 @@ fn weighted_summary_build_yields_within_a_batch() { Operator::source(input.clone(), vec![batch]).unwrap(), ) .unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); let operator = Operator::per_group_keyed_summary_build( input, family, diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index 857d76bf9..cfacc2731 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -1,21 +1,7 @@ #![allow(dead_code)] -use planner_types::ir::physical_export::PhysicalASAPDAG; -use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, OperatorNode, - QueryRoot, -}; -use std::rc::Rc; -pub fn compile_physical_asap_dag( - root: &Rc, -) -> Result> { - let root = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::default(), - )?; - Ok(planner_types::ir::physical_export::compile_physical_asap_dag(&root)?) -} +use planner_types::ir::{OperatorNode, QueryRoot}; +use std::rc::Rc; /// The logical DAG the planner pipeline (`asap_planner::run_planning`) selects for the /// single query `root` at `accuracy`, over a continuously ingested source, @@ -42,12 +28,11 @@ pub fn computation_candidates(root: Rc) -> Vec> { use asap_logical_asap_optimization::test_support::{ compose_logical_candidate, generate_per_computation_candidates, }; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ QueryRoot::Operator(root), - ]), - ) - .expect("Pass 1 accepts the query"); + ])) + .expect("Pass 1 accepts the query"); let mut choices = vec![vec![0; inventory.targets.len()]]; for (index, target) in inventory.targets.iter().enumerate() { for choice in 1..target.choice_count() { diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index a78efe426..1368b006f 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -1,5 +1,5 @@ //! Spatial heap weights come from a fresh instant vector, never sample history. -use crate::common; + use asap_executor::sources::InMemoryDataStore; use asap_executor::{ operators::Operator, @@ -10,7 +10,6 @@ use asap_executor::{ runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use common::compile_physical_asap_dag; use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::PhysicalASAPOperatorPayload; use planner_types::ir::properties::AccuracyTarget; @@ -161,7 +160,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { SketchAlgorithm::CmsWithHeap, SketchAlgorithm::CountSketchWithHeap, ] { - let params = match algorithm { + let _params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width: 2048, depth: 5, @@ -173,7 +172,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { heap_size: 100, }, }; - let family = FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()); + let family = FieldDataType::Sketch(algorithm, Default::default()); let build = Operator::per_group_keyed_summary_build( schema(), family, @@ -341,7 +340,7 @@ fn population_fixture_compiles_with_dynamic_identity() { topk, &["job"], ); - let logical = compile_physical_asap_dag(&selected).unwrap(); + let logical = planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap(); let (source, raw_schema) = logical .nodes .iter() diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index a774d2289..bbd1c7a40 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -65,7 +65,8 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { }, ), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 70804d9f4..e79652f97 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -1,27 +1,20 @@ //! Exercise the public library without a backend server, store, or scheduler. use asap_executor::planner::{ ir::scalar::ColumnRef, - ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }, + ir::schema::{FieldDataType, GroupingStrategy, SketchAlgorithm, SketchParams, SummaryUpdate}, }; use asap_executor::{factory::create_planner_accumulator, AggregateCore}; -fn family(k: u32) -> FieldDataType { +fn family(_k: u32) -> FieldDataType { FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), + SketchAlgorithm::Kll, GroupingStrategy::PerSubpopulationInstance, ) } fn build(values: &[f64]) -> Box { let mut operator = create_planner_accumulator( &family(512), + None, &SummaryUpdate::column(ColumnRef::SampleValue), &Default::default(), ) @@ -58,6 +51,7 @@ fn raw_partial_and_fully_precomputed_use_the_same_kernels() { fn invalid_kll_parameters_are_rejected_at_binding() { let result = create_planner_accumulator( &family(0), + None, &SummaryUpdate::column(ColumnRef::SampleValue), &Default::default(), ); @@ -69,20 +63,11 @@ fn invalid_kll_parameters_are_rejected_at_binding() { #[test] fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { use asap_executor::planner::ir::schema::SummaryInputExpr; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 1200, - depth: 55, - heap_size: 3, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let mut operator = + create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); operator .update_keyed(&["a".to_string()], 7.0, 1000) .unwrap(); @@ -99,14 +84,8 @@ fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { ); } -fn plain_count_sketch(width: u32, depth: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - ), - Default::default(), - ) +fn plain_count_sketch(_width: u32, _depth: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::CountSketch, Default::default()) } fn host(i: usize) -> [String; 1] { @@ -129,7 +108,8 @@ fn build_weighted_frequency( use asap_executor::planner::ir::schema::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = create_planner_accumulator(family, &update, &Default::default()).unwrap(); + let mut operator = + create_planner_accumulator(family, None, &update, &Default::default()).unwrap(); for (at, (i, weight)) in updates.into_iter().enumerate() { operator.validate_single_input(weight).unwrap(); operator.update_keyed(&host(i), weight, at as i64).unwrap(); @@ -221,7 +201,7 @@ fn heap_sketch( depth: u32, heap_size: u32, ) -> FieldDataType { - let params = match algorithm { + let _params = match algorithm { SketchAlgorithm::CmsWithHeap => SketchParams::CmsWithHeap { width, depth, @@ -233,7 +213,7 @@ fn heap_sketch( heap_size, }, }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) + FieldDataType::Sketch(algorithm, Default::default()) } // The factory's top-k sketches use the same per-row-seeded kernel: past 20 @@ -339,7 +319,7 @@ fn count_min_heap_rejects_negative_inputs() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = heap_sketch(SketchAlgorithm::CmsWithHeap, 64, 3, 4); - let operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let operator = create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); assert!(operator.validate_single_input(-1.0).is_err()); assert!(operator.validate_single_input(0.5).is_ok()); } @@ -405,7 +385,8 @@ fn keyed_update_overflow_is_an_error_not_a_panic() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = plain_count_sketch(64, 3); - let mut operator = create_planner_accumulator(&family, &update, &Default::default()).unwrap(); + let mut operator = + create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); operator.validate_single_input(f64::MAX).unwrap(); operator.update_keyed(&host(0), f64::MAX, 0).unwrap(); let error = operator.update_keyed(&host(0), f64::MAX, 1).unwrap_err(); diff --git a/crates/executor/tests/deployment_computation.rs b/crates/executor/tests/deployment_computation.rs index 7fad67cef..080454c79 100644 --- a/crates/executor/tests/deployment_computation.rs +++ b/crates/executor/tests/deployment_computation.rs @@ -1,20 +1,20 @@ //! Planner-selected PromQL computation compiles from the timed DAG alone; //! the deployment supplies only raw rows at its time-series scans. -use crate::common; + use asap_executor::sources::InMemoryDataStore; use asap_executor::{ - physical_planner::{compile, promql_rows, test_support::views_at}, + physical_planner::{compile, promql_rows}, runtime::{Limits, RunContext, Scope}, values::{Batch, Value}, }; -use common::{compile_physical_asap_dag, computation_candidates, selected_dag}; + use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::maintained_population::PopulationStatistic; use planner_types::ir::operator::Source; use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; use planner_types::ir::properties::AccuracyTarget; use planner_types::ir::schema::*; -use planner_types::ir::ASAPOp; + use planner_types::workload::*; use std::{collections::BTreeMap, rc::Rc, sync::Arc}; @@ -52,13 +52,6 @@ fn lower_with(query: &str, accuracy: AccuracyTarget) -> Rc PhysicalASAPDAG { - let expression = lower(query); - let root = promql_rows::with_series_identity(&expression).unwrap_or(expression); - compile_physical_asap_dag(&selected_dag(root, AccuracyTarget::exact())).unwrap() -} - /// The population sub-DAG of `query`, which computes `statistic` grouped by /// `grouping` over one selector. fn population_dag( @@ -70,7 +63,7 @@ fn population_dag( let selected = asap_logical_asap_optimization::test_support::population_over_selector( &root, statistic, grouping, ); - compile_physical_asap_dag(&selected).unwrap() + planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap() } /// The time-series scans: each metric's raw data slot and its schema. @@ -265,433 +258,6 @@ fn global_population_aggregate_of_no_members_is_empty() { } } -// Scalar operands on either side apply to every grouped value, including negation. -#[test] -fn scalar_literal_arithmetic_applies_to_grouped_values() { - // sum_over_time over 5m per job: api = 4 + 1 + 7 + 2 = 14, db = 5. - for (query, expected) in [ - ( - "sum by (job) (sum_over_time(m[5m])) * 2", - reference(&[("api", 28.), ("db", 10.)]), - ), - ( - "100 - sum by (job) (sum_over_time(m[5m]))", - reference(&[("api", 86.), ("db", 95.)]), - ), - ( - "-sum by (job) (sum_over_time(m[5m]))", - reference(&[("api", -14.), ("db", -5.)]), - ), - ] { - assert_eq!( - run(&exact_dag(query), SAMPLES, 60_000).unwrap(), - expected, - "{query}" - ); - } -} - -// Grouped vectors match one-to-one on labels; unmatched groups are dropped and -// unchecked division by zero yields +Inf as in PromQL. -#[test] -fn grouped_vector_arithmetic_matches_labels() { - let samples: &[RawDataSample] = &[ - ("a", "api", "x", 10_000, 6.), - ("a", "api", "y", 20_000, 3.), - ("a", "db", "x", 10_000, 1.), - ("a", "web", "x", 10_000, 1.), - ("b", "api", "x", 10_000, 3.), - ("b", "db", "x", 10_000, 0.), - ("b", "cache", "x", 10_000, 1.), - ]; - let dag = - exact_dag("sum by (job) (sum_over_time(a[5m])) / sum by (job) (sum_over_time(b[5m]))"); - assert_eq!( - run(&dag, samples, 60_000).unwrap(), - reference(&[("api", 3.), ("db", f64::INFINITY)]) - ); -} - -// Exact observation counts finalize to the Float64 value PromQL declares, -// then roll up per job: api has 2 + 1 + 1 samples in 5m, db has 1. -#[test] -fn exact_count_finalizes_to_declared_float_value() { - let dag = exact_dag("sum by (job) (count_over_time(m[5m]))"); - assert_eq!( - run(&dag, SAMPLES, 60_000).unwrap(), - reference(&[("api", 4.), ("db", 1.)]) - ); -} - -/// Prometheus's `quantile` test series (`promql/promqltest/testdata/aggregators.test` -/// in Prometheus v0.314.0), one job per `test` label and one instance per -/// `point`, all sampled at 50 s. -const QUANTILE_SAMPLES: &[RawDataSample] = &[ - ("data", "two", "a", 50_000, 0.), - ("data", "two", "b", 50_000, 1.), - ("data", "three", "a", 50_000, 0.), - ("data", "three", "b", 50_000, 1.), - ("data", "three", "c", 50_000, 2.), - ("data", "uneven", "a", 50_000, 0.), - ("data", "uneven", "b", 50_000, 1.), - ("data", "uneven", "c", 50_000, 4.), - ("data", "nan", "a", 50_000, 0.), - ("data", "nan", "b", 50_000, 1.), - ("data", "nan", "c", 50_000, f64::NAN), -]; - -/// Asserts `(job, value)` rows equal `expected`, NaN equal to NaN and finite -/// values to within 1e-12 (Prometheus's own tests compare approximately too). -fn assert_rows(actual: &BTreeMap, expected: &[(&str, f64)], query: &str) { - let mut jobs: Vec<&str> = expected.iter().map(|(job, _)| *job).collect(); - jobs.sort(); - assert_eq!(actual.keys().collect::>(), jobs, "{query}"); - for (job, want) in expected { - let got = actual[*job]; - let same = if want.is_finite() { - (got - want).abs() <= 1e-12 * want.abs().max(1.) - } else { - got.to_bits() == want.to_bits() || (got.is_nan() && want.is_nan()) - }; - assert!(same, "{query}: {job} = {got}, want {want}"); - } -} - -// An exact instant `quantile by (G)` compiles to the grouped aggregate and -// returns Prometheus's values: linear interpolation between closest ranks, -// a NaN sample ordered first (so a rank between NaN and 0 is NaN), and ±Inf -// for φ outside [0, 1], NaN for a NaN φ. The 0.8, 0.2 and NaN rows are Prometheus's own expected -// values for `quantile without(point)` over the same series. -#[test] -fn exact_grouped_quantile_matches_prometheus() { - let (inf, nan) = (f64::INFINITY, f64::NAN); - for (query, expected) in [ - ( - "quantile by (job) (0.8, data)", - vec![("two", 0.8), ("three", 1.6), ("uneven", 2.8), ("nan", 0.6)], - ), - ( - "quantile by (job) (0.2, data)", - vec![("two", 0.2), ("three", 0.4), ("uneven", 0.4), ("nan", nan)], - ), - ( - "quantile by (job) (0, data)", - vec![("two", 0.), ("three", 0.), ("uneven", 0.), ("nan", nan)], - ), - ( - "quantile by (job) (1, data)", - vec![("two", 1.), ("three", 2.), ("uneven", 4.), ("nan", 1.)], - ), - ( - "quantile by (job) (-0.5, data)", - vec![ - ("two", -inf), - ("three", -inf), - ("uneven", -inf), - ("nan", -inf), - ], - ), - ( - "quantile by (job) (1.5, data)", - vec![("two", inf), ("three", inf), ("uneven", inf), ("nan", inf)], - ), - // Prometheus answers a NaN φ with NaN for every group. - ( - "quantile by (job) (NaN, data)", - vec![("two", nan), ("three", nan), ("uneven", nan), ("nan", nan)], - ), - // One group over every series: sorted [NaN, 0, 0, 0, 0, 1, 1, 1, 1, 2, 4], - // rank 0.5 * 10 = 5 → 1. - ("quantile(0.5, data)", vec![("", 1.)]), - ] { - let rows = run(&exact_dag(query), QUANTILE_SAMPLES, 60_000).unwrap(); - assert_rows(&rows, &expected, query); - } -} - -// With no input series the result is an empty vector, grouped or not, as in -// Prometheus; a group of one series returns that series' value for any φ in -// [0, 1]. -#[test] -fn exact_grouped_quantile_of_no_series_is_empty() { - for query in ["quantile by (job) (0.5, data)", "quantile(0.5, data)"] { - let rows = run(&exact_dag(query), &[], 60_000).unwrap(); - assert!(rows.is_empty(), "{query}: {rows:?}"); - } - let one: &[RawDataSample] = &[("data", "solo", "a", 50_000, 3.)]; - let query = "quantile by (job) (0.9, data)"; - assert_rows( - &run(&exact_dag(query), one, 60_000).unwrap(), - &[("solo", 3.)], - query, - ); -} - -/// `dag` with its Binary operator replaced by `kind`. -fn with_kind( - mut dag: PhysicalASAPDAG, - kind: planner_types::ir::operator::BinaryOpKind, - bool_result: bool, -) -> PhysicalASAPDAG { - for node in &mut dag.nodes { - if let PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { - operator, - return_bool, - .. - }) = &mut node.payload - { - operator.kind = kind.clone(); - *return_bool = bool_result; - } - } - dag -} - -// A comparison Binary over grouped values keeps the groups whose comparison -// holds, with their value, on either side of the literal; `bool` yields 1 or 0. -#[test] -fn grouped_comparisons_filter_or_return_bool() { - for (query, expected) in [ - ( - "sum by(job)(sum_over_time(m[5m])) > 10", - reference(&[("api", 14.)]), - ), - ( - "sum by(job)(sum_over_time(m[5m])) > bool 10", - reference(&[("api", 1.), ("db", 0.)]), - ), - ( - "10 > sum by(job)(sum_over_time(m[5m]))", - reference(&[("db", 5.)]), - ), - ] { - assert_eq!(run(&exact_dag(query), SAMPLES, 60_000).unwrap(), expected); - } -} - -// A `bool` comparison Binary over per-series evaluations matches one-to-one and -// drops the metric name; a filter keeps the surviving left value. -#[test] -fn per_series_comparisons_filter_or_return_bool() { - use planner_types::ir::operator::BinaryOpKind::*; - use planner_types::ir::scalar::CompareOpKind::*; - - let samples = counter("a", "api", 10., 10.) - .chain(counter("a", "db", 10., 10.)) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "db", 20., 20.)) - .collect::>(); - // rate: a{api} = a{db} = 50/300, b{api} = 25/300, b{db} = 100/300. - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - assert_eq!( - run_series( - &with_kind(dag.clone(), Compare(Gt), false), - &samples, - 300_000 - ) - .unwrap(), - series(&[("api", "x", 50. / 300.)]) - ); - assert_eq!( - run_series(&with_kind(dag, Compare(Lt), true), &samples, 300_000).unwrap(), - series(&[("api", "x", 0.), ("db", "x", 1.)]) - ); -} - -/// [`execute`], returning per-series `(identity, value)` rows of the root, -/// with NaN-aware formatting for comparison. -fn run_series( - dag: &PhysicalASAPDAG, - samples: &[RawDataSample], - end: i64, -) -> Result { - let mut rows = BTreeMap::new(); - for batch in execute(dag, samples, end)? { - let schema = batch.schema(); - let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); - let (identity, value) = (column(promql_rows::SERIES_IDENTITY_COLUMN), column("value")); - for row in &batch.to_rows() { - let (Value::Utf8(identity), Value::Float64(value)) = (&row[identity], &row[value]) - else { - return Err(format!("unexpected row {row:?}")); - }; - assert!(rows.insert(identity.to_string(), *value).is_none()); - } - } - Ok(format!("{rows:?}")) -} - -fn series(pairs: &[(&str, &str, f64)]) -> String { - let rows = pairs - .iter() - .map(|(job, instance, value)| { - ( - format!(r#"{{"instance":"{instance}","job":"{job}"}}"#), - *value, - ) - }) - .collect::>(); - format!("{rows:?}") -} - -/// Five counter samples per series, one per minute up to 300s, at `base + step * i`. -fn counter( - metric: &'static str, - job: &'static str, - base: f64, - step: f64, -) -> impl Iterator { - (1..=5).map(move |i| (metric, job, "x", i * 60_000, base + step * (i - 1) as f64)) -} - -// avg_over_time over stored per-series sum/count state divides per series and -// drops the metric name, as Prometheus does. An overflowing stored sum fails -// the checked division instead of returning +Inf. -#[test] -fn per_series_average_divides_stored_sum_by_count() { - let dag = exact_dag("avg_over_time(m[5m])"); - assert_eq!( - run_series(&dag, SAMPLES, 60_000).unwrap(), - series(&[ - ("api", "a", 2.5), - ("api", "b", 7.), - ("api", "c", 2.), - ("db", "d", 5.) - ]) - ); - let huge: &[RawDataSample] = &[ - ("m", "api", "a", 10_000, 1.7e308), - ("m", "api", "a", 20_000, 1.7e308), - ]; - assert!(run_series(&dag, huge, 60_000).is_err()); -} - -// rate(a) / rate(b) matches series on their labels without the metric name. -// Unmatched series are dropped; x/0 is +Inf and 0/0 is NaN; an empty side -// yields an empty vector. -#[test] -fn per_series_rate_ratio_matches_prometheus() { - // Window (0, 300s]: first sample at 60s, extrapolated 60s to the start - // (durationToZero is exactly 60s too), so rate = (last - first) * 1.25 / 300. - // a{api} = 40 * 1.25 / 300, b{api} = 20 * 1.25 / 300, so the ratio is 2. - let samples = counter("a", "api", 10., 10.) - .chain(counter("a", "db", 10., 10.)) - .chain(counter("a", "web", 10., 10.)) - .chain(counter("a", "cache", 7., 0.)) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "web", 7., 0.)) - .chain(counter("b", "cache", 7., 0.)) - .chain(counter("b", "other", 5., 5.)) - .collect::>(); - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - assert_eq!( - run_series(&dag, &samples, 300_000).unwrap(), - series(&[ - ("api", "x", 2.), - ("cache", "x", f64::NAN), - ("web", "x", f64::INFINITY), - ]) - ); - let only_a = samples - .iter() - .filter(|s| s.0 == "a") - .copied() - .collect::>(); - assert_eq!(run_series(&dag, &only_a, 300_000).unwrap(), series(&[])); -} - -// A literal operand applies to every stored per-series value, on either side, -// and drops the metric name. -#[test] -fn per_series_scalar_arithmetic_applies_to_stored_evaluations() { - let samples = counter("m", "api", 10., 10.).collect::>(); - // rate = 40 * 1.25 / 300 = 1/6. - for (query, expected) in [ - ("rate(m[5m]) * 2", 50. / 300. * 2.), - ("1 - rate(m[5m])", 1. - 50. / 300.), - // The stored sum evaluation keeps `__name__`; the arithmetic drops it. - ("sum_over_time(m[5m]) * 2", 150. * 2.), - ] { - assert_eq!( - run_series(&exact_dag(query), &samples, 300_000).unwrap(), - series(&[("api", "x", expected)]), - "{query}" - ); - } -} - -// A literal operand drops the metric name; series that then share a label set -// are an error, as in Prometheus, rather than duplicate output series. -#[test] -fn per_series_scalar_arithmetic_rejects_label_sets_equal_without_the_name() { - let dag = exact_dag("sum_over_time(m[5m]) * 2"); - let samples = counter("m", "api", 10., 10.) - .chain(counter("m", "api", 1., 1.).map(|s| (s.0, s.1, "y", s.3, s.4))) - .collect::>(); - let distinct = BTreeMap::from([("y", ("n", "y"))]); - assert!(execute_relabeled(&dag, &samples, 300_000, &distinct).is_ok()); - // Both series are then {instance="x",job="api"}, named m and n. - let equal = BTreeMap::from([("y", ("n", "x"))]); - let Err(error) = execute_relabeled(&dag, &samples, 300_000, &equal) else { - panic!("duplicate label sets were accepted"); - }; - assert!(error.contains("same labelset"), "{error}"); -} - -fn with_vector_match( - mut dag: PhysicalASAPDAG, - kind: planner_types::ir::operator::VectorMatchKind, - labels: &[&str], -) -> PhysicalASAPDAG { - for node in &mut dag.nodes { - if let PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { - operator, - .. - }) = &mut node.payload - { - operator.vector_match = Some(planner_types::ir::operator::VectorMatch { - kind: kind.clone(), - labels: labels.iter().map(|l| l.to_string()).collect(), - grouping: None, - }); - } - } - dag -} - -// `on` and `ignoring` reduce each side to the matching labels, which become -// the result's labels; a duplicate match group on either side is an error. -#[test] -fn per_series_vector_matching_follows_on_and_ignoring() { - use planner_types::ir::operator::VectorMatchKind; - let samples = counter("a", "api", 10., 10.) - .chain(counter("b", "api", 5., 5.)) - .chain(counter("b", "db", 5., 5.)) - .collect::>(); - let dag = exact_dag("rate(a[5m]) / rate(b[5m])"); - for (kind, labels) in [ - (VectorMatchKind::On, ["job"]), - (VectorMatchKind::Ignoring, ["instance"]), - ] { - let dag = with_vector_match(dag.clone(), kind, &labels); - assert_eq!( - run_series(&dag, &samples, 300_000).unwrap(), - format!( - "{:?}", - BTreeMap::from([(r#"{"job":"api"}"#.to_string(), 2.)]) - ) - ); - let mut duplicate = samples.clone(); - duplicate.extend(counter("b", "api", 5., 5.).map(|s| (s.0, s.1, "y", s.3, s.4))); - let error = run_series(&dag, &duplicate, 300_000).unwrap_err(); - assert!(error.contains("duplicate series"), "{error}"); - let mut duplicate = samples.clone(); - duplicate.extend(counter("a", "api", 5., 5.).map(|s| (s.0, s.1, "y", s.3, s.4))); - let error = run_series(&dag, &duplicate, 300_000).unwrap_err(); - assert!(error.contains("many-to-one"), "{error}"); - } -} - // Current-series sums and averages are compensated like Prometheus, and an // overflowing running sum does not turn the average into +Inf. #[test] @@ -728,98 +294,3 @@ fn population_sums_and_averages_are_compensated() { ); } } - -// A bare count over stored Count-Min state compiles to a Planner evaluation that -// returns the sketch's total update weight, including colliding items. -#[test] -fn stored_count_min_bare_count_compiles_to_a_evaluation() { - use asap_executor::summary_kernels::CountMinSketchAccumulator; - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_with("count(up)", accuracy.clone()); - let mut dag = computation_candidates(root) - .into_iter() - .find_map(|node| { - let dag = compile_physical_asap_dag(&node).ok()?; - let bare_count = dag.nodes.iter().any(|n| { - matches!( - &n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryEstimate { - query: SketchStatistic::PointCount { value: None, .. }, - .. - }) - ) - }); - let count_min = dag.nodes.iter().any(|n| { - matches!(&n.payload, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm() == &SketchAlgorithm::Cms) - }); - (bare_count && count_min).then_some(dag) - }) - .expect("Planner lists a Count-Min candidate for count(up)"); - let state = dag - .nodes - .iter() - .find(|n| { - matches!( - n.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - ) - }) - .unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &state.payload - else { - unreachable!() - }; - let SketchParams::Cms { width, depth } = kind.params() else { - unreachable!() - }; - let (state_id, schema) = (state.id as u64, Arc::new(state.output_schema.clone())); - let (width, depth) = (*width, *depth); - let views = views_at(&mut dag, &[state_id]).unwrap(); - let program = compile(&dag, &views, &[dag.roots[0] as u64]).unwrap(); - let mut sketch = CountMinSketchAccumulator::new(depth as usize, width as usize).unwrap(); - sketch.update("a", 3.0); - sketch.update("b", 7.0); - let row = schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::Plain(_) => panic!("unexpected stored column {field:?}"), - family => Value::Summary { - family: family.clone(), - state: Arc::new(sketch.clone()), - }, - }) - .collect(); - let batch = Batch::try_new(schema.clone(), vec![row]).unwrap(); - let store = InMemoryDataStore::default() - .with_materialized_view(state_id as usize, vec![batch]) - .unwrap(); - let physical_dag = program.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits::default(), - ) - .unwrap(); - let values = block_on(async { - let mut stream = physical_dag - .execute(program.roots(), context) - .unwrap() - .remove(0); - let mut values = Vec::new(); - while let Some(batch) = stream.next().await { - values.extend(batch.unwrap().to_rows().iter().map(|row| row[0].clone())); - } - values - }); - assert!( - matches!(values.as_slice(), [Value::Int64(10)]), - "{values:?}" - ); -} diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 7502bc10d..fc23257c5 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -17,7 +17,7 @@ use planner_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use planner_types::ir::schema::DataType; use planner_types::ir::schema::*; use planner_types::ir::{ASAPOp, ExprSemantics, Predicate, ScalarExpr}; -use planner_types::physical::Materialization; +use planner_types::physical::Lifetime; use std::{collections::BTreeMap, sync::Arc}; const WIDTH: u32 = 64; @@ -39,26 +39,11 @@ const ROWS: [(&str, &str, f64); 9] = [ fn grouping() -> GroupingStrategy { GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: WIDTH, - depth: 3, - shared_rows: 3, - shared_columns: SHARED_COLUMNS, - }, } } fn family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: WIDTH, - depth: 3, - }, - ), - grouping(), - ) + FieldDataType::Sketch(SketchAlgorithm::Cms, grouping()) } fn schema(fields: Vec) -> Schema { @@ -75,7 +60,8 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: output, guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 91f8bf032..19ecf85b9 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -17,7 +17,7 @@ use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::*; use planner_types::ir::ASAPOp; -use planner_types::physical::Materialization; +use planner_types::physical::Lifetime; use std::sync::Arc; /// `(job, service)` rows: jobs are the groups, services the items. @@ -32,14 +32,7 @@ const ROWS: [(&str, &str); 6] = [ fn heap_family() -> FieldDataType { FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 1024, - depth: 3, - heap_size: 8, - }, - ), + SketchAlgorithm::CmsWithHeap, GroupingStrategy::PerSubpopulationInstance, ) } @@ -159,7 +152,8 @@ fn query_time_count_heap_binds_a_constant_row_weight() { filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: (*schema(vec![ Field::plain("job", DataType::Utf8, false), Field { @@ -248,14 +242,7 @@ fn int64_weight_column_equals_float64_column() { counts(limit as f64) ); let count_sketch = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 1024, - depth: 3, - heap_size: 8, - }, - ), + SketchAlgorithm::CountSketchWithHeap, GroupingStrategy::PerSubpopulationInstance, ); let signed = Operator::per_group_keyed_summary_build( diff --git a/crates/executor/tests/main.rs b/crates/executor/tests/main.rs index 89abe7da8..ee0b768e5 100644 --- a/crates/executor/tests/main.rs +++ b/crates/executor/tests/main.rs @@ -17,7 +17,6 @@ mod physical_dag; mod physical_semantics; mod plan_properties; mod planspace_series_identity_heap; -mod precompute_candidates; mod precompute_population; mod promql_binary; mod promql_values; diff --git a/crates/executor/tests/node_ids.rs b/crates/executor/tests/node_ids.rs index 04a46fb54..961aa096d 100644 --- a/crates/executor/tests/node_ids.rs +++ b/crates/executor/tests/node_ids.rs @@ -1,7 +1,6 @@ //! Compilation rejects Planner node IDs outside the range its helper IDs avoid. -use crate::common; + use asap_executor::physical_planner::compile; -use common::compile_physical_asap_dag; use planner_types::ir::scalar::ScalarValue; use planner_types::ir::schema::{DataType, Field, Schema}; use planner_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; @@ -20,7 +19,7 @@ fn node_ids_above_u32_are_rejected() { child: values, })) .unwrap(); - let mut dag = compile_physical_asap_dag(&filter).unwrap(); + let mut dag = planner_types::test_support::physical_asap_dag_at_query_time(&filter).unwrap(); let root = dag.roots[0] as u64; assert!(compile(&dag, &[], &[root]).is_ok()); diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index bd22b97d5..7a0595e9e 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -350,17 +350,11 @@ fn binding_rejects_unsupported_operations() { // KLL is one family example: precomputation changes input sources, not operators. #[test] fn kll_raw_partial_and_precomputed_are_native_dags() { - use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm}; let input = schema(&[("value", DataType::Float64, false)]); let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 512, - failure_probability: None, - }, - ), + SketchAlgorithm::Kll, GroupingStrategy::PerSubpopulationInstance, ); let build = Operator::per_group_summary_build(input.clone(), family, 0, None, vec![]).unwrap(); @@ -539,7 +533,8 @@ fn bind_physical_asap_dag_before_execution() { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, @@ -700,7 +695,8 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { payload, output_schema: (**schema).clone(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }; @@ -975,7 +971,7 @@ fn weighted_rate_topk_preserves_partitions_fractional_scores_and_evaluation_scop } } fn assert_weighted_rate_topk(count_sketch: bool) { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; let raw = schema(&[ ("service", DataType::Utf8, false), @@ -1015,26 +1011,11 @@ fn assert_weighted_rate_topk(count_sketch: bool) { ) .unwrap(); let family = FieldDataType::Sketch( - SketchKind::new( - if count_sketch { - SketchAlgorithm::CountSketchWithHeap - } else { - SketchAlgorithm::CmsWithHeap - }, - if count_sketch { - SketchParams::CountSketchWithHeap { - width: 4096, - depth: 5, - heap_size: 8, - } - } else { - SketchParams::CmsWithHeap { - width: 4096, - depth: 5, - heap_size: 8, - } - }, - ), + if count_sketch { + SketchAlgorithm::CountSketchWithHeap + } else { + SketchAlgorithm::CmsWithHeap + }, Default::default(), ); let build = Operator::per_group_keyed_summary_build( @@ -1136,7 +1117,8 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*input).clone(), guarantee: None, retention_ms: None, @@ -1226,7 +1208,8 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { id: 2, output_schema: (*schema).clone(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Join { @@ -1331,7 +1314,8 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { id: 2, output_schema: (*input).clone(), output_state: ExecutionDataState::INGESTION_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, payload: PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::BinaryOp { diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 2831befe1..78bca3e36 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -385,7 +385,8 @@ fn global_extrema_bind_with_planner_derived_schema() { child: 0, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*output).clone(), guarantee: None, retention_ms: None, @@ -554,19 +555,10 @@ fn boolean_truth_tables_agree_between_expression_paths() { // Partial/final execution must agree with one build for an uncompacted KLL population. #[test] fn kll_partial_merge_and_multiple_evaluations_preserve_population() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams}; + use planner_types::ir::schema::SketchAlgorithm; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 512, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let mut dag = PhysicalDAG::default(); for (id, range) in [(0, 0..64), (1, 64..128), (2, 0..128)] { let rows = range.map(|n| vec![Value::Float64(n as f64)]).collect(); @@ -650,7 +642,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { // single-build rank error: 60 tumbling windows of 1000 values at k = 200. #[test] fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { - use planner_types::ir::schema::{SketchAlgorithm, SketchKind, SketchParams, SketchStatistic}; + use planner_types::ir::schema::{SketchAlgorithm, SketchStatistic}; const TUMBLING_WINDOW_COUNT: u64 = 60; const ROWS_PER_TUMBLING_WINDOW: u64 = 1000; const K: u32 = 200; @@ -659,16 +651,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { let bound = 2.296 / f64::from(K).powf(0.9723); let n = (TUMBLING_WINDOW_COUNT * ROWS_PER_TUMBLING_WINDOW) as f64; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: K, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let build = || { Operator::per_group_summary_build(input.clone(), family.clone(), 0, None, vec![]).unwrap() }; @@ -935,7 +918,8 @@ fn planner_node( id: 1, payload, output_state: planner_types::ir::properties::ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: planner_types::ir::schema::Schema::new( fields .iter() @@ -1032,21 +1016,11 @@ fn filtered_summary_build_keeps_empty_groups() { use planner_types::ir::operator::{GroupKeys, Reduction as Plan}; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ - ExactKind, ExactParams, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, - SummaryUpdate, + ExactKind, ExactParams, SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use planner_types::ir::ASAPOp; let (input, rows) = filtered_input(); - let kll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let count = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let cases = [ ( @@ -1192,7 +1166,8 @@ fn exact_aggregate_node( child: 0, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema(&[(name, dtype, false)])).clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 3b664a12a..5b39368fc 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -49,7 +49,8 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { id, payload, output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -128,72 +129,6 @@ fn blocking_inputs_over_a_raw_data_slot_are_finite() { assert_eq!(opens.load(Ordering::SeqCst), 1); } -/// A KLL kernel accepts `k` only up to the 26 602 `asap_sketchlib` caps it at. -#[test] -fn kll_kernel_rejects_a_k_above_the_library_cap() { - use asap_executor::capability::validate_summary_kernel; - use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, - SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let update = SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: Default::default(), - }; - let kll = |k| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - grouping.clone(), - ) - }; - assert!(validate_summary_kernel(&kll(26_602), &update, &grouping).is_ok()); - assert!(validate_summary_kernel(&kll(26_603), &update, &grouping).is_err()); -} - -/// A plain CountSketch reads the median of its rows, so plan validation and -/// kernel construction both require a positive odd depth. -#[test] -fn count_sketch_kernel_requires_an_odd_depth() { - use asap_executor::{capability::validate_summary_kernel, factory::create_planner_accumulator}; - use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryInputExpr, - SummaryUpdate, - }; - let grouping = GroupingStrategy::default(); - let mut update = SummaryUpdate::column(ColumnRef::SampleValue); - update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let count_sketch = |depth| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width: 64, depth }, - ), - grouping.clone(), - ) - }; - for (depth, valid) in [(5, true), (4, false)] { - let family = count_sketch(depth); - assert_eq!( - validate_summary_kernel(&family, &update, &grouping).is_ok(), - valid - ); - assert_eq!( - create_planner_accumulator(&family, &update, &grouping).is_ok(), - valid - ); - } -} - // Kernel support must not be mistaken for executable native state/evaluation support. #[test] fn summary_capability_levels_are_distinct() { @@ -202,20 +137,9 @@ fn summary_capability_levels_are_distinct() { planner::ir::schema::SketchStatistic, }; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }; + use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; let grouping = GroupingStrategy::default(); - let cms = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 4, - }, - ), - grouping.clone(), - ); + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms, grouping.clone()); let update = SummaryUpdate { item: Some(planner_types::ir::schema::SummaryInputExpr::Column( ColumnRef::Named("host".into()), @@ -239,16 +163,7 @@ fn summary_capability_levels_are_distinct() { } ) .is_err()); - let kll = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 128, - failure_probability: None, - }, - ), - grouping, - ); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, grouping); assert!(validate_native_family(&kll).is_ok()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Quantile { q: 1.5 }).is_err()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Cardinality).is_err()); diff --git a/crates/executor/tests/planspace_series_identity_heap.rs b/crates/executor/tests/planspace_series_identity_heap.rs index 18919c26c..4ae5aff80 100644 --- a/crates/executor/tests/planspace_series_identity_heap.rs +++ b/crates/executor/tests/planspace_series_identity_heap.rs @@ -2,7 +2,7 @@ //! the query's rows do not carry: the planner pipeline plans the logical DAG //! the frontend lowered, and a deployment prices heaps. use crate::common; -use common::{compile_physical_asap_dag, selected_dag}; +use common::selected_dag; use planner_types::ir::OperatorNode; use asap_executor::physical_planner::promql_rows::SERIES_IDENTITY_COLUMN; @@ -44,7 +44,7 @@ fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { } fn carries_identity(root: &Rc) -> bool { - compile_physical_asap_dag(root) + planner_types::test_support::physical_asap_dag_at_query_time(root) .unwrap() .nodes .iter() diff --git a/crates/executor/tests/precompute_candidates.rs b/crates/executor/tests/precompute_candidates.rs deleted file mode 100644 index b540f8110..000000000 --- a/crates/executor/tests/precompute_candidates.rs +++ /dev/null @@ -1,363 +0,0 @@ -//! A grouped Rate→Sum answers the same whether the deployment stores the -//! Rate state or the grouped Sum state. -use crate::common; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::{ - factory::create_planner_accumulator, - physical_planner::{compile, test_support::views_at, CompiledPhysicalDAG}, - runtime::{Limits, RunContext, Scope}, - values::{Batch, Value}, -}; -use common::{compile_physical_asap_dag, computation_candidates, selected_dag}; -use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; -use planner_types::ir::schema::{DataType, *}; -use planner_types::ir::ASAPOp; -use planner_types::workload::*; -use std::{collections::BTreeMap, sync::Arc}; - -/// `sum by(job)(rate(m[1m]))` with its rows carrying the series identity. -fn grouped_rate_root() -> std::rc::Rc { - exact_promql_root("sum by(job)(rate(m[1m]))") -} - -/// `query`, exact, lowered with its rows carrying the series identity. -fn exact_promql_root(query: &str) -> std::rc::Rc { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1000), - )), - }; - let root = asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .remove(0) - .as_operator() - .cloned() - .expect("operator root"); - asap_executor::physical_planner::promql_rows::with_series_identity(&root).unwrap() -} - -fn grouped_rate() -> PhysicalASAPDAG { - compile_physical_asap_dag(&selected_dag(grouped_rate_root(), AccuracyTarget::exact())).unwrap() -} -fn run(plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope) -> Vec { - let dag = plan.instantiate(Arc::new(store)).unwrap(); - block_on(async { - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut output = dag.execute(plan.roots(), context).unwrap().remove(0); - let mut batches = vec![]; - while let Some(batch) = output.next().await { - batches.push((*batch.unwrap()).clone()); - } - batches - }) -} - -/// The root's output when the deployment stores the Rate `state` (bound to -/// `states`) and, with `stored`, also the output of node `stored`, which an -/// ingestion run over `ingestion` computes from the states. -fn answer( - dag: &PhysicalASAPDAG, - state: u64, - states: &Batch, - stored: Option, - ingestion: Scope, - query: Scope, -) -> Vec { - let mut dag = dag.clone(); - let views = views_at( - &mut dag, - &[state].into_iter().chain(stored).collect::>(), - ) - .unwrap(); - let state_store = || { - InMemoryDataStore::default() - .with_materialized_view(state as usize, vec![states.clone()]) - .unwrap() - }; - let root = [dag.roots[0] as u64]; - let Some(stored) = stored else { - return run(&compile(&dag, &views, &root).unwrap(), state_store(), query); - }; - let outputs = run( - &compile(&dag, &views[..1], &[stored]).unwrap(), - state_store(), - ingestion, - ); - assert_eq!(outputs.len(), 1); - let store = InMemoryDataStore::default() - .with_materialized_view(stored as usize, outputs) - .unwrap(); - run(&compile(&dag, &views[1..], &root).unwrap(), store, query) -} - -/// Rate evaluations and grouped Sum can run together at ingestion into a -/// stored Sum state, which answers as the query-time Rate→Sum over the stored -/// Rate state. Moving Sum before Rate would answer differently across -/// counter resets. -#[test] -fn grouped_rate_can_be_materialized_before_or_after_grouped_sum() { - let dag = grouped_rate(); - let state = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - .. - }) - ) - }) - .unwrap(); - let input_schema = Arc::new(state.output_schema.clone()); - let (family, update, grouping) = match &state.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - grouping, - .. - }) => (family, input, grouping), - _ => unreachable!(), - }; - let range_ms = Some((-58_000, 2_000)); - let mut expected_rate_sum = 0.; - let rows = [[100., 0., 100.], [100., 200., 0.]] - .into_iter() - .enumerate() - .map(|(index, values)| { - let mut accumulator = create_planner_accumulator(family, update, grouping).unwrap(); - for (i, value) in values.into_iter().enumerate() { - accumulator.update_single(value, i as i64 * 1000).unwrap(); - } - let state = accumulator.into_accumulator(); - expected_rate_sum += state - .as_any() - .downcast_ref::() - .unwrap() - .evaluation(ExactKind::Rate, range_ms, None) - .unwrap() - .unwrap(); - let summary = Value::Summary { - family: family.clone(), - state: Arc::from(state), - }; - input_schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::ExactAggregate(..) => summary.clone(), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(2000), - FieldDataType::Plain(DataType::Utf8) => Value::Utf8(if field.name == "job" { - "api".into() - } else { - serde_json::to_string(&BTreeMap::from([ - ("__name__", "m".to_string()), - ("job", "api".to_string()), - ("instance", format!("series-{index}")), - ])) - .unwrap() - .into() - }), - _ => panic!("unexpected input field {field:?}"), - }) - .collect() - }) - .collect(); - let batch = Batch::try_new(input_schema.clone(), rows).unwrap(); - let state_id = state.id as u64; - let sum_id = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }) - .unwrap() - .id as u64; - let ingestion = Scope::Ingestion { - window_start_ms: -58_000, - window_end_ms: 2000, - }; - let query = Scope::Query { - evaluation_time_ms: 2000, - }; - let mut results = vec![]; - for stored in [None, Some(sum_id)] { - let output = answer( - &dag, - state_id, - &batch, - stored, - ingestion.clone(), - query.clone(), - ); - assert_eq!(output.len(), 1, "{stored:?}"); - assert_eq!(output[0].to_rows().len(), 1); - assert!(matches!(&output[0].to_rows()[0][0], Value::Utf8(job) if job.as_ref() == "api")); - assert!( - matches!(output[0].to_rows()[0][1], Value::Float64(value) if value == expected_rate_sum) - ); - results.push( - output[0].to_rows()[0] - .iter() - .map(|value| value.key().unwrap()) - .collect::>(), - ); - } - assert_eq!(results[0], results[1]); - let mut wrong_order = create_planner_accumulator(family, update, grouping).unwrap(); - for (i, value) in [200., 200., 100.].into_iter().enumerate() { - wrong_order.update_single(value, i as i64 * 1000).unwrap(); - } - let rate_of_sum = wrong_order - .into_accumulator() - .as_any() - .downcast_ref::() - .unwrap() - .evaluation(ExactKind::Rate, range_ms, None) - .unwrap() - .unwrap(); - assert_ne!( - expected_rate_sum, rate_of_sum, - "counter resets prohibit moving Sum before Rate" - ); -} - -/// Every Pass 1 computation candidate of the grouped Rate answers a numeric -/// value at query time, from the stored Rate state or a stored grouped Sum. -#[test] -fn enumerated_grouped_rate_candidates_execute_numeric_query_outputs() { - let mut executed = 0; - for root in computation_candidates(grouped_rate_root()) { - let dag = compile_physical_asap_dag(&root).unwrap(); - let Some(state) = dag.nodes.iter().find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - .. - }) - ) - }) else { - continue; - }; - let boundary = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - ) - }) - .map(|node| node.id as u64) - .unwrap_or(dag.roots[0] as u64); - let (family, input, grouping) = match &state.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - grouping, - .. - }) => (family, input, grouping), - _ => unreachable!(), - }; - let schema = Arc::new(state.output_schema.clone()); - let rows = ["a", "b"] - .into_iter() - .map(|instance| { - let mut accumulator = create_planner_accumulator(family, input, grouping).unwrap(); - for (timestamp, value) in [(1_000, 1.), (31_000, 31.), (59_000, 59.)] { - accumulator.update_single(value, timestamp).unwrap(); - } - let summary = Value::Summary { - family: family.clone(), - state: Arc::from(accumulator.into_accumulator()), - }; - schema - .fields - .iter() - .map(|field| match &field.dtype { - FieldDataType::ExactAggregate(..) => summary.clone(), - FieldDataType::Plain(DataType::Timestamp) => Value::Timestamp(60_000), - FieldDataType::Plain(DataType::Utf8) - if field.name == "$promql_series_identity" => - { - Value::Utf8( - serde_json::to_string(&BTreeMap::from([ - ("job", "api"), - ("instance", instance), - ])) - .unwrap() - .into(), - ) - } - FieldDataType::Plain(DataType::Utf8) => Value::Utf8("api".into()), - _ => panic!("unexpected input field {field:?}"), - }) - .collect() - }) - .collect(); - let batch = Batch::try_new(schema, rows).unwrap(); - for stored in [None, Some(boundary)] { - let output = answer( - &dag, - state.id as u64, - &batch, - stored, - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 60_000, - }, - Scope::Query { - evaluation_time_ms: 60_000, - }, - ); - assert_eq!(output.len(), 1); - assert_eq!(output[0].to_rows().len(), 1); - assert!(output[0] - .schema() - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - assert!( - output[0].to_rows()[0] - .iter() - .any(|value| matches!(value, Value::Float64(x) if (*x - 2.).abs() < 1e-12)), - "{:?}", - output[0].to_rows() - ); - executed += 1; - } - } - assert!( - executed >= 2, - "must execute both stored and query-time grouped Rate candidates: {executed}" - ); -} diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 81e9b58cb..432b38337 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -30,7 +30,7 @@ fn logical_schema(family: FieldDataType) -> Schema { time_index: None, } } -/// The stored state at node 0 (`MatAtIngestion`): a global `family` +/// The stored state at node 0 (materialized at ingestion time): a global `family` /// `SummaryAgg` over a minute of raw `m` samples (`Scan` node `scan`, /// `TimeRange` node `scan + 1`). fn view_over_samples( @@ -50,14 +50,15 @@ fn view_over_samples( id, payload, output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema, guarantee: None, retention_ms: None, }; [ PhysicalASAPDAGNode { - materialization: planner_types::physical::Materialization::MatAtIngestion( + lifetime: planner_types::physical::Lifetime::AcrossEvaluations( planner_types::physical::Medium::Memory, ), ..node( @@ -137,9 +138,10 @@ fn try_state_dag( group_by: None, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::physical::Materialization::MatAtIngestion( + lifetime: planner_types::physical::Lifetime::AcrossEvaluations( planner_types::physical::Medium::Memory, ), + parameters: None, output_schema: logical_schema(family.clone()), guarantee: None, retention_ms: None, @@ -152,7 +154,8 @@ fn try_state_dag( child: read_id - 1, }), output_state: ExecutionDataState::INGESTION_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), guarantee: None, retention_ms: None, @@ -169,9 +172,10 @@ fn try_state_dag( child: read_id, }), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: planner_types::physical::Materialization::MatAtIngestion( + lifetime: planner_types::physical::Lifetime::AcrossEvaluations( planner_types::physical::Medium::Memory, ), + parameters: None, output_schema: logical_schema(target), guarantee: None, retention_ms: None, @@ -286,13 +290,7 @@ fn explicit_merge_changes_tumbling_window_cardinality() { #[test] fn precompute_rejects_nonfinite_and_out_of_range_dds_updates() { let source = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let target = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - GroupingStrategy::default(), - ); + let target = FieldDataType::Sketch(SketchAlgorithm::DDSketch, GroupingStrategy::default()); let program = state_dag(source.clone(), Some(target), false); for value in [20., -20., 0.] { assert!( diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 8bff8a966..47a13aebd 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -81,7 +81,8 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { rhs: 1, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*schema).clone(), guarantee: None, retention_ms: None, @@ -416,7 +417,8 @@ fn stored_series_evaluations_support_filters_and_sets() { 5 | 6 => ExecutionDataState::QUERY_ROWS, _ => ExecutionDataState::QUERY_ROWS, }, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: match id { 0 | 1 => (*state_schema).clone(), 5 | 6 => sample_schema(), diff --git a/crates/executor/tests/promql_values.rs b/crates/executor/tests/promql_values.rs index 0585c4198..57f56c020 100644 --- a/crates/executor/tests/promql_values.rs +++ b/crates/executor/tests/promql_values.rs @@ -540,6 +540,7 @@ fn exact_state_evaluations_recover_and_finalize_tumbling_windows() { .map(|samples| { let mut state = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), &GroupingStrategy::PerSubpopulationInstance, ) @@ -588,6 +589,7 @@ fn recovered_exact_counter_uses_window_and_omits_insufficient_samples() { .map(|count| { let mut state = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), &GroupingStrategy::PerSubpopulationInstance, ) diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 5cdafc837..7a2ff24bf 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -84,7 +84,8 @@ fn plan( id, payload, output_state: state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (**schema).clone(), guarantee: None, retention_ms: None, @@ -504,7 +505,8 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { id, payload: PhysicalASAPOperatorPayload::NonASAP(payload), output_state: state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -837,7 +839,8 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { id, payload, output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: output_schema.clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index a87ee00bf..3957844d8 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -1,13 +1,13 @@ //! The per-sample `StateColumnUpdater` builds the same states as the //! batch `SummaryAgg` build and as each family's kernel updated row by row, //! for every family a per-group build supports. -use asap_executor::factory::create_planner_accumulator; + use asap_executor::operators::{Operator, RowWeight}; use asap_executor::plan::PhysicalDAG; use asap_executor::planner::ir::scalar::ColumnRef; use asap_executor::planner::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, - HydraParams, Schema, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, + DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, Schema, + SketchAlgorithm, SketchParams, SketchStatistic, }; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; @@ -91,63 +91,14 @@ fn input_batch() -> Batch { .unwrap() } -fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) +fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm, Default::default()) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { FieldDataType::ExactAggregate(kind, params) } -/// Every family a per-group value build supports. -fn value_families() -> Vec { - vec![ - exact(ExactKind::Sum, ExactParams::Sum), - exact(ExactKind::Count, ExactParams::Count), - exact(ExactKind::Min, ExactParams::Min), - exact(ExactKind::Max, ExactParams::Max), - exact(ExactKind::Rate, ExactParams::Rate), - exact(ExactKind::Increase, ExactParams::Increase), - sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - // Above any group's sample count (at most 600): no - // compaction, so the random compaction coin never flips. - k: 1024, - failure_probability: None, - }, - ), - sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 10, - relative_error: None, - }, - ), - sketch( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 32, - relative_error: None, - }, - ), - sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 64, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - distinct_failure_probability: None, - }, - ), - ] -} - /// Every keyed family a per-group keyed build supports. fn keyed_families() -> Vec { vec![ @@ -281,223 +232,6 @@ fn batch_build(build: Operator) -> BTreeMap { /// One test sample: series, timestamp, value, item. type TestSample = (String, i64, f64, String); -/// How a test feeds one sample to an updater. -type FeedSample<'a> = dyn FnMut(&mut StateColumnUpdater, u32, &TestSample) + 'a; - -/// The per-sample build: one updater, groups numbered as series appear. -/// With `sharded`, the first half of the samples go to one updater and the -/// rest to another, merged at the end as a runtime's shards are. -fn per_sample_build( - family: &FieldDataType, - sharded: bool, - mut update: impl FnMut(&mut StateColumnUpdater, u32, &(String, i64, f64, String)), -) -> BTreeMap { - let samples = samples(); - let mut groups: BTreeMap = BTreeMap::new(); - for sample in &samples { - let next = groups.len() as u32; - groups.entry(sample.0.clone()).or_insert(next); - } - let shard = |part: &[(String, i64, f64, String)], update: &mut FeedSample<'_>| { - let mut updater = StateColumnUpdater::new(family).unwrap(); - updater.ensure_groups(groups.len()).unwrap(); - for sample in part { - update(&mut updater, groups[&sample.0], sample); - } - updater - }; - let split = if sharded { - samples.len() / 2 - } else { - samples.len() - }; - let (first, second) = samples.split_at(split); - let mut updater = shard(first, &mut update); - if sharded { - updater.merge(shard(second, &mut update)).unwrap(); - } - assert_eq!(updater.group_count(), groups.len()); - let states = updater.seal(); - groups - .into_iter() - .map(|(series, group)| (series, fingerprint(states.state(group as usize).as_ref()))) - .collect() -} - -/// The pre-columnar reference: each family's kernel, one per series, -/// updated row by row. With `sharded`, two halves merged through the -/// kernel's own merge. -fn kernel_build(family: &FieldDataType, keyed: bool, sharded: bool) -> BTreeMap { - let update = if keyed { - SummaryUpdate { - item: Some( - asap_executor::planner::ir::schema::SummaryInputExpr::Column(ColumnRef::Named( - "item".into(), - )), - ), - weight: asap_executor::planner::ir::schema::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), - } - } else { - SummaryUpdate::column(ColumnRef::SampleValue) - }; - let samples = samples(); - let split = if sharded { - samples.len() / 2 - } else { - samples.len() - }; - let (first, second) = samples.split_at(split); - let half = |part: &[(String, i64, f64, String)]| { - let mut states = BTreeMap::new(); - for (series, ts, value, item) in part { - let state = states.entry(series.clone()).or_insert_with(|| { - if keyed { - let FieldDataType::Sketch(kind, _) = family else { - unreachable!() - }; - let (algorithm, width, depth, capacity) = match kind.params() { - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => ( - asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::Cms, - width, - depth, - heap_size, - ), - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => ( - asap_executor::summary_kernels::weighted_frequency::FrequencyAlgorithm::CountSketch, - width, - depth, - heap_size, - ), - _ => unreachable!(), - }; - Err(Box::new( - WeightedFrequency::new( - algorithm, - *width as usize, - *depth as usize, - *capacity as usize, - ) - .unwrap(), - )) - } else { - Ok(create_planner_accumulator(family, &update, &Default::default()).unwrap()) - } - }); - match state { - Ok(updater) => updater.update_value(&Value::Float64(*value), *ts).unwrap(), - Err(keyed) => keyed - .update(&[Value::Utf8(item.as_str().into())], 1.0) - .unwrap(), - } - } - states - .into_iter() - .map(|(series, state)| { - ( - series, - match state { - Ok(updater) => Arc::::from(updater.into_accumulator()), - Err(keyed) => Arc::new(*keyed) as Arc, - }, - ) - }) - .collect::>() - }; - let (mut merged, second) = (half(first), half(second)); - for (series, state) in second { - let combined = match merged.get(&series) { - Some(existing) => Arc::from(existing.merge_with(state.as_ref()).unwrap()), - None => state, - }; - merged.insert(series, combined); - } - merged - .into_iter() - .map(|(series, state)| (series, fingerprint(state.as_ref()))) - .collect() -} - -// For every value family: the per-sample updater, the batch `SummaryAgg` -// build and the family's own kernel build the same state for every group, -// byte for byte; two updater shards merged equal the kernel's merge of the -// same two halves. -#[test] -fn per_sample_updates_build_the_batch_states_for_every_value_family() { - for family in value_families() { - let batch = batch_build( - Operator::per_group_summary_build(input_schema(), family.clone(), 2, Some(1), vec![0]) - .unwrap(), - ); - let update = - |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { - updater.update(group, sample.2, sample.1).unwrap() - }; - assert_eq!(batch.len(), 7, "{family:?}"); - assert_eq!( - per_sample_build(&family, false, update), - batch, - "{family:?}: updater vs batch build" - ); - assert_eq!( - kernel_build(&family, false, false), - batch, - "{family:?}: kernel vs batch build" - ); - assert_eq!( - per_sample_build(&family, true, update), - kernel_build(&family, false, true), - "{family:?}: merged shards vs kernel merge" - ); - } -} - -// The same for the keyed families (an item and a weight per sample). -#[test] -fn per_sample_updates_build_the_batch_states_for_every_keyed_family() { - for family in keyed_families() { - let batch = batch_build( - Operator::per_group_keyed_summary_build( - input_schema(), - family.clone(), - vec![3], - RowWeight::Constant(NonZeroU32::new(1).unwrap()), - vec![0], - ) - .unwrap(), - ); - let update = - |updater: &mut StateColumnUpdater, group, sample: &(String, i64, f64, String)| { - updater - .update_item(group, &[Value::Utf8(sample.3.as_str().into())], 1.0) - .unwrap() - }; - assert_eq!(batch.len(), 7, "{family:?}"); - assert_eq!( - per_sample_build(&family, false, update), - batch, - "{family:?}: updater vs batch build" - ); - assert_eq!( - kernel_build(&family, true, false), - batch, - "{family:?}: kernel vs batch build" - ); - assert_eq!( - per_sample_build(&family, true, update), - kernel_build(&family, true, true), - "{family:?}: merged shards vs kernel merge" - ); - } -} // A counter refuses a sample that is not after its group's previous one, // and a value outside the kernel's domain, leaving the state unchanged. @@ -594,21 +328,9 @@ fn an_unknown_group_is_bad_input() { /// A HydraCms family: one shared grid for all groups. fn hydra_family() -> FieldDataType { FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 3, - }, - ), + SketchAlgorithm::Cms, GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 64, - depth: 3, - shared_rows: 3, - shared_columns: 16, - }, }, ) } diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 350e66212..946e786fb 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -5,9 +5,7 @@ use asap_executor::arrow::array::{DictionaryArray, UInt32Array}; use asap_executor::arrow::datatypes::UInt32Type; use asap_executor::operators::{Operator, SortKey}; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, BatchColumn}; use asap_executor::{Error, StateColumnUpdater}; @@ -27,16 +25,7 @@ fn tumbling_window() -> (Arc, Batch) { /// One tumbling window of `series` KLL states (k = 200), keyed by series. fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let schema = Arc::new(Schema { fields: vec![ Field { @@ -328,7 +317,8 @@ fn merge_over_a_stored_tumbling_window( payload, output_schema: output_schema.clone(), output_state, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }; diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index 80af9b222..a14d3c713 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -3,9 +3,7 @@ //! adds them, a bounded number of rows at a time. use asap_executor::operators::Operator; use asap_executor::plan::PhysicalDAG; -use asap_executor::planner::ir::schema::{ - DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchKind, SketchParams, -}; +use asap_executor::planner::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::values::{Batch, Value}; use asap_executor::Error; @@ -57,16 +55,7 @@ fn one_large_batch_fails_near_the_memory_limit() { .map(|i| vec![Value::Utf8(format!("s{i}").into()), Value::Float64(1.0)]) .collect(); let batch = Batch::try_new(input.clone(), rows).unwrap(); - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); let mut dag = PhysicalDAG::default(); dag.add( 0, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 392568d84..3e64a23c3 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -52,7 +52,8 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { payload: PhysicalASAPOperatorPayload::NonASAP(payload), output_schema: sample_schema(), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, } @@ -123,7 +124,8 @@ fn summary_projection_survives_recovery() { }), output_schema: (*schema).clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }, @@ -151,7 +153,8 @@ fn summary_projection_survives_recovery() { }), output_schema: output.clone(), output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }, @@ -191,6 +194,7 @@ fn summary_projection_survives_recovery() { assert_eq!(*program.output_schema(1).unwrap(), output); let mut accumulator = create_planner_accumulator( &family, + None, &SummaryUpdate::column(ColumnRef::SampleValue), &GroupingStrategy::PerSubpopulationInstance, ) @@ -259,7 +263,8 @@ fn summary_merge_without_inputs_is_rejected() { }), output_schema: schema, output_state: ExecutionDataState::INGESTION_SUMMARY, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, guarantee: None, retention_ms: None, }], diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index 792a2881b..db76a9d73 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -8,12 +8,11 @@ use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use futures::{executor::block_on, StreamExt}; use planner_types::ir::operator::{Source, TimeShift}; -use planner_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; +use planner_types::ir::physical_export::PhysicalASAPDAG; use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; use planner_types::ir::schema::{DataType, Field, Schema, PROMQL_SERIES_IDENTITY}; use planner_types::ir::{ - apply_materialization_timings, ExprSemantics, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, + ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, }; use std::collections::BTreeMap; use std::sync::Arc; @@ -84,13 +83,7 @@ fn window_over_rows( child, })) .unwrap(); - let timed = apply_materialization_timings( - &range, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() + planner_types::test_support::physical_asap_dag_at_query_time(&range).unwrap() } /// `(instance, ts, value)` samples of `latency_ms` as a store's raw data. diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 5533e6c31..cbc8e9128 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -15,26 +15,14 @@ use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, Schema}; use planner_types::ir::schema::{ - Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, + Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, + SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, }; use planner_types::ir::ASAPOp; use std::sync::Arc; fn family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 64, - sketch_rows: 5, - sketch_cols: 128, - layers: 4, - distinct_failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::UnivMon, GroupingStrategy::default()) } fn context() -> RunContext { @@ -191,7 +179,8 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Default::default(), + lifetime: Default::default(), + parameters: None, output_schema: (*state).clone(), guarantee: None, retention_ms: None, diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index eacadcca7..4e878920a 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -1,255 +1,9 @@ //! Planner output binds directly to the shared runtime over raw counter samples. -use crate::common; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::{ - physical_planner::{ - compile, - promql_rows::{series_row, with_series_identity}, - }, - runtime::{Limits, RunContext, Scope}, - values::{Batch, Value}, -}; -use common::{compile_physical_asap_dag, computation_candidates}; -use futures::{executor::block_on, StreamExt}; -use planner_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use planner_types::ir::properties::AccuracyTarget; use planner_types::ir::schema::*; -use planner_types::ir::ASAPOp; -use std::{collections::BTreeMap, rc::Rc, sync::Arc}; -// The heap candidates of `topk by(job)(2, sum by(service, job)(rate(m[1m])))` -// rank the summed per-series rates of raw counter samples bound at the -// `Scan`, both maintained at ingestion and run at query time. -#[test] -fn planner_weighted_topk_binds_at_either_deployment_phase() { - assert_weighted_binding(SketchAlgorithm::CmsWithHeap); - assert_weighted_binding(SketchAlgorithm::CountSketchWithHeap); -} - -/// Whether `dag` builds a heap sketch of `algorithm`. -fn builds(dag: &PhysicalASAPDAG, algorithm: &SketchAlgorithm) -> bool { - dag.nodes.iter().any(|node| { - matches!(&node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if kind.algorithm() == algorithm) - }) -} - -/// The logical candidate of `root` at ε = 0.1 that builds a heap sketch of -/// `algorithm`. -fn heap_candidate( - root: Rc, - algorithm: &SketchAlgorithm, -) -> Rc { - computation_candidates(root) - .into_iter() - .find(|candidate| builds(&compile_physical_asap_dag(candidate).unwrap(), algorithm)) - .unwrap_or_else(|| panic!("missing {algorithm:?} candidate")) -} -/// `plan` as Stage 3 hands it to the executor when the data declares `m` a -/// counter: its rate weights' open requirement confirmed non-negative. -fn confirmed_as_counter( - plan: Rc, -) -> Rc { - use planner_types::ir::QueryRoot; - use planner_types::workload::{PrometheusMetricType, SourceFacts}; - let data = DataWorkload { - sources: [( - "m".to_string(), - SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..SourceFacts::default() - }, - )] - .into(), - ..DataWorkload::default() - }; - match planner_types::workload::bounds::with_confirmed_weights( - &[QueryRoot::Operator(plan)], - &data, - ) - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => unreachable!("an operator root"), - } -} - -fn assert_weighted_binding(algorithm: SketchAlgorithm) { - let root = lower_promql( - "topk by(job)(2, sum by(service, job)(rate(m[1m])))", - AccuracyTarget::max_error(0.1), - ) - .unwrap(); - // The per-series rate reads each raw series by its full identity. - let root = with_series_identity(&root).unwrap(); - let plan = confirmed_as_counter(heap_candidate(root, &algorithm)); - let dag = compile_physical_asap_dag(&plan).unwrap(); - let scan = dag - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(planner_types::ir::NonASAPOp::Scan { - source, - .. - }) => Some((source.clone(), Arc::new(node.output_schema.clone()))), - _ => None, - }) - .unwrap(); - let (source, samples) = scan; - // One counter series per rate: two samples 32 s apart in `(0, 60 s]`, - // the first an increase above zero, so `rate(m[1m])` extrapolates to the - // whole minute and is exactly `increase / 32 s`. The two `auth` series - // differ in `instance`. - let rows = [ - ("auth", "api", "a", 0.125), - ("auth", "api", "b", 0.25), - ("checkout", "api", "a", 0.3125), - ("search", "api", "a", 0.0625), - ("ingest", "batch", "a", 100.), - ("export", "batch", "a", 80.), - ("cleanup", "batch", "a", 20.), - ] - .into_iter() - .flat_map(|(service, job, instance, rate)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("instance".to_string(), instance.to_string()), - ("job".to_string(), job.to_string()), - ("service".to_string(), service.to_string()), - ]); - let increase = rate * 32.; - [(14_000, increase), (46_000, 2. * increase)] - .map(|(ts, value)| series_row(&samples, &labels, ts, value).unwrap()) - }) - .collect(); - let batch = Batch::try_new(samples.clone(), rows).unwrap(); - // The summaries `MatAtIngestion` run as data arrives; `NotMaterialized` - // (the default) runs everything when the query does. - for (placed, scope) in [ - ( - continuously_maintained_dag(&plan), - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 60_000, - }, - ), - ( - dag.clone(), - Scope::Query { - evaluation_time_ms: 60_000, - }, - ), - ] { - let compiled = compile(&placed, &[], &[dag.roots[0] as u64]).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data(source.clone(), samples.clone(), vec![batch.clone()]) - .unwrap(); - let physical_dag = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new(scope, Limits::default()).unwrap(); - let output = block_on(async { - let mut output = Vec::new(); - let mut stream = physical_dag - .execute(&[dag.roots[0] as u64], context) - .unwrap() - .remove(0); - while let Some(batch) = stream.next().await { - output.extend(batch.unwrap().to_rows().iter().cloned()); - } - output - }); - // Two items per `job`: the outer `by(job)` partitions the ranking. - // Each row is `[job, topk_2, service]`: the top-k's columns, then - // the ranked item's label. - let schema = &dag - .nodes - .iter() - .find(|node| node.id == dag.roots[0]) - .unwrap() - .output_schema; - let column = |name: &str| schema.fields.iter().position(|f| f.name == name).unwrap(); - let (job, value) = (column("job"), column("topk_2")); - assert_eq!(column("service"), 2); - let mut ranked = output - .iter() - .map(|row| match (&row[job], &row[value]) { - (Value::Utf8(job), Value::Float64(score)) => (job.to_string(), *score), - other => panic!("unexpected (job, value) {other:?}"), - }) - .collect::>(); - ranked.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); - let expected = [ - ("api", 0.3125), - ("api", 0.375), - ("batch", 80.), - ("batch", 100.), - ]; - assert_eq!( - ranked, - expected.map(|(job, score)| (job.to_string(), score)), - "{}", - if placed == dag { - "query time" - } else { - "ingestion time" - } - ); - } -} - -use asap_frontend_promql::lower_promql_query_workload; -use planner_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; -pub fn lower_promql( - query: &str, - accuracy: AccuracyTarget, -) -> Result, asap_frontend_promql::PromqlError> { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - Ok(lower_promql_query_workload(&workload)? - .remove(0) - .as_operator() - .cloned() - .expect("operator root")) -} - -// The old untyped heap updater must not silently round a Planner rate update. #[test] fn rate_updates_cannot_enter_integer_heap_factory() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 272, - depth: 5, - heap_size: 100, - }, - ), - Default::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); let input = SummaryUpdate { item: Some(SummaryInputExpr::Column( planner_types::ir::scalar::ColumnRef::Named("service".into()), @@ -261,43 +15,9 @@ fn rate_updates_cannot_enter_integer_heap_factory() { }; assert!(asap_executor::factory::create_planner_accumulator( &family, + None, &input, &Default::default() ) .is_err()); } - -/// `candidate` with every `SummaryAgg` `MatAtIngestion`, the materialization -/// a continuously served query would get: each summary and every input it -/// consumes run at ingestion time, the rest at query time. The candidate pins -/// its finalize boundary to query time, which would put the summaries above -/// it upstream of a query-time node, so the pin is lifted first. -fn continuously_maintained_dag(candidate: &Rc) -> PhysicalASAPDAG { - use planner_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - Operator as IrOperator, OperatorNode, - }; - use planner_types::physical::{Materialization, Medium}; - fn unpinned(node: &Rc) -> Rc { - let mut copy = (**node).clone(); - copy.operator = node.operator.map_children(unpinned); - if matches!( - copy.operator, - IrOperator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) { - copy.timing = None; - } - Rc::new(copy) - } - let candidate = unpinned(candidate); - let mut assignment = MaterializationAssignment::default(); - for node in OperatorNode::reachable(&candidate) { - if matches!(node.operator, IrOperator::ASAP(ASAPOp::SummaryAgg { .. })) { - assignment.set(&node, Materialization::MatAtIngestion(Medium::Memory)); - } - } - let timed = - apply_materialization_timings(&candidate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - planner_types::ir::physical_export::compile_physical_asap_dag(&timed).unwrap() -} diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index b208f7ae9..09a3b2797 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -90,16 +90,10 @@ fn frequency_count_candidates_use_unit_weights() { continue; }; assert!( - !matches!( - kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::CountSketch - ), + !matches!(kind, SketchAlgorithm::Hll | SketchAlgorithm::CountSketch), "{query}: {node:?}" ); - if !matches!( - kind.algorithm(), - SketchAlgorithm::Cms | SketchAlgorithm::UnivMon - ) { + if !matches!(kind, SketchAlgorithm::Cms | SketchAlgorithm::UnivMon) { continue; } let dag = physical_asap_dag(node); @@ -110,7 +104,7 @@ fn frequency_count_candidates_use_unit_weights() { )), "post-ASAP DAG must preserve the count update contract" ); - algorithms.push(*kind.algorithm()); + algorithms.push(*kind); assert!(input.item.is_some(), "frequency keys must be explicit"); assert_eq!( input.weight, @@ -242,7 +236,7 @@ fn cms_count_updates_total_ten_for_zero_positive_and_negative_samples() { .any(|node| { matches!(&node.payload, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Cms) + if kind == &SketchAlgorithm::Cms) }) .then_some(dag) }) diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 19834b75d..28cd4fc27 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -76,24 +76,14 @@ pub fn promql_scalar(node: &ScalarExpr) -> Option { } } -/// Time `root` under the default materialization assignment (every summary +/// Time `root` under the default physical properties assignment (every summary /// at query time) and export the physical DAG — export needs every node /// timed first. #[allow(dead_code)] pub fn physical_asap_dag( root: &Rc, ) -> asap_types::ir::physical_export::PhysicalASAPDAG { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default materialization timings"); - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) - .expect("post-ASAP DAG export") + asap_types::test_support::physical_asap_dag_at_query_time(root).expect("post-ASAP DAG export") } #[allow(dead_code)] @@ -158,12 +148,11 @@ fn local_computation_candidates(root: Rc) -> Result) -> Option> { @@ -17,106 +15,10 @@ fn univmon_estimate(root: &Rc) -> Option> { return false; }; matches!(&summary_input.operator, Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::UnivMon) + if kind == &SketchAlgorithm::UnivMon) }) } -/// The logical candidate of `query` that evaluates a UnivMon state. -fn candidate(query: &str, accuracy: AccuracyTarget) -> Rc { - let root = lower_promql(query, accuracy.clone()).unwrap(); - logical_candidates(root) - .into_iter() - .find_map(|node| univmon_estimate(&node)) - .expect("UnivMon candidate") -} - -#[test] -fn four_evaluations_size_their_value_frequency_states_and_keep_honest_guarantees() { - // Pass 1 sizes UnivMon per readout: the distinct count and L2 get their - // own shapes, while the total and entropy keep the baseline shape and so - // share one state. Sharing across readouts is the summary-capability - // rule's job. - let accuracy = AccuracyTarget::max_error(0.02); - let roots: Vec<_> = [ - ("distinct_over_time(m[5m])", accuracy.clone()), - ("count_over_time(m[5m])", accuracy.clone()), - ("l2_over_time(m[5m])", accuracy.clone()), - ("entropy_over_time(m[5m])", accuracy), - ] - .into_iter() - .enumerate() - .map(|(id, (query, accuracy))| (id, candidate(query, accuracy))) - .collect(); - let roots = share_common_sub_dags(roots); - let state = |index: usize| { - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = - &roots[index].1.operator - else { - panic!() - }; - Rc::clone(summary_input) - }; - assert!(Rc::ptr_eq(&state(1), &state(3)), "total and entropy share"); - for (a, b) in [(0, 1), (0, 2), (1, 2)] { - assert!( - !Rc::ptr_eq(&state(a), &state(b)), - "{a} and {b} are sized apart" - ); - } - for (index, root) in &roots { - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!() - }; - let Operator::ASAP(ASAPOp::SummaryAgg { input, family, .. }) = &summary_input.operator - else { - panic!() - }; - assert!(matches!(input.item, Some(SummaryInputExpr::Column(_)))); - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - let guarantee = root.guarantee.as_ref(); - match index { - // The distinct count's failure probability waits for the - // source's distinct bound, which Stage 1 does not read. - 0 => { - let guarantee = guarantee.expect("a distinct-count guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!( - guarantee.spec.error.evaluate().map(|e| e <= 0.02), - Some(true) - ); - assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(local_guarantee(family, query).is_none()); - } - 1 => { - assert!(matches!( - query, - SketchStatistic::PointCount { value: None, .. } - )); - assert!(guarantee.is_some_and(|g| g.is_exact())); - } - 2 => { - let guarantee = guarantee.expect("an L2 guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(satisfies( - guarantee, - &AccuracyTarget::max_error(0.02).accuracy_metric(AccuracyMetric::Relative) - )); - assert!(guarantee.spec.failure_probability.evaluate().unwrap() <= 0.01); - } - // Entropy has no certified bound, so composition certifies nothing. - _ => { - assert!(guarantee.is_none()); - assert!(local_guarantee(family, query).is_none()); - } - } - physical_asap_dag(root); - } -} - #[test] fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { // An unmeasured heuristic remains inspectable but is never certified or diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index dbf709609..8b2fc3df6 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -130,12 +130,11 @@ async fn stage1_offers_exact_and_summary_l2_candidates() { let root = lower_sql(CAST_COUNT, &catalog(false), target) .await .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ QueryRoot::Operator(root), - ]), - ) - .unwrap(); + ])) + .unwrap(); let l2 = inventory .targets .iter() diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index c6f31079a..167d03fcb 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -5,10 +5,7 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{ - apply_materialization_timings, physical_export::compile_physical_asap_dag, - MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, OperatorNode, ScalarExpr, -}; +use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; fn catalog() -> SqlCatalog { let schema = Schema::new(vec![ @@ -143,47 +140,3 @@ async fn corr_filter_is_a_measure_filter() { filters(&query) ); } - -// The plan the planner pipeline selects retains the complete typed query and -// compiles to a physical ASAP DAG. -#[tokio::test] -async fn corr_survives_exact_plan_compilation() { - let query = lower("SELECT corr(x, y) AS r FROM a").await; - // `a` holds 60,000 rows: the query scans all of them. - let data: asap_types::workload::DataWorkload = serde_json::from_value( - serde_json::json!({"arrival": "at_rest", "sources": {"a": {"rows_held": 60000}}}), - ) - .unwrap(); - let run = asap_planner::run_planning( - vec![asap_types::ir::QueryRoot::Operator(query.clone())], - &[asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy: asap_types::ir::properties::AccuracyTarget::exact(), - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }], - &data, - asap_planner::PlanningModels::builtin(), - asap_planner::PlanSearchSettings::default(), - ) - .unwrap(); - let [asap_types::ir::QueryRoot::Operator(plan)] = run.selected_plan.as_deref().unwrap() else { - panic!("one operator root"); - }; - assert!(plan.guarantee.as_ref().unwrap().is_exact()); - // Pass 1 has no candidate for `corr`: the plan is the query's own - // operator DAG, no ASAP node added. - assert!(!plan.contains_asap(), "expected the exact plan"); - assert_eq!(aggregate(plan).0, aggregate(&query).0); - let timed = apply_materialization_timings( - plan, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap(); -} diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index 055c031b1..c8ee40b02 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -10,22 +10,22 @@ asap-types = { path = "../types", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["reference-pricer"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } +asap-plan-selection = { path = "../plan-selection" } # The fixtures plan through the facade and the developer tools' reference # enumeration. -asap-planner = { path = "../planner", features = ["test-support"] } +asap-planner = { path = "../planner" } asap-devtools = { path = "../devtools" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } asap-devtools = { path = "../devtools" } # The tests plan PromQL and SQL workloads through the facade. -asap-planner = { path = "../planner", features = ["test-support"] } +asap-planner = { path = "../planner" } asap_sketchlib = { workspace = true } serde_json = "1" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 49534547c..9a3e785c8 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -95,54 +95,21 @@ pub mod fixtures { } } -/// Timing and physical ASAP DAG export helpers for planned candidates. +/// Physical ASAP DAG export helpers for planned candidates. pub mod physical_export { - use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - OperatorNode, - }; - use asap_types::physical::{Materialization, Medium}; + use asap_types::ir::physical_export::PhysicalASAPDAG; + use asap_types::ir::OperatorNode; use std::rc::Rc; - /// Time `root` under the default assignment (every sub-DAG - /// `NotMaterialized`, so every summary computed at query time). Returns the timed copy; read `node.timing` on it. - pub fn timed(root: &Rc) -> Rc { - timed_with(root, &MaterializationAssignment::default()) - } - - /// Time `root` with every sub-DAG `MatAtIngestion` where the DAG allows it. - pub fn at_ingestion(root: &Rc) -> Rc { - timed_with( - root, - &MaterializationAssignment::with_default(Materialization::MatAtIngestion( - Medium::Memory, - )), - ) - } - - fn timed_with( - root: &Rc, - assignment: &MaterializationAssignment, - ) -> Rc { - apply_materialization_timings(root, assignment, &mut NodeToTimedNodeCache::new()) - .expect("materialization timing failed") - } - - /// Time `root` (default assignment), then export the physical DAG. + /// Export the physical DAG of `root`. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&timed(root)).expect("physical ASAP DAG export failed") - } - - /// Time `root` with every sub-DAG `MatAtIngestion` where the DAG allows - /// it, then export the physical DAG. - pub fn at_ingestion_physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&at_ingestion(root)).expect("physical ASAP DAG export failed") + asap_types::test_support::physical_asap_dag_at_query_time(root) + .expect("physical ASAP DAG export failed") } } /// Plans and candidates from the planner pipeline (the facade's -/// `run_planning`, with the developer tools' reference enumeration) for one +/// `run_planning`) for one /// query, with the built-in models. pub mod planning { use asap_types::ir::properties::AccuracyTarget; @@ -184,8 +151,7 @@ pub mod planning { } } - /// Every planning stage's output for the single query `root`, with - /// the reference enumeration of its memo. + /// Every planning stage's output for the single query `root`. pub fn run_all_stages( root: Rc, accuracy: AccuracyTarget, @@ -213,7 +179,6 @@ pub mod planning { range_evaluation: None, }], &data, - None, models, ) .expect("Pass 1 accepts the query") @@ -235,9 +200,7 @@ pub mod planning { compose_logical_candidate, generate_per_computation_candidates, }; let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]), ) .unwrap(); let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); @@ -273,9 +236,7 @@ pub mod planning { }; use asap_types::stages::HowToComputeAggregateMeasure; let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]), ) .unwrap(); let choices = local.targets.iter().map(|target| target.ways_to_compute.iter() @@ -334,99 +295,6 @@ pub mod planning { }) .collect() } - /// Pass 1, local tumbling windows for `entries`' recurrences and every Pass 2 rule - /// on the seed grids (no query pruning). Rule tests read a group's - /// candidates by `OriginOfCandidateForMemoGroup`. - pub fn expanded_memo( - roots: Vec, - entries: &[asap_types::workload::QueryWorkloadEntry], - ) -> asap_types::stages::Memo { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_logical_asap_optimization::test_support::{ - add_local_window_composition, try_populate_pass1, - }; - let initial = asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - add_local_window_composition(&mut memo, entries); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() - } - - /// One plan of `memo`'s first `queries` queries (`memo::enumeration::build_logical_asap_dags_from_chosen_candidates`): - /// each query's first root group's DAG, every group `choice` does not - /// name implemented by its first candidate. - pub fn compose_memo_plan( - memo: &asap_types::stages::Memo, - queries: usize, - choice: &std::collections::BTreeMap, - ) -> Vec> { - use asap_logical_asap_optimization::build_logical_asap_dags_from_chosen_candidates; - use asap_types::stages::MemoGroupId; - let choice = choice - .iter() - .map(|(&group, &index)| (MemoGroupId(group as u32), index)) - .collect(); - let roots = build_logical_asap_dags_from_chosen_candidates(memo, &choice) - .expect("the choice composes"); - (0..queries) - .map(|query| { - let (_, root) = roots - .iter() - .find(|(group, _)| memo.query_roots[query].operator_refs().contains(&group)) - .expect("the query's root group"); - Rc::clone(root) - }) - .collect() - } - - /// Stage 2 and plan selection over fixed workload plans, without a timed - /// search: each `(name, roots)` is implemented by Stage 2 (plan ids start - /// with `name`), and all are priced together. - pub fn select_among( - variants: &[(&str, Vec)], - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &DataWorkload, - ) -> ( - Vec, - asap_plan_selection::reference_pricer::WholePlanSelection, - ) { - let models = asap_planner::PlanningModels::builtin(); - let mut physical = Vec::new(); - for (name, roots) in variants { - // Composed nodes lose their guarantees; recompose them as - // `run_explicit_candidates` does. - let roots = asap_accuracy::propagation::recompose_guarantees(roots) - .unwrap_or_else(|error| panic!("{name} composes guarantees: {error}")); - // Composed outside the memo: sized as Pass 1 sizes a candidate. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let roots = - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, data) - .unwrap_or_else(|error| panic!("{name} sizes its summaries: {error}")); - let mut plans = asap_physical_asap_optimization::optimize_physical_asap( - name, - &roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - ) - .unwrap_or_else(|error| panic!("Stage 2 implements {name}: {error}")); - for (index, plan) in plans.iter_mut().enumerate() { - plan.id = format!("{name}-{index}"); - } - physical.extend(plans); - } - let selection = - asap_devtools::memo_pricing::reference_select(&physical, entries, data, models) - .unwrap(); - (physical, selection) - } /// Whether `node` builds a summary. fn is_build(node: &Rc) -> bool { @@ -448,12 +316,13 @@ pub mod planning { keep: impl Fn(&asap_types::stages::CandidateForMemoGroup) -> bool, ) -> Vec<(usize, usize)> { let mut found: Vec<_> = memo - .candidates + .groups .iter() .enumerate() - .filter(|(group, _)| memo.groups[*group].reached_by_queries() == [q]) - .flat_map(|(group, candidates)| { - candidates + .filter(|(_, group)| group.reached_by_queries() == [q]) + .flat_map(|(group, memo_group)| { + memo_group + .candidates .iter() .enumerate() .filter(|(_, c)| keep(c)) @@ -491,105 +360,6 @@ pub mod planning { found } - /// The selected plan's roots among fixed plans of `memo`'s queries, and - /// the selection's lowest prices of `separate` and of the `shared` plans: - /// - `separate{k}`: each query reads its own summary build, its topmost - /// own group's `k`-th Pass 1 candidate that builds one (the last when - /// it has fewer); - /// - `shared{g}`: every query reads the summary state group `g`, its own - /// estimate or a regrouping merge of it, when every query can. - pub fn select_rollup( - memo: &asap_types::stages::Memo, - queries: usize, - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &DataWorkload, - ) -> (Vec>, f64, f64) { - let builds = |node: &Rc| OperatorNode::reachable(node).iter().any(is_build); - let own_builds: Vec> = (0..queries) - .map(|q| { - let found = own_choices(memo, q, |c| { - c.origin == asap_types::stages::OriginOfCandidateForMemoGroup::Pass1 - && builds(&c.root_node) - }); - let top = found - .first() - .unwrap_or_else(|| panic!("Q{} has a summary candidate", q + 1)) - .0; - found - .into_iter() - .filter(|(group, _)| *group == top) - .collect() - }) - .collect(); - let separate_plans = own_builds.iter().map(Vec::len).max().unwrap_or(0); - let mut plans: Vec<(String, Vec>)> = (0..separate_plans) - .map(|k| { - let choice = own_builds - .iter() - .map(|found| found[k.min(found.len() - 1)]) - .collect(); - ( - format!("separate{k}"), - compose_memo_plan(memo, queries, &choice), - ) - }) - .collect(); - for (state, group) in memo.groups.iter().enumerate() { - if !is_build(&group.logical_node) || group.reached_by_queries().len() < queries { - continue; - } - let reading: Option> = (0..queries) - .map(|q| { - own_choices(memo, q, |c| { - c.input_groups.iter().any(|child| child.0 as usize == state) - }) - .into_iter() - .next() - }) - .collect(); - if let Some(choice) = reading { - plans.push(( - format!("shared{state}"), - compose_memo_plan(memo, queries, &choice), - )); - } - } - assert!( - plans.len() > separate_plans, - "a plan in which every query reads one build" - ); - let named: Vec<(&str, Vec)> = plans - .iter() - .map(|(name, roots)| { - ( - name.as_str(), - roots.iter().cloned().map(QueryRoot::Operator).collect(), - ) - }) - .collect(); - let (physical, selection) = select_among(&named, entries, data); - let selected = physical - .iter() - .find(|plan| plan.id == selection.selected) - .expect("the selected plan"); - let roots = plans - .iter() - .find(|(name, _)| *name == selected.from_logical) - .map(|(_, roots)| roots.clone()) - .unwrap(); - let cheapest = |plans: &[(String, Vec>)]| { - plans - .iter() - .map(|(name, _)| cheapest_of(&selection, name)) - .fold(f64::INFINITY, f64::min) - }; - let separate = cheapest(&plans[..separate_plans]); - // Every separate plan rejected would make `shared < separate` hold - // trivially. - assert!(separate.is_finite(), "a valid plan with a build per query"); - (roots, separate, cheapest(&plans[separate_plans..])) - } - /// The summary builds and rollups (regrouping merges) `roots` read, each /// once. pub fn builds_and_rollups(roots: &[Rc]) -> (usize, usize) { @@ -609,18 +379,4 @@ pub mod planning { } (builds, rollups) } - - /// The lowest price per second among `selection`'s plans whose id starts - /// with `name`. - pub fn cheapest_of( - selection: &asap_plan_selection::reference_pricer::WholePlanSelection, - name: &str, - ) -> f64 { - selection - .costs - .iter() - .filter(|(id, _)| id.starts_with(&format!("{name}-"))) - .filter_map(|(_, cost)| cost.cost_per_second) - .fold(f64::INFINITY, f64::min) - } } diff --git a/crates/integration-tests/tests/declared_group_counts.rs b/crates/integration-tests/tests/declared_group_counts.rs deleted file mode 100644 index 6cad286f1..000000000 --- a/crates/integration-tests/tests/declared_group_counts.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! The groups plan selection prices a `by (...)` at, from each label's -//! declared distinct-value count (`SourceFacts::distinct_values_per_label`), -//! over the workload of #840: two per-series quantiles and two grouped -//! quantiles of one gauge with 10,000 series. - -use crate::planner_layering_common; - -use std::collections::BTreeSet; - -use asap_types::workload::PlanningWorkload; -use planner_layering_common::*; - -/// #840's workload (Q1–Q4) with `distinct_values_per_label` declared for -/// `data`. -fn workload(distinct_values_per_label: serde_json::Value) -> PlanningWorkload { - let query = |text: &str| { - serde_json::json!({ - "query": text, "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"max_error": 0.01, "failure_probability": 0.001}}, - "predictability": {"predictable": {"known_at": null}} - }) - }; - serde_json::from_value(serde_json::json!({ - "query_workload": {"language": "promql", "query_batch": null, "repeating_queries": [ - query("quantile_over_time(0.5, data[1m])"), - query("quantile_over_time(0.9, data[5m])"), - query("quantile by (host, job) (0.5, data)"), - query("quantile by (job) (0.5, data)"), - ]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10000, "metric_type": "gauge", - "sample_interval": 15000, - "distinct_values_per_label": distinct_values_per_label}}} - })) - .expect("workload") -} - -/// Isolate the declared-cardinality pricing contract from search order. -fn priced_exact(workload: &PlanningWorkload) -> Run { - let roots = lower_promql(workload); - run_explicit_candidates( - workload, - roots.clone(), - vec![roots], - asap_planner::PlanningModels::builtin(), - ) -} - -/// The group counts every priced plan gives the builds and aggregates read -/// only by `query`, from their cost details (`... into N states`, `... into -/// N groups`). -fn priced_groups(run: &Run, query: usize) -> BTreeSet { - let mut groups = BTreeSet::new(); - for physical in &run.physical { - let Some(cost) = run.selection.costs.get(&physical.id) else { - continue; - }; - for node in &physical.dag().nodes { - if readers(physical.dag(), &physical.query_roots, node.id) != BTreeSet::from([query]) { - continue; - } - let detail = &cost.per_node[&node.id].detail; - let words: Vec<&str> = detail.split([' ', ':', ';']).collect(); - for window in words.windows(3) { - if window[0] == "into" && matches!(window[2], "states" | "groups") { - groups.insert(window[1].parse().expect("a group count")); - } - } - } - } - groups -} - -const Q3_BY_HOST_JOB: usize = 2; -const Q4_BY_JOB: usize = 3; - -/// With `host` and `job` declared, `by (host, job)` is priced at 1,000 × 10 -/// groups, at most the 10,000 series, and `by (job)` at 10. -#[test] -fn a_grouping_is_priced_at_the_product_of_its_labels_counts() { - let run = priced_exact(&workload(serde_json::json!({"host": 1000, "job": 10}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); - assert_eq!(priced_groups(&run, Q4_BY_JOB), BTreeSet::from([10])); -} - -/// A product above `series_count` is capped at it: each group holds at -/// least one series. -#[test] -fn a_grouping_has_at_most_one_group_per_series() { - let run = priced_exact(&workload(serde_json::json!({"host": 5000, "job": 10}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); -} - -/// A grouping with an undeclared label gets no invented count: each of the -/// 10,000 series is in one group, and each of the 10,000 samples the -/// instant selector reads in at most one, so `by (host, job)` with only -/// `host` declared, and both groupings with nothing declared, are priced at -/// 10,000 groups. -#[test] -fn a_grouping_with_an_undeclared_label_is_bound_by_the_series_read() { - let run = priced_exact(&workload(serde_json::json!({"host": 1000}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); - assert_eq!(priced_groups(&run, Q4_BY_JOB), BTreeSet::from([10_000])); - let run = priced_exact(&workload(serde_json::json!({}))); - assert_eq!( - priced_groups(&run, Q3_BY_HOST_JOB), - BTreeSet::from([10_000]) - ); -} diff --git a/crates/integration-tests/tests/deployment_capabilities.rs b/crates/integration-tests/tests/deployment_capabilities.rs deleted file mode 100644 index 482ca9ac5..000000000 --- a/crates/integration-tests/tests/deployment_capabilities.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! The deployment's capabilities (#509's execution-capability input) over -//! the planner-layering examples: the summaries the executor supports -//! restrict Stage 1 (Pass 1, and so Pass 2). - -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_devtools::planning_stages::lower_sql_roots; -use asap_logical_asap_optimization::{DeploymentCapabilities, SupportedSummary}; -use asap_types::ir::flat::FlatDag; -use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::Materialization; -use planner_layering_common::*; - -/// A deployment whose executor supports only `supported`: each family with -/// its readouts. -fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> DeploymentCapabilities { - DeploymentCapabilities { - supported_summaries: Some( - supported - .iter() - .map(|(family, readouts)| SupportedSummary { - family: *family, - readouts: readouts.to_vec(), - }) - .collect(), - ), - ..DeploymentCapabilities::default() - } -} - -/// Every (family, readout) a logical candidate's DAG uses: each sketch -/// build with each statistic estimated from it. -fn summaries_used(dag: &FlatDag) -> Vec<(SketchAlgorithm, StatisticKind)> { - sketch_builds(dag) - .into_iter() - .flat_map(|(build, family, _)| { - estimates_of(dag, build) - .into_iter() - .map(move |(_, statistic)| (family, statistic.kind())) - }) - .collect() -} - -/// Capabilities filter each local computation before physical selection. -/// Compare explicit one-rule applications, not a timed search's archive. -#[test] -fn restricting_summaries_removes_exactly_the_candidates_that_use_them() { - let heap = supporting(&[(SketchAlgorithm::CountSketchWithHeap, &[StatisticKind::TopK])]); - assert_removes_exactly("example 1", lower_promql(&example1()), &heap); - let distinct_and_l2 = supporting(&[ - ( - SketchAlgorithm::UnivMon, - &[StatisticKind::Distinct, StatisticKind::FrequencyL2], - ), - (SketchAlgorithm::Hll, &[StatisticKind::Distinct]), - ]); - let (workload, catalog) = example2(); - assert_removes_exactly( - "example 2", - lower_sql_roots(&workload, &catalog).unwrap(), - &distinct_and_l2, - ); - let kll = supporting(&[(SketchAlgorithm::Kll, &[StatisticKind::Quantile])]); - assert_removes_exactly("pattern B", lower_promql(&pattern_b()), &kll); -} - -fn assert_removes_exactly( - name: &str, - roots: Vec, - deployment: &DeploymentCapabilities, -) { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let enumerate_local = - |inventory: &asap_logical_asap_optimization::test_support::PerComputationCandidates| { - let mut result = Vec::new(); - for (target, options) in inventory.targets.iter().enumerate() { - for choice in 0..options.choice_count() { - let mut choices = vec![0; inventory.targets.len()]; - choices[target] = choice; - if let Ok(roots) = compose_logical_candidate(inventory, &choices) { - result.push(asap_types::ir::flat::flatten(&roots).0); - } - } - } - result - }; - let unrestricted = enumerate_local(&local); - let restricted = enumerate_local( - &local.without_unsupported_summaries(deployment.supported_summaries.as_deref()), - ); - let expected = unrestricted - .iter() - .filter(|dag| { - summaries_used(dag) - .iter() - .all(|(family, readout)| deployment.supports(family, *readout)) - }) - .collect::>(); - assert!( - expected.len() < unrestricted.len(), - "{name}: unsupported candidates removed" - ); - assert!( - restricted.iter().any(|dag| !summaries_used(dag).is_empty()), - "{name}: supported summary retained" - ); - assert_eq!(restricted.iter().collect::>(), expected, "{name}"); -} - -/// Pattern B with a p50 panel beside its p99, both every minute over the -/// same 5-min window. One 1-min tumbling KLL chain serves both panels. -fn pattern_b_with_p50() -> asap_types::workload::PlanningWorkload { - let mut workload = pattern_b(); - let repeating = workload - .query_workload - .repeating_queries - .as_mut() - .expect("pattern B repeats"); - let mut p50 = repeating[0].clone(); - p50.query = asap_types::workload::Query("quantile_over_time(0.5, latency_ms[5m])".into()); - repeating.push(p50); - workload -} - -/// One explicitly shared KLL tumbling-window chain, passed through normal physical rules. -fn shared_tumbling_window_configurations(deployment: &DeploymentCapabilities) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::QueryRoot; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_b_with_p50(); - let original = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&original), - ) - .unwrap(); - let choices = local - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap() - }) - .collect::>(); - let composed = compose_logical_candidate(&local, &choices).unwrap(); - let shared = asap_logical_asap_optimization::test_support::share_common_sub_dags( - composed - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - _ => unreachable!(), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(&shared, &[Some(60_000), Some(60_000)]); - assert_eq!(tumbling_windows.len(), 1, "both estimators share one build"); - let tumbling_window_roots = window_composition_pass2_candidates(&shared, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let settings = CostInputs::default(); - run_explicit_candidates( - &workload, - original, - vec![tumbling_window_roots], - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Physical candidate ids with their cost per second. -type CandidateCosts = Vec<(String, f64)>; - -/// The per-second cost of every physical candidate whose tumbling-window chain both -/// panels read is materialized at query time: shared within one evaluation -/// (`MatAtQuery`), and kept across evaluations -/// (`MatAtQueryKeptAcrossEvaluations`, which Stage 3 prices as such). -fn shared_chain_costs(run: &Run) -> (CandidateCosts, CandidateCosts) { - let (mut within_one, mut kept) = (Vec::new(), Vec::new()); - for p in &run.physical { - for n in &p.dag().nodes { - let shared_chain = matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && readers(p.dag(), &p.query_roots, n.id).len() == 2; - let Some(cost) = run.selection.costs.get(&p.id).filter(|_| shared_chain) else { - continue; - }; - let entry = (p.id.clone(), cost.cost_per_second.expect("cost model 1")); - let kept_detail = cost.per_node[&n.id].detail.contains("across evaluations"); - match n.materialization { - Materialization::MatAtQuery(_) => within_one.push(entry), - Materialization::MatAtQueryKeptAcrossEvaluations(_) if kept_detail => { - kept.push(entry) - } - Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - panic!("{}: a kept chain priced as not kept", p.id) - } - _ => {} - } - } - } - (within_one, kept) -} - -/// With query-time results retained, a chain two panels read is offered -/// both ways: shared within one evaluation (rebuilt from the 5-min window -/// every minute, then reused by both panels) and kept across evaluations -/// (B3: one new tumbling window per evaluation, the 4 older tumbling windows held in memory -/// between evaluations). Holding 1M per-series KLLs between evaluations -/// costs far more than rebuilding them, so the shared option is cheaper -/// and the kept one is not selected. Without query-time retention, the -/// kept option is not offered. -#[test] -fn query_time_results_retained_still_offers_the_chain_shared_within_one_evaluation() { - let keeping = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let run = shared_tumbling_window_configurations(&keeping); - let (within_one, kept) = shared_chain_costs(&run); - assert!(!within_one.is_empty(), "no shared option: kept {kept:?}"); - assert!(!kept.is_empty(), "no kept option: shared {within_one:?}"); - let cheapest = |options: &[(String, f64)]| { - options - .iter() - .map(|(_, cost)| *cost) - .fold(f64::INFINITY, f64::min) - }; - assert!( - cheapest(&within_one) < cheapest(&kept), - "shared {within_one:?}, kept {kept:?}" - ); - assert!( - kept.iter().all(|(id, _)| *id != run.selection.selected), - "{}", - run.selection.selected - ); - assert_selects_cheapest_valid(&run); - - let (_, kept) = shared_chain_costs(&shared_tumbling_window_configurations( - &DeploymentCapabilities::default(), - )); - assert!(kept.is_empty(), "{kept:?}"); -} diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index 63d051298..4ed6ff21a 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -9,17 +9,15 @@ use std::rc::Rc; -use asap_integration_tests::physical_export::{at_ingestion, physical_asap_dag, timed}; +use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::explicit_local_candidates; use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryUpdate}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; // ── fixtures ──────────────────────────────────────────────────────────── @@ -58,17 +56,6 @@ fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child, - }) -} - /// `quantile by (zone, host) (latency)` — the fine-grained inner summary. fn fine_quantile() -> Rc { agg( @@ -110,210 +97,12 @@ fn summary_evaluation(node: &Rc) -> Option<&Rc> { node.children().into_iter().find_map(summary_evaluation) } -fn is_plain(node: &OperatorNode) -> bool { - node.schema - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_))) -} - -fn names(node: &OperatorNode) -> Vec<&str> { - node.schema.fields.iter().map(|f| f.name.as_str()).collect() -} - // ── step 1: pin every already-supported exact-accumulator nesting ─────── -#[test] -fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { - use std::time::Duration; - let cases: Vec<(Rc, ExactKind)> = vec![ - ( - agg( - vec![2], - AggIntent::Sum { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Sum, - ), - ( - agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - metric_scan(&["zone"]), - ), - ExactKind::Count, - ), - ( - agg( - vec![2], - AggIntent::Min { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Min, - ), - ( - agg( - vec![2], - AggIntent::Max { col: None }, - metric_scan(&["zone"]), - ), - ExactKind::Max, - ), - ( - per_entity( - AggIntent::Rate, - node(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: metric_scan(&["zone"]), - }), - ), - ExactKind::Rate, - ), - ( - per_entity( - AggIntent::Increase, - node(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: metric_scan(&["zone"]), - }), - ), - ExactKind::Increase, - ), - ]; - for (inner, kind) in cases { - let outer = agg(vec![], default_quantile(0.9), inner); - // The KLL candidate over the inner exact accumulator. - let root = explicit_local_candidates(Rc::clone(&outer)) - .into_iter() - .map(|candidate| Rc::clone(below_projects(&candidate))) - .find(|candidate| { - matches!(candidate.asap(), Some(ASAPOp::SummaryEstimate { summary_input, .. }) - if matches!(summary_input.asap(), Some(ASAPOp::SummaryAgg { child, .. }) - if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) - }) - .unwrap_or_else(|| panic!("{kind:?}: a KLL over the exact accumulator")); - // Timing is not stored on the plan: time it with the outer summary - // maintained (which also validates every edge) and inspect the copy. - let root = at_ingestion(&root); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!("expected KLL evaluation, got {:?}", root.operator); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &summary_input.operator else { - panic!("expected outer SummaryAgg"); - }; - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: finalized }) = &child.operator - else { - panic!("{kind:?}: missing maintenance finalization"); - }; - assert_eq!( - child.timing, - Some(ExecutionTiming::IngestionTime), - "{kind:?}: finalization runs at maintenance time" - ); - assert!( - matches!( - &finalized.operator, - Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(k, _), .. }) if *k == kind - ), - "{kind:?}: expected the exact accumulator under its finalization, got {:?}", - finalized.operator - ); - } -} - // ── direction 1: outer exact fold over an inner summary evaluation ──────── -/// `max`/`avg` over a quantile composes as an exact fold at query time over -/// the inner quantile's evaluation, with the pre-ASAP target's own schema. -#[test] -fn max_and_avg_over_quantile_compose_at_query_time() { - for intent in [AggIntent::Max { col: None }, AggIntent::Avg { col: None }] { - let root = agg(vec![0], intent.clone(), fine_quantile()); - let composed = composed(&root); - let child = summary_evaluation(&composed).unwrap(); - assert!( - child.guarantee.is_some(), - "child has its KLL rank guarantee" - ); - assert!( - composed.guarantee.is_none(), - "rank error has no definition-backed conversion through max/average" - ); - assert!(is_plain(&composed)); - assert_eq!( - names(&composed), - root.schema - .fields - .iter() - .map(|c| c.name.as_str()) - .collect::>(), - "the composed plan's schema is the pre-ASAP target's own" - ); - assert_eq!( - timed(&composed).timing, - Some(ExecutionTiming::QueryTime), - "{intent:?}: the exact fold runs at query time" - ); - } -} - -/// Grouped fine-to-coarse fold (`by (zone)` over `by (zone, host)`) and the -/// identity fold (`by (zone)` over `by (zone)`) both compose; the operator -/// is the same, only the fold's row multiplicity differs. -#[test] -fn identity_and_genuine_multi_row_folds_both_compose() { - let identity_inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); - for (label, inner) in [ - ("identity", identity_inner), - ("fine-to-coarse", fine_quantile()), - ] { - let root = agg(vec![0], AggIntent::Max { col: None }, inner); - let composed = composed(&root); - assert_eq!( - timed(&composed).timing, - Some(ExecutionTiming::QueryTime), - "{label}" - ); - assert_eq!(names(&composed), vec!["zone", "max"], "{label}"); - } -} - // ── summary construction phase ────────────────────────────────────────── -/// Summary construction can consume query-time values without pretending -/// they are available to an ingestion-time consumer. -#[test] -fn summary_construction_follows_its_value_input_phase() { - let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let post = composed(&root); - let illegal = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: post, - family: FieldDataType::ExactAggregate( - ExactKind::Max, - asap_types::ir::schema::ExactParams::Max, - ), - input: SummaryUpdate::column(asap_types::ir::scalar::ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - }), - Schema::lifted(vec![], None), - ) - .with_guarantee(None), - ); - // Even under an ingestion-time consumer the state is built at query - // time, because a evaluation sits below it. - let state = asap_types::ir::planned_data_state(&illegal, ExecutionTiming::IngestionTime); - assert_eq!(state.timing, ExecutionTiming::QueryTime); - asap_types::ir::validate_at_ingestion(&illegal, state.timing).unwrap(); -} - // ── Physical export: explicit timing ───────────────────────────────────── #[test] diff --git a/crates/integration-tests/tests/filtered_aggregates.rs b/crates/integration-tests/tests/filtered_aggregates.rs index 47dfd4efd..40c52db98 100644 --- a/crates/integration-tests/tests/filtered_aggregates.rs +++ b/crates/integration-tests/tests/filtered_aggregates.rs @@ -1,7 +1,6 @@ //! Filtered aggregates (`FILTER (WHERE …)`, a per-measure filter, or a //! filtered `SummaryAgg`) execute with SQL semantics: a row that fails the //! filter contributes nothing, but its group is kept. -use crate::physical_common; use std::rc::Rc; use std::sync::Arc; @@ -57,7 +56,7 @@ fn rows() -> Vec> { /// Run the logical DAG `root` over [`rows`]; rows come back shown and sorted. fn execute(root: &Rc) -> Vec { - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -259,7 +258,7 @@ async fn hand_filtered_kll_compiles_and_executes() { .await; let roots = hand_filtered( "SELECT g, approx_percentile_cont(x, 0.5) AS q FROM events GROUP BY g", - |f| matches!(f, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::Kll), + |f| matches!(f, FieldDataType::Sketch(kind, _) if kind == &SketchAlgorithm::Kll), &filter, ) .await; @@ -278,7 +277,7 @@ async fn kll_candidates(sql: &str) -> Vec> { .filter(|root| { OperatorNode::reachable(root).iter().any(|n| { matches!(n.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Kll) + if kind == &SketchAlgorithm::Kll) }) }) .collect(); diff --git a/crates/integration-tests/tests/finer_aggregate_reuse.rs b/crates/integration-tests/tests/finer_aggregate_reuse.rs index ed8a1b06e..443181401 100644 --- a/crates/integration-tests/tests/finer_aggregate_reuse.rs +++ b/crates/integration-tests/tests/finer_aggregate_reuse.rs @@ -2,18 +2,10 @@ //! when its groups are coarser, a rollup of values or states (#950). use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; use asap_devtools::planning_stages::run_search_over; -use asap_executor::values::Value; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; + use serde_json::json; use std::path::Path; -use crate::physical_common; - const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; const WINDOW: &str = "ts > now() - INTERVAL '5' MINUTE"; @@ -41,257 +33,6 @@ fn workload(queries: &[String]) -> FuzzWorkload { w } -/// The expanded candidate memo of `queries`, as plan selection receives it. -fn expanded(queries: &[String]) -> Memo { - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(queries)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() -} - -/// The aggregate group of query `query` alone and its rollup candidates. -fn rollups(memo: &Memo, query: usize) -> (usize, Vec<&CandidateForMemoGroup>) { - let owner = memo - .groups - .iter() - .position(|group| { - group.reached_by_queries() == [query] - && matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .expect("the query's own aggregate group"); - let candidates = memo.candidates[owner] - .iter() - .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) - .collect(); - (owner, candidates) -} - -/// Whether `candidate` reads another group's result through a `Filter` -/// (the value path), rather than merging summary states. -fn reads_finer_result(candidate: &CandidateForMemoGroup) -> bool { - !OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. }))) - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Filter { .. }))) -} - -fn row(city: Option<&str>, host: &str, value: Option) -> Vec { - vec![ - Value::Timestamp(0), - city.map_or(Value::Null, |city| Value::Utf8(city.into())), - Value::Utf8(host.into()), - value.map_or(Value::Null, Value::Float64), - ] -} - -/// Row sets covering an empty input, an absent selected group, NULL samples, -/// several hosts per city and other cities. -fn row_sets() -> Vec>> { - vec![ - vec![], - vec![row(Some("LA"), "h1", Some(5.0))], - vec![row(Some("NYC"), "h1", None)], - vec![ - row(Some("NYC"), "h1", Some(10.0)), - row(Some("NYC"), "h1", Some(30.0)), - row(Some("NYC"), "h2", Some(20.0)), - row(Some("SF"), "h1", Some(7.0)), - row(Some("LA"), "h1", Some(90.0)), - row(None, "h1", Some(1.0)), - ], - ] -} - -/// The candidate gives the original query's rows on every row set. -fn assert_same_rows(memo: &Memo, owner: usize, candidate: &CandidateForMemoGroup) { - let sorted = |rows: Vec>| { - let mut rows: Vec<_> = rows.iter().map(|row| format!("{row:?}")).collect(); - rows.sort(); - rows - }; - for rows in row_sets() { - let before = physical_common::execute_raw_rows_at( - &memo.groups[owner].logical_node, - rows.clone(), - 1000, - ); - let after = physical_common::execute_raw_rows_at(&candidate.root_node, rows.clone(), 1000); - assert_eq!(sorted(after), sorted(before), "rows {rows:?}"); - } -} - -/// Example A: both dropped keys are pinned, so the quantile query reads one -/// row of the finer quantile result; no quantile rollup is needed. -#[test] -fn pinned_keys_read_one_finer_quantile_row() { - let memo = expanded(&[ - format!("SELECT city, host, {P90} FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city = 'NYC' AND host = 'h1'"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a read of the finer quantile result"); - assert_same_rows(&memo, owner, read); -} - -/// Example B: a filtered global sum keeps the `city = 'NYC'` rows of the -/// finer sums and sums them. -#[test] -fn filtered_sum_rolls_up_finer_sums() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a rollup of the finer sums"); - assert_same_rows(&memo, owner, read); -} - -/// A grouped query whose filter keeps several groups: filter, then roll the -/// finer `(city, host)` sums and maxima up to `city`. -#[test] -fn filtered_grouped_aggregates_roll_up_finer_values() { - let memo = expanded(&[ - format!( - "SELECT city, host, SUM(value), MAX(value) FROM metrics WHERE {WINDOW} GROUP BY city, host" - ), - format!( - "SELECT city, SUM(value), MAX(value) FROM metrics WHERE {WINDOW} AND city IN ('NYC', 'SF') GROUP BY city" - ), - ]); - let (owner, candidates) = rollups(&memo, 1); - let read = candidates - .into_iter() - .find(|c| reads_finer_result(c)) - .expect("a filtered rollup of the finer values"); - assert_same_rows(&memo, owner, read); -} - -/// Example C: two cities' p90 values cannot be combined, but their KLL -/// states can: keep the NYC and SF states, merge them, estimate. The merged -/// read gives the coarse KLL read's rows, SQL's one NULL row over an empty -/// selection included; the query's own read keeps the rows the `OR` form -/// keeps. -#[test] -fn filtered_kll_states_merge() { - let memo = expanded(&[ - format!("SELECT city, {P90} FROM metrics WHERE {WINDOW} GROUP BY city"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city IN ('NYC', 'SF')"), - ]); - let (owner, candidates) = rollups(&memo, 1); - let merged = candidates - .into_iter() - .find(|candidate| { - OperatorNode::reachable(&candidate.root_node).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { children, group_by: Some(_), .. }) - if matches!(children[0].non_asap(), Some(NonASAPOp::Filter { .. }))) - }) - }) - .expect("a candidate merging the filtered finer states"); - let own_kll = |memo: &Memo, owner: usize| { - memo.candidates[owner] - .iter() - .find(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::Pass1 - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family, .. }) - if format!("{family:?}").contains("Kll")) - }) - }) - .expect("the query's own KLL candidate") - .clone() - }; - let direct = own_kll(&memo, owner); - let or_memo = expanded(&[format!( - "SELECT {P90} FROM metrics WHERE {WINDOW} AND (city = 'NYC' OR city = 'SF')" - )]); - let or_direct = own_kll(&or_memo, rollups(&or_memo, 0).0); - for rows in row_sets() { - let before = physical_common::execute_raw_rows_at(&direct.root_node, rows.clone(), 1000); - let or_form = - physical_common::execute_raw_rows_at(&or_direct.root_node, rows.clone(), 1000); - assert_eq!( - format!("{before:?}"), - format!("{or_form:?}"), - "rows {rows:?}" - ); - let after = physical_common::execute_raw_rows_at(&merged.root_node, rows.clone(), 1000); - assert_eq!(format!("{after:?}"), format!("{before:?}"), "rows {rows:?}"); - assert_eq!(after.len(), 1); - } -} - -/// The same groups in another key order are one computation: neither -/// aggregate reads the other (two such reads would be a cycle). -#[test] -fn permuted_group_by_is_not_a_finer_read() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT host, city, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY host, city"), - ]); - for query in 0..2 { - let (_, candidates) = rollups(&memo, query); - assert!(!candidates.into_iter().any(reads_finer_result)); - } -} - -/// A conjunct on a non-key column cannot be applied to finer sums. -#[test] -fn value_conjunct_has_no_finer_read() { - let memo = expanded(&[ - format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC' AND value > 3"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - -/// A dropped key that is not pinned leaves several quantile values per -/// group, and quantile values do not roll up. -#[test] -fn unpinned_key_has_no_quantile_value_read() { - let memo = expanded(&[ - format!("SELECT city, host, {P90} FROM metrics WHERE {WINDOW} GROUP BY city, host"), - format!("SELECT {P90} FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - -/// The finer result lacks rows the query needs: compensation only removes rows. -#[test] -fn narrower_finer_aggregate_has_no_read() { - let memo = expanded(&[ - format!( - "SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} AND city = 'SF' GROUP BY city, host" - ), - format!("SELECT SUM(value) FROM metrics WHERE {WINDOW} AND city = 'NYC'"), - ]); - let (_, candidates) = rollups(&memo, 1); - assert!(!candidates.into_iter().any(reads_finer_result)); -} - /// The selected plan's cost per second for `queries`. fn cost(queries: &[String]) -> f64 { use asap_cost::CostInputs; @@ -311,7 +52,6 @@ fn cost(queries: &[String]) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs deleted file mode 100644 index 24c083091..000000000 --- a/crates/integration-tests/tests/frequency_execution.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! #509 Example 2's PromQL stand-in executes natively: the distinct count, -//! entropy and L2 norm of one series' sample values over 1 min, through the -//! exact candidate and through the one that builds a UnivMon, at query time. - -use std::collections::BTreeMap; -use std::rc::Rc; - -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::Value; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, -}; - -use crate::physical_common; - -const EVALUATION_MS: i64 = 300_000; -/// `flows_src_ip` samples: source 1 twice, 2 three times, 3 once. -const SAMPLES: [f64; 6] = [1.0, 2.0, 1.0, 2.0, 3.0, 2.0]; - -/// The summary family a candidate builds: `None` for the exact plan. -fn built_family(candidate: &Rc) -> Option { - OperatorNode::reachable(candidate) - .iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(family.clone()), - _ => None, - }) -} - -/// Execute `candidate` at query time over `SAMPLES`, all inside the window. -fn execute(candidate: &Rc) -> Vec { - let timed = apply_materialization_timings( - candidate, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let ranges = dag - .nodes - .iter() - .filter(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::TimeRange { .. }) - ) - }) - .count(); - assert_eq!(ranges, 1, "one range selector"); - // The whole plan runs at query time over the raw samples. - let compiled = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); - let store = physical_common::raw_data_store(&dag, |_, schema| { - SAMPLES - .iter() - .enumerate() - .map(|(i, value)| { - promql_rows::series_row(schema, &labels, 290_000 + i as i64, *value).unwrap() - }) - .collect() - }); - let result = physical_common::execute( - &compiled, - store, - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - ); - result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(value) => Some(value), - Value::Int64(value) => Some(value as f64), - _ => None, - }) - .collect() -} - -// The exact candidate and the UnivMon candidate of each statistic both -// execute and give the statistic of the window's sample values (entropy in -// bits). -#[test] -fn standin_exact_and_univmon_candidates_execute() { - let entropy = -[2.0_f64, 3.0, 1.0] - .iter() - .map(|count| count / 6.0 * (count / 6.0).log2()) - .sum::(); - // A UnivMon sized for a distinct count at ε = 0.01 holds a heap of - // millions of keys on each of its 19 layers, past the default 64 MiB. - for (query, expected, epsilon, delta) in [ - ("distinct_over_time(flows_src_ip[1m])", 3.0, 0.2, 0.1), - ("entropy_over_time(flows_src_ip[1m])", entropy, 0.01, 0.01), - ( - "l2_over_time(flows_src_ip[1m])", - 14.0_f64.sqrt(), - 0.01, - 0.01, - ), - ] { - let accuracy = AccuracyTarget::max_error(epsilon).failure_probability(delta); - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - let mut executed = Vec::new(); - // This is an operator execution test. Enumerate each local rule once, - // not an arbitrary sized configuration visited by a budgeted search. - for candidate in explicit_local_candidates(root) { - let family = built_family(&candidate); - let name = match &family { - None => "exact".to_string(), - Some(FieldDataType::Sketch(kind, _)) - if kind.algorithm() == &SketchAlgorithm::UnivMon => - { - "UnivMon".to_string() - } - _ => continue, - }; - if query.starts_with("distinct_over_time") { - if let Some(FieldDataType::Sketch(kind, _)) = &family { - let asap_types::ir::schema::SketchParams::UnivMon { - distinct_failure_probability, - .. - } = kind.params() - else { - unreachable!("family filtered above") - }; - assert_eq!(*distinct_failure_probability, Some(delta)); - } - } - let values = execute(&candidate); - assert!( - matches!(values.as_slice(), [value] if (value - expected).abs() < 1e-9), - "{query} {name}: {values:?}, expected {expected}" - ); - executed.push(name); - } - executed.sort(); - assert_eq!(executed, ["UnivMon", "exact"], "{query}"); - } -} diff --git a/crates/integration-tests/tests/having_shared_aggregate.rs b/crates/integration-tests/tests/having_shared_aggregate.rs deleted file mode 100644 index 554e63d38..000000000 --- a/crates/integration-tests/tests/having_shared_aggregate.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! A key-only `HAVING` of one query is not pushed below an aggregate it -//! shares with another query (#1089): every candidate of the shared -//! aggregate's memo group returns the group's own groups. -use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; -use asap_executor::values::Value; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, -}; -use asap_types::ir::NonASAPOp; -use asap_types::stages::Memo; -use serde_json::json; -use std::path::Path; - -use crate::physical_common; - -const P90: &str = "percentile_cont(0.9) WITHIN GROUP (ORDER BY value)"; -const WINDOW: &str = "ts > now() - INTERVAL '1' HOUR"; - -/// `queries` over `metrics(ts, city, host, value)`, each repeated with -/// `max_error` 0.01 and `failure_probability` 0.001. -fn workload(queries: &[String]) -> FuzzWorkload { - let mut w = read_seed_file( - &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/sharing_issue_fixtures/902.json"), - ) - .unwrap(); - w.tables = vec![json!({"name":"metrics", "time_index":0, "columns":[ - {"name":"ts","type":"timestamp","nullable":false}, - {"name":"city","type":"utf8","nullable":true}, - {"name":"host","type":"utf8","nullable":true}, - {"name":"value","type":"float64","nullable":true} - ]})]; - let template = w.queries[0].clone(); - w.queries = queries - .iter() - .map(|query| { - let mut entry = template.clone(); - entry.entry.query.0 = query.clone(); - entry - }) - .collect(); - w -} - -/// The Pass 1 memo of `queries` with every Pass 2 sharing rule applied. -fn expanded(queries: &[String]) -> Memo { - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(queries)) - .unwrap() - .into_iter() - .collect(); - let dag = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates(&dag).unwrap(); - let memo = try_populate_pass1(dag, &inventory).unwrap(); - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap() -} - -fn row(city: &str, host: &str, value: f64) -> Vec { - vec![ - Value::Timestamp(0), - Value::Utf8(city.into()), - Value::Utf8(host.into()), - Value::Float64(value), - ] -} - -/// The cities `node` returns over the #1089 rows, sorted. -fn cities(node: &std::rc::Rc) -> Vec { - let rows = vec![ - row("NYC", "h1", 10.0), - row("NYC", "h2", 20.0), - row("LA", "h1", 5.0), - row("LA", "h2", 7.0), - row("SF", "h1", 100.0), - ]; - let mut cities: Vec = physical_common::execute_raw_rows_at(node, rows, 1000) - .into_iter() - .map(|row| format!("{:?}", row[0])) - .collect(); - cities.sort(); - cities -} - -/// #1089: A (`HAVING city = 'NYC'`) listed before B (the same query without -/// it) share one aggregate group. Every candidate of that group returns all -/// three cities, as B needs; A filters above it. -#[test] -fn key_only_having_of_one_reader_stays_above_a_shared_aggregate() { - let grouped = format!("SELECT city, {P90} FROM metrics WHERE {WINDOW} GROUP BY city"); - let memo = expanded(&[format!("{grouped} HAVING city = 'NYC'"), grouped]); - let shared = memo - .groups - .iter() - .position(|group| { - group.reached_by_queries() == [0, 1] - && matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .expect("the aggregate both queries read"); - let expected = cities(&memo.groups[shared].logical_node); - assert_eq!(expected.len(), 3, "{expected:?}"); - assert!(memo.candidates[shared].len() > 1, "summary candidates too"); - for candidate in &memo.candidates[shared] { - assert_eq!( - cities(&candidate.root_node), - expected, - "{:?} candidate {:?}", - candidate.origin, - candidate.root_node - ); - } -} diff --git a/crates/integration-tests/tests/identical_exact_queries.rs b/crates/integration-tests/tests/identical_exact_queries.rs index 55fb53010..9a31850a3 100644 --- a/crates/integration-tests/tests/identical_exact_queries.rs +++ b/crates/integration-tests/tests/identical_exact_queries.rs @@ -35,7 +35,6 @@ fn cost(w: &FuzzWorkload) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); diff --git a/crates/integration-tests/tests/independent_plan_seed.rs b/crates/integration-tests/tests/independent_plan_seed.rs index 7dcee170e..77767994d 100644 --- a/crates/integration-tests/tests/independent_plan_seed.rs +++ b/crates/integration-tests/tests/independent_plan_seed.rs @@ -51,7 +51,6 @@ fn cost(queries: &[String]) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index bc0ad0698..217f15cfe 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -13,18 +13,13 @@ use asap_executor::{ use asap_types::ir::materialized_view::MaterializedView; use asap_types::ir::operator::operator_properties::TimeShift; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; -use asap_types::ir::physical_export::{ - compile_physical_asap_workload_with_node_ids, PhysicalASAPDAG, -}; +use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, NonASAPOp, - OperatorNode, QueryRoot, TimeRangeKind, + DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchStatistic, + SummaryUpdate, }; +use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use futures::{executor::block_on, StreamExt}; use std::{ rc::Rc, @@ -36,17 +31,8 @@ use std::{ const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; -fn family(k: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - Default::default(), - ) +fn family(_k: u32) -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) } /// `t(ts, value)`, time index `ts`. fn table_schema() -> Schema { @@ -120,20 +106,17 @@ fn fixture() -> Fixture { query: SketchStatistic::Quantile { q }, })) }; - let roots = apply_materialization_timings_to_roots( - &[ - QueryRoot::Operator(estimate(0.5)), - QueryRoot::Operator(estimate(0.99)), - ], - &MaterializationAssignment::default(), - ) - .unwrap(); - let compiled = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); + let roots = [ + QueryRoot::Operator(estimate(0.5)), + QueryRoot::Operator(estimate(0.99)), + ]; + let (dag, operator_node_of_id) = + asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots).unwrap(); let mut tumbling_window_ids = vec![0; 5]; let (mut merge, mut p50, mut p99) = (0, 0, 0); - for node in &compiled.dag.nodes { + for node in &dag.nodes { let id = node.id as u64; - match &compiled.node_ids.operator_node(node.id).unwrap().operator { + match &operator_node_of_id[node.id].operator { asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => { let shift = match child.children()[0].non_asap() { Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms, @@ -152,7 +135,7 @@ fn fixture() -> Fixture { _ => {} } } - let mut dag = compiled.dag; + let mut dag = dag; let views = views_at(&mut dag, &tumbling_window_ids).unwrap(); Fixture { dag, diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index bf81ee35d..08e7d6f74 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -8,52 +8,31 @@ mod planner_layering_common; mod aggregate; mod binary_op; mod crate_graph; -mod declared_group_counts; -mod deployment_capabilities; mod exact_composition; mod filtered_aggregates; mod finer_aggregate_reuse; -mod frequency_execution; mod frontend_timestamps; -mod having_shared_aggregate; mod identical_exact_queries; mod independent_plan_seed; mod joint_search_equivalence; mod kll_tumbling_window_execution; -mod materialization_e2e; mod materialized_views; -mod memo_enumeration; mod nested; mod operator_design_examples; mod operator_sharing; mod per_computation_candidates_sql; mod per_series_exact_execution; -mod planner_layering_example1; -mod planner_layering_example2; -mod planner_layering_example3; -mod planner_layering_example4; -mod precompute_raw_samples; mod production_rollup_workloads; mod promql_numeric_regressions; mod promql_to_post_asap; -mod range_evaluation; -mod reuse_rules; mod rollup_controlled; -mod runtime_handoff; mod scan; mod schema; mod sharing_issue_regressions; mod sketch_rollup; -mod sketch_rollup_planning; mod sql_cardinality; mod sql_frequency; -mod sql_group_bound; -mod sql_to_physical; mod sql_to_post_asap; -mod sql_trailing_window; -mod sql_two_percentiles; mod stage1_logical_asap_optimization; -mod stage2_physical_asap_optimization_sql; mod summary_coverage_e2e; mod time_range; -mod tumbling_window_merge; diff --git a/crates/integration-tests/tests/materialization_e2e.rs b/crates/integration-tests/tests/materialization_e2e.rs deleted file mode 100644 index 1751d3e95..000000000 --- a/crates/integration-tests/tests/materialization_e2e.rs +++ /dev/null @@ -1,356 +0,0 @@ -//! Materialization decides placement, not answers: the same selected DAG -//! timed with its states `NotMaterialized` (everything at query time) and -//! `MatAtIngestion` (materialized views refreshed at ingestion time) -//! executes to the same result. - -use std::collections::{BTreeMap, HashMap}; -use std::rc::Rc; - -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::{Batch, SchemaRef, Value}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{ - explicit_local_candidates, explicit_summary_candidate, selected_dag, -}; -use asap_physical_asap_optimization::test_support::split_shared_by_timing; -use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, PhysicalASAPDAG, PhysicalASAPOperatorPayload, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, -}; -use asap_types::physical::{Materialization, Medium}; - -use crate::physical_common; - -const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - -/// The logical DAG the planner pipeline selects for `query`, optionally with -/// a PromQL series identity (per-entity state needs one). -fn selected(query: &str, series_identity: bool) -> Rc { - selected_with(query, series_identity, AccuracyTarget::max_error(0.01)) -} - -/// [`selected`] for a quantile query, whose target names its metric (a -/// rank error). -fn selected_quantile(query: &str, series_identity: bool) -> Rc { - selected_with( - query, - series_identity, - AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), - ) -} - -fn selected_with(query: &str, series_identity: bool, accuracy: AccuracyTarget) -> Rc { - let mut root = lower_promql(query, accuracy.clone()).unwrap(); - if series_identity { - root = promql_rows::with_series_identity(&root).unwrap(); - } - selected_dag(root, accuracy) -} - -/// The first logical candidate for `query` that builds a sketch: plan -/// selection picks the exact plan while every summary runs at query time. -fn sketch_candidate(query: &str) -> Rc { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql(query, accuracy.clone()).unwrap(); - explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - OperatorNode::reachable(candidate).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .expect("a sketch candidate") -} - -/// The two placements under test: nothing stored, and every state stored at -/// ingestion time where the DAG allows it. -fn placements() -> [MaterializationAssignment; 2] { - [ - MaterializationAssignment::default(), - MaterializationAssignment::with_default(AT_INGESTION), - ] -} - -/// `root` timed under `assignment`, as Stage 2 times a plan: a node read at -/// two timings (Pass 1 composes identical inputs as one memo group's node, -/// e.g. the scan of two quantiles of one metric) is split first, one copy -/// per timing. -fn timed_dag(root: &Rc, assignment: &MaterializationAssignment) -> PhysicalASAPDAG { - let (split, assignment) = split_shared_by_timing(std::slice::from_ref(root), assignment); - let timed = - apply_materialization_timings(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() -} - -fn states(dag: &PhysicalASAPDAG) -> Vec { - dag.nodes - .iter() - .filter(|node| node.materialization == AT_INGESTION) - .map(|node| node.id as u64) - .collect() -} - -fn floats(result: &[Vec]) -> Vec { - result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(value) => Some(value), - _ => None, - }) - .collect() -} - -/// Execute a timed DAG over one raw-source batch per `Scan` -/// (`batch(schema, metric)`): each refresh group of materialized views -/// refreshed in one ingestion run over `ingestion`, then the query at `evaluation_time_ms` -/// reading the views' states. Returns the root's values and the views. -fn execute( - dag: &PhysicalASAPDAG, - batch: &dyn Fn(&SchemaRef, &str) -> Batch, - ingestion: Scope, - evaluation_time_ms: i64, -) -> (Vec, Vec) { - let mut dag = dag.clone(); - let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let raw = |dag: &PhysicalASAPDAG| { - physical_common::raw_data_store(dag, |source, schema| { - let Source::TimeSeries { metric } = source else { - panic!("a time series source") - }; - batch(schema, metric).to_rows() - }) - }; - // Each refresh group's ingestion run refreshes its views from the raw - // data, compiled once with all of them as roots. - let mut store = raw(&dag); - for group in dag.ingestion_refresh_groups().unwrap() { - let roots: Vec = group.views.iter().map(|view| view.node_id as u64).collect(); - let refresh = compile(&dag, &[], &roots).unwrap(); - let states = physical_common::execute(&refresh, raw(&dag), ingestion.clone()); - for (view, states) in group.views.iter().zip(states) { - store = store.with_materialized_view(view.node_id, states).unwrap(); - } - } - let query = compile(&dag, &views, &[dag.roots[0] as u64]).unwrap(); - let result = physical_common::execute(&query, store, Scope::Query { evaluation_time_ms }); - ( - floats(&result), - views.iter().map(|view| view.node_id as u64).collect(), - ) -} - -/// `(metric, job, seconds, value)` samples as one batch per metric. -fn job_samples<'a>( - samples: &'a [(&'a str, &'a str, i64, f64)], -) -> impl Fn(&SchemaRef, &str) -> Batch + 'a { - move |schema, name| { - let rows = samples - .iter() - .filter(|sample| sample.0 == name) - .map(|(metric, job, seconds, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("job".to_string(), job.to_string()), - ]); - promql_rows::series_row(schema, &labels, seconds * 1000, *value).unwrap() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() - } -} - -/// Values 1..=100 at 300 s, one row each, in every Float64 column of -/// `schema`. -fn values_1_to_100(schema: &SchemaRef, _metric: &str) -> Batch { - let rows = (1..=100) - .map(|value| { - schema - .fields - .iter() - .map(|field| match field.dtype { - asap_types::ir::schema::FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - ) => Value::Float64(f64::from(value)), - asap_types::ir::schema::FieldDataType::Plain( - asap_types::ir::schema::DataType::Timestamp, - ) => Value::Timestamp(300_000), - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - Batch::try_new(schema.clone(), rows).unwrap() -} - -/// `MatAtIngestion` puts the quantile's state in a materialized view; -/// `NotMaterialized` has no view and reads the raw source at -/// query time; both answer alike. -#[test] -fn materialization_decides_precompute_contents() { - let root = sketch_candidate("quantile(0.99, latency)"); - let mut answers = Vec::new(); - for assignment in placements() { - let dag = timed_dag(&root, &assignment); - let ingestion = Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - }; - let (values, frontier) = execute(&dag, &values_1_to_100, ingestion, 300_000); - assert_eq!(frontier, states(&dag)); - answers.push(values); - } - assert_eq!(answers[1].len(), 1); - assert!(!states(&timed_dag(&root, &placements()[1])).is_empty()); - assert_eq!(answers[0], answers[1]); -} - -/// Mixed placement in one DAG: one summary `MatAtIngestion` (a materialized view) -/// and the other `NotMaterialized` (built from the raw source at query -/// time) answer like the all-query-time placement. -#[test] -fn mixed_placement_answers_like_query_time() { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "quantile(0.99, latency) + quantile(0.5, latency)", - accuracy.clone(), - ) - .unwrap(); - let sketches = |candidate: &Rc| -> Vec> { - OperatorNode::reachable(candidate) - .into_iter() - .filter(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(..), - .. - }) - ) - }) - .collect() - }; - let root = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::Kll); - assert_eq!( - sketches(&root).len(), - 2, - "two explicitly selected KLL builds" - ); - let mut mixed = MaterializationAssignment::default(); - mixed.set(&sketches(&root)[0], AT_INGESTION); - let mut answers = Vec::new(); - for assignment in [MaterializationAssignment::default(), mixed] { - let dag = timed_dag(&root, &assignment); - let ingestion = Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 300_000, - }; - let (values, frontier) = execute(&dag, &values_1_to_100, ingestion, 300_000); - assert_eq!(frontier, states(&dag)); - answers.push((values, frontier.len())); - } - // Only the assigned sketch is stored. - assert_eq!(answers[0].1, 0); - assert_eq!(answers[1].1, 1); - assert_eq!(answers[1].0.len(), 1); - assert_eq!(answers[0].0, answers[1].0); -} - -/// Whether the DAG has an ingestion-time `BinaryOp`. -fn ingestion_binary(dag: &PhysicalASAPDAG) -> bool { - dag.nodes.iter().any(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::BinaryOp { .. }) - ) && node.output_state.timing == ExecutionTiming::IngestionTime - }) -} - -const INGESTION_WINDOW: Scope = Scope::Ingestion { - window_start_ms: 240_000, - window_end_ms: 300_000, -}; - -/// Prometheus drops series without a match: arithmetic over different -/// selectors keeps only label sets present on both sides (none when -/// disjoint), at either placement, and never becomes aligned maintenance. -#[test] -fn arithmetic_over_different_selectors_matches_prometheus() { - // The selected root reads the outer `sum`'s exact state through its - // finalize. - let root = selected("sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))", true); - assert!(matches!( - root.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )); - let disjoint = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("n", "b", 250, 5.0), - ]; - // Only job a is on both sides: m_a + n_a = (1 + 2) + 7; m{job="b"} is dropped. - let overlapping = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ("n", "a", 250, 7.0), - ]; - for assignment in placements() { - let dag = timed_dag(&root, &assignment); - assert!(!ingestion_binary(&dag)); - let (values, _) = execute(&dag, &job_samples(&disjoint), INGESTION_WINDOW, 300_000); - assert!(values.is_empty(), "{values:?}"); - let (values, _) = execute(&dag, &job_samples(&overlapping), INGESTION_WINDOW, 300_000); - assert_eq!(values, [10.0]); - } - let root = selected_quantile( - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(n[1m]))", - true, - ); - assert!(!ingestion_binary(&timed_dag(&root, &placements()[1]))); -} - -/// Arithmetic over one selector is aligned maintenance when stored at -/// ingestion time, and adds each series' two evaluations before the -/// quantile at either placement. -#[test] -#[ignore = "the two identical selectors stay separate until Pass 2 merges identical sub-DAGs (#580)"] -fn arithmetic_over_one_selector_executes() { - let root = selected_quantile( - "quantile(0.9, sum_over_time(m[1m]) + sum_over_time(m[1m]))", - true, - ); - let samples = [ - ("m", "a", 250, 1.0), - ("m", "a", 290, 2.0), - ("m", "b", 250, 5.0), - ]; - let [not_materialized, at_ingestion] = placements().map(|a| timed_dag(&root, &a)); - assert!(!ingestion_binary(¬_materialized)); - assert!( - ingestion_binary(&at_ingestion), - "one selector shares its key set" - ); - for dag in [not_materialized, at_ingestion] { - let (values, _) = execute(&dag, &job_samples(&samples), INGESTION_WINDOW, 300_000); - // job a: 3 + 3 = 6; job b: 5 + 5 = 10 (mispairing a with b gives 8 and 8). - // KLL at epsilon 0.01 returns an input value within 0.01 of rank 0.9; - // of two values only the larger is. - assert_eq!(values, [10.0]); - } -} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json index 515db4302..d2e26ea78 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json @@ -1 +1 @@ -{"dag":{"schema_version":10,"dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]}},"queries":[{"query":"sum by (job) (sum_over_time(m[1m]))","root_node_id":5},{"query":"sum by (job, instance) (sum_over_time(m[1m]))","root_node_id":6}],"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}],"query_time_raw_windows":[],"selection":null} +{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "family": {"ExactAggregate": ["Sum", "Sum"]}, "filter": null, "grouping": "PerSubpopulationInstance", "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "family": {"ExactAggregate": ["Sum", "Sum"]}, "filter": null, "grouping": "PerSubpopulationInstance", "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} diff --git a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json index 3a73c4fcf..f822d6b70 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/stored_outputs_on_main.json @@ -1,10 +1,10 @@ [ -{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, -{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, -{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"materialization":{"MatAtQuery":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"materialization":{"MatAtQuery":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, -{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, -{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install a_kept_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"materialization":{"MatAtQueryKeptAcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, -{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, -{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"materialization":{"MatAtIngestion":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"materialization":"NotMaterialized","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} +{"name":"repeating p99-60m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":35,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":37,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":36,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":38,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":37,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":39,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":40,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":39,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":41,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":40,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":42,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":43,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":42,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":44,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":43,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":45,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":46,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":45,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":47,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":46,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":48,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":49,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":48,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":50,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":49,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":51,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":52,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":51,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":53,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":52,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":38,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":41,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":44,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":47,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":50,"role":"Input","window":"NotApplicable"},{"consumer":54,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":53,"role":"Input","window":"NotApplicable"},{"consumer":55,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":54,"role":"Input","window":"NotApplicable"},{"consumer":56,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":55,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":200000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":400000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":800000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1000000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1200000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1400000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1600000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":1800000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2000000}}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":30,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":32,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2200000}}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":33,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":35,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":34,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2400000}}}}},{"guarantee":null,"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":36,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":38,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":37,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":39,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2600000}}}}},{"guarantee":null,"id":40,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":39,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":41,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":40,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":42,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":2800000}}}}},{"guarantee":null,"id":43,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":42,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":44,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":43,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":45,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3000000}}}}},{"guarantee":null,"id":46,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":45,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":47,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":46,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":48,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3200000}}}}},{"guarantee":null,"id":49,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":48,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":50,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":49,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":51,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3400000}}}}},{"guarantee":null,"id":52,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":51,"kind":"range","range":{"nanos":0,"secs":200}}}}},{"guarantee":null,"id":53,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":52,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":54,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29,32,35,38,41,44,47,50,53]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":55,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":54}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":56,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":55,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[56],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-200000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":200000}],"stored_outputs":[]},"node_id":2,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-400000},{"included":-200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":200000,"scan_node_id":0,"time_range_node_id":4,"width_ms":200000}],"stored_outputs":[]},"node_id":5,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":400000,"scan_node_id":0,"time_range_node_id":7,"width_ms":200000}],"stored_outputs":[]},"node_id":8,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-800000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":10,"width_ms":200000}],"stored_outputs":[]},"node_id":11,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1000000},{"included":-800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":800000,"scan_node_id":0,"time_range_node_id":13,"width_ms":200000}],"stored_outputs":[]},"node_id":14,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1200000},{"included":-1000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1000000,"scan_node_id":0,"time_range_node_id":16,"width_ms":200000}],"stored_outputs":[]},"node_id":17,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1400000},{"included":-1200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1200000,"scan_node_id":0,"time_range_node_id":19,"width_ms":200000}],"stored_outputs":[]},"node_id":20,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1600000},{"included":-1400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1400000,"scan_node_id":0,"time_range_node_id":22,"width_ms":200000}],"stored_outputs":[]},"node_id":23,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-1800000},{"included":-1600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1600000,"scan_node_id":0,"time_range_node_id":25,"width_ms":200000}],"stored_outputs":[]},"node_id":26,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2000000},{"included":-1800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":1800000,"scan_node_id":0,"time_range_node_id":28,"width_ms":200000}],"stored_outputs":[]},"node_id":29,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2200000},{"included":-2000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2000000,"scan_node_id":0,"time_range_node_id":31,"width_ms":200000}],"stored_outputs":[]},"node_id":32,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2400000},{"included":-2200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2200000,"scan_node_id":0,"time_range_node_id":34,"width_ms":200000}],"stored_outputs":[]},"node_id":35,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2600000},{"included":-2400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2400000,"scan_node_id":0,"time_range_node_id":37,"width_ms":200000}],"stored_outputs":[]},"node_id":38,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-2800000},{"included":-2600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2600000,"scan_node_id":0,"time_range_node_id":40,"width_ms":200000}],"stored_outputs":[]},"node_id":41,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3000000},{"included":-2800000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":2800000,"scan_node_id":0,"time_range_node_id":43,"width_ms":200000}],"stored_outputs":[]},"node_id":44,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3200000},{"included":-3000000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3000000,"scan_node_id":0,"time_range_node_id":46,"width_ms":200000}],"stored_outputs":[]},"node_id":47,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3400000},{"included":-3200000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3200000,"scan_node_id":0,"time_range_node_id":49,"width_ms":200000}],"stored_outputs":[]},"node_id":50,"retention_ms":3600000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-3600000},{"included":-3400000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3400000,"scan_node_id":0,"time_range_node_id":52,"width_ms":200000}],"stored_outputs":[]},"node_id":53,"retention_ms":3600000}]}, +{"name":"repeating offset-chain/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":17,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":18,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":2,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":1,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":3,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":2,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":4,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":6,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":5,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":7,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":11,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":10,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":12,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":11,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":14,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":13,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":14,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":600000}}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":16,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":18,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":17,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":660000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":720000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":780000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":840000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Sketch":[{"algorithm":"Kll","params":{"Kll":{"failure_probability":1e-05,"k":469}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[3,6,9,12,15,18,21,24,27,30]}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"quantile","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"SummaryEstimate":{"query":{"Quantile":{"q":0.99}},"summary_input":31}}}},{"guarantee":{"provenance":[{"algorithm":"Kll","contract":"apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b","kind":"sketch_evaluation","params":{"Kll":{"failure_probability":1e-05,"k":469}},"query":"Quantile { q: 0.99 }"}],"spec":{"accuracy_metric":"rank","accuracy_scope":"per_value","error":{"op":"constant","value":0.009954464416322084},"failure_probability":{"op":"constant","value":1e-05}}},"id":33,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":32,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":2}}],"qualifier":null}}}}],"roots":[33],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":2,"width_ms":60000}],"stored_outputs":[]},"node_id":3,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":5,"width_ms":60000}],"stored_outputs":[]},"node_id":6,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":8,"width_ms":60000}],"stored_outputs":[]},"node_id":9,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":11,"width_ms":60000}],"stored_outputs":[]},"node_id":12,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":14,"width_ms":60000}],"stored_outputs":[]},"node_id":15,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-660000},{"included":-600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":600000,"scan_node_id":0,"time_range_node_id":17,"width_ms":60000}],"stored_outputs":[]},"node_id":18,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-720000},{"included":-660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":660000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-780000},{"included":-720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":720000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-840000},{"included":-780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":780000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":900000},{"built_at":"ingestion_run","coverage":{"definition_id":"9f0c9036ac7789a53d65f287a16aad22","selection":[{"columns":[],"relative_time":[{"excluded":-900000},{"included":-840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":840000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":900000}]}, +{"name":"repeating rollup/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":16,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":15,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":16,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":18,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":18,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":21,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":24,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":27,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":20,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":23,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":26,"role":"Input","window":"NotApplicable"},{"consumer":30,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":29,"role":"Input","window":"NotApplicable"},{"consumer":31,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":31,"role":"Input","window":"NotApplicable"},{"consumer":33,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":32,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":34,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":300000}}}}},{"guarantee":null,"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":15,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":17,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":360000}}}}},{"guarantee":null,"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":18,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":20,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":19,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":21,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":420000}}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":21,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":23,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":22,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":24,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":480000}}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":24,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":26,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":25,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":27,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":540000}}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":27,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":29,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":28,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":30,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14,17,20,23,26,29]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":30}}}},{"guarantee":null,"id":32,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":31,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":33,"lifetime":{"OneEvaluation":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":32}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":33,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}}],"roots":[33,35],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-360000},{"included":-300000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":300000,"scan_node_id":0,"time_range_node_id":16,"width_ms":60000}],"stored_outputs":[]},"node_id":17,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-420000},{"included":-360000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":360000,"scan_node_id":0,"time_range_node_id":19,"width_ms":60000}],"stored_outputs":[]},"node_id":20,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-480000},{"included":-420000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":420000,"scan_node_id":0,"time_range_node_id":22,"width_ms":60000}],"stored_outputs":[]},"node_id":23,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-540000},{"included":-480000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":480000,"scan_node_id":0,"time_range_node_id":25,"width_ms":60000}],"stored_outputs":[]},"node_id":26,"retention_ms":600000},{"built_at":"ingestion_run","coverage":{"definition_id":"89d70c922a6f3de8ed2220ac945ac927","selection":[{"columns":[],"relative_time":[{"excluded":-600000},{"included":-540000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":540000,"scan_node_id":0,"time_range_node_id":28,"width_ms":60000}],"stored_outputs":[]},"node_id":29,"retention_ms":600000}]}, +{"name":"repeating rate-5m/no_raw","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"},{"consumer":17,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":16,"role":"Input","window":"NotApplicable"},{"consumer":18,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":17,"role":"Input","window":"NotApplicable"},{"consumer":19,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":20,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":19,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":21,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":20,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":22,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":23,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":22,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":24,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":23,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":25,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":26,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":25,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":27,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":26,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":28,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":29,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":28,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":30,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":29,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":31,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":32,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":31,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":33,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":32,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":21,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":24,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":27,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":30,"role":"Input","window":"NotApplicable"},{"consumer":34,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":33,"role":"Input","window":"NotApplicable"},{"consumer":35,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":34,"role":"Input","window":"NotApplicable"},{"consumer":36,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":35,"role":"Input","window":"NotApplicable"},{"consumer":37,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"producer":36,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"latency_ms"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}},{"guarantee":null,"id":17,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":16,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":18,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":17}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":19,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3600000}}}}},{"guarantee":null,"id":20,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":19,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":21,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":20,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":22,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3660000}}}}},{"guarantee":null,"id":23,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":22,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":24,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":23,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":25,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3720000}}}}},{"guarantee":null,"id":26,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":25,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":27,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":26,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":28,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3780000}}}}},{"guarantee":null,"id":29,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":28,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":30,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":29,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":31,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":3840000}}}}},{"guarantee":null,"id":32,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":31,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":33,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":32,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":34,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[21,24,27,30,33]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":35,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":34}}}},{"guarantee":null,"id":36,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":35,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":37,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":36}}}}],"roots":[18,37],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":300000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3660000},{"included":-3600000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3600000,"scan_node_id":0,"time_range_node_id":20,"width_ms":60000}],"stored_outputs":[]},"node_id":21,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3720000},{"included":-3660000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3660000,"scan_node_id":0,"time_range_node_id":23,"width_ms":60000}],"stored_outputs":[]},"node_id":24,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3780000},{"included":-3720000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3720000,"scan_node_id":0,"time_range_node_id":26,"width_ms":60000}],"stored_outputs":[]},"node_id":27,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3840000},{"included":-3780000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3780000,"scan_node_id":0,"time_range_node_id":29,"width_ms":60000}],"stored_outputs":[]},"node_id":30,"retention_ms":3900000},{"built_at":"ingestion_run","coverage":{"definition_id":"b615b22773d1e90b57e97e16aecb4b80","selection":[{"columns":[],"relative_time":[{"excluded":-3900000},{"included":-3840000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":3840000,"scan_node_id":0,"time_range_node_id":32,"width_ms":60000}],"stored_outputs":[]},"node_id":33,"retention_ms":3900000}]}, +{"name":"runtime_install a_coarse_sum_rolls_up_from_the_stored_fine_sum","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":2,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":{"Named":"sum"}},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[1]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}}],"roots":[5,6],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"db0c9241a79202b6925e84aef010e89a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"3c90a6b5ff3ecaa445ea9bf285e1dc0b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install a_materialized_sliding_window_merge_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":1,"role":"Input","window":"NotApplicable"},{"consumer":3,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":3,"role":"Input","window":"NotApplicable"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":6,"role":"Input","window":"NotApplicable"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":7,"role":"Input","window":"NotApplicable"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":10,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":9,"role":"Input","window":"NotApplicable"},{"consumer":11,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":10,"role":"Input","window":"NotApplicable"},{"consumer":12,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":0,"role":"Input","window":"NotApplicable"},{"consumer":13,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":12,"role":"Input","window":"NotApplicable"},{"consumer":14,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":13,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":8,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":11,"role":"Input","window":"NotApplicable"},{"consumer":15,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":14,"role":"Input","window":"NotApplicable"},{"consumer":16,"data_state":{"primitive":"SummaryState","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"producer":15,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":null,"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":60000}}}}},{"guarantee":null,"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":3,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":120000}}}}},{"guarantee":null,"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":8,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":9,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":180000}}}}},{"guarantee":null,"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":9,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":11,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":10,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":12,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeShift":{"child":0,"shift":{"at":null,"offset_ms":240000}}}}},{"guarantee":null,"id":13,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"TimeRange":{"child":12,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":14,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryAgg":{"child":13,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":null,"id":15,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"SummaryState","timing":"query_time"},"payload":{"ASAP":{"SummaryMerge":{"children":[2,5,8,11,14]}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":16,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":15}}}}],"roots":[16],"scalar_roots":[]},"stored_outputs":[{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-120000},{"included":-60000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":60000,"scan_node_id":0,"time_range_node_id":4,"width_ms":60000}],"stored_outputs":[]},"node_id":5,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-180000},{"included":-120000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":120000,"scan_node_id":0,"time_range_node_id":7,"width_ms":60000}],"stored_outputs":[]},"node_id":8,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-240000},{"included":-180000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":180000,"scan_node_id":0,"time_range_node_id":10,"width_ms":60000}],"stored_outputs":[]},"node_id":11,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":-240000}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":240000,"scan_node_id":0,"time_range_node_id":13,"width_ms":60000}],"stored_outputs":[]},"node_id":14,"retention_ms":3900000},{"built_at":"query_run","coverage":{"definition_id":"71610a93a8af5ae7ee6ecc0839e28d3b","selection":[{"columns":[],"relative_time":[{"excluded":-300000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2,5,8,11,14]},"node_id":15,"retention_ms":3900000}]}, +{"name":"runtime_install a_summary_across_series_from_per_series_summaries_is_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":4,"role":"Input","window":"NotApplicable"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":2,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"Raw","timing":"query_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":6,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":null,"id":4,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":3,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":5,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":4}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[2,0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"NonASAP":{"Project":{"child":6,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"$promql_series_identity","expr":{"Column":3}}],"qualifier":null}}}}],"roots":[5,7],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"5555de6812a18c62345c199d5ba8f06e","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"f35ee4e847802c130cbd5221a81f7ea1","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":4,"retention_ms":60000}]}, +{"name":"runtime_install group_outputs_from_one_stored_per_series_sum_are_listed_and_built","dag":{"edges":[{"consumer":1,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":0,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":2,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":1,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":3,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":4,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":3,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":5,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"producer":4,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":6,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"producer":5,"role":"Input","window":"NotApplicable"},{"consumer":7,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":2,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":8,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":7,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":9,"data_state":{"primitive":"Raw","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"producer":8,"role":"Input","window":"RequiresAlignedPanePhaseOrExactBoundaryResidual"},{"consumer":10,"data_state":{"primitive":"SummaryState","timing":"ingestion_time"},"grouping":"NotApplicable","intermediate_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"producer":9,"role":"Input","window":"NotApplicable"}],"nodes":[{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":0,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"source":{"TimeSeries":{"metric":"m"}}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":1,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}}},{"guarantee":null,"id":2,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":3,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":4,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":3,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":5,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Max","Max"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":4,"family":{"ExactAggregate":["Max","Max"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":6,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"max","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":5}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":7,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":2}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":8,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[4,0]]},"output_state":{"primitive":"Raw","timing":"ingestion_time"},"payload":{"NonASAP":{"Project":{"child":7,"cols":[{"alias":"ts","expr":{"Column":0}},{"alias":"value","expr":{"Column":1}},{"alias":"instance","expr":{"Column":3}},{"alias":"job","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":4}}],"qualifier":null}}}},{"guarantee":null,"id":9,"lifetime":{"AcrossEvaluations":"Memory"},"output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"SummaryState","timing":"ingestion_time"},"payload":{"ASAP":{"SummaryAgg":{"child":8,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2,3]}}}}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"id":10,"lifetime":"None","output_schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"instance","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0,1]]},"output_state":{"primitive":"Raw","timing":"query_time"},"payload":{"ASAP":{"FinalizeExactAccumulator":{"child":9}}}}],"roots":[6,10],"scalar_roots":[]},"stored_outputs":[{"built_at":"ingestion_run","coverage":{"definition_id":"672f965318f2bf2ba350748044719b9b","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[{"end_offset_ms":0,"scan_node_id":0,"time_range_node_id":1,"width_ms":60000}],"stored_outputs":[]},"node_id":2,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"7976839b25e74a9cfac2ab1d6d01a63a","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":5,"retention_ms":60000},{"built_at":"ingestion_run","coverage":{"definition_id":"84209f44520311bced823b9e98eaf85c","selection":[{"columns":[],"relative_time":[{"excluded":-60000},{"included":0}]}]},"inputs":{"raw_sample_windows":[],"stored_outputs":[2]},"node_id":9,"retention_ms":60000}]} ] diff --git a/crates/integration-tests/tests/materialized_views.rs b/crates/integration-tests/tests/materialized_views.rs index c6fad2f29..090b07416 100644 --- a/crates/integration-tests/tests/materialized_views.rs +++ b/crates/integration-tests/tests/materialized_views.rs @@ -3,153 +3,21 @@ use std::collections::HashSet; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_planner::{ - e2e_plan, installable_plan_of, plan_installable, FrontendInput, InstallablePlan, - InstallablePlanError, MaterializedViewRefresh, PlanningModels, RawDataRetention, UserInput, - INSTALLABLE_PLAN_WIRE_VERSION, + InstallablePlan, InstallablePlanError, MaterializedViewRefresh, INSTALLABLE_PLAN_WIRE_VERSION, }; use asap_types::ir::operator::Source; -use asap_types::ir::physical_export::{compile_physical_asap_workload, PhysicalASAPDAG}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use asap_types::physical::Materialization; -use asap_types::workload::PlanningWorkload; -use serde_json::Value; +use asap_types::ir::physical_export::PhysicalASAPDAG; -/// A repeating PromQL workload over ingesting `m`, each query every minute. -fn repeating(queries: &[&str]) -> PlanningWorkload { - repeating_with(queries, "") -} +use asap_types::ir::{NonASAPOp, Operator}; -/// [`repeating`], each entry with `extra` fields (e.g. a range evaluation). -fn repeating_with(queries: &[&str], extra: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|q| { - format!( - r#"{{"query": {q:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{"accuracy": {{"max_error": 0.01, "failure_probability": 0.01}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}{extra}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"m": {{"series_count": 1000, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// Stage 3's priced raw lookback of `dag` (the selected candidate's -/// physical DAG, tumbling-window chains unexpanded), per source: each query-time -/// scan's `PhysicalASAPDAG::scan_window` with every chain kept across -/// evaluations reading its newest tumbling window, as `asap-plan-selection` prices -/// it, the longest per source. -fn stage3_raw_lookback_ms(dag: &PhysicalASAPDAG) -> Vec<(Source, Option)> { - let kept_chains: HashSet<_> = dag - .nodes - .iter() - .filter(|node| { - matches!( - (&node.payload, node.materialization), - ( - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - ) - }) - .map(|node| node.id) - .collect(); - let mut lookback: Vec<(Source, Option)> = Vec::new(); - for node in &dag.nodes { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload else { - continue; - }; - if node.output_state.timing != ExecutionTiming::QueryTime { - continue; - } - let scan = dag.scan_window(node.id, &kept_chains).lookback_ms(); - match lookback.iter_mut().find(|(kept, _)| kept == source) { - Some((_, kept)) => *kept = kept.zip(scan).map(|(a, b)| a.max(b)), - None => lookback.push((source.clone(), scan)), - } - } - lookback -} - -/// `rate(m[5m])` and `rate(m[5m] offset 1h)`, both at query time: the -/// runtime keeps 65 minutes of `m`, the raw lookback Stage 3 prices for the -/// same plan. With no raw data kept, both run from views refreshed at -/// ingestion and no source is read at query time. -#[tokio::test(flavor = "multi_thread")] -async fn raw_data_retention_is_the_lookback_stage3_prices() { - let workload = repeating(&["rate(m[5m])", "rate(m[5m] offset 1h)"]); - let input = || UserInput::new(&workload, FrontendInput::Promql, PlanningModels::builtin()); - let (plan, _) = plan_installable(input()).await.unwrap(); - let m = Source::TimeSeries { metric: "m".into() }; - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new(m.clone(), Some(65 * 60_000))] - ); - let output = e2e_plan(input()).await.unwrap(); - let selected = compile_physical_asap_workload(&output.roots()).unwrap(); - assert_eq!(stage3_raw_lookback_ms(&selected), [(m, Some(65 * 60_000))]); - - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let (plan, _) = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap(); - assert!(!plan.materialized_views().unwrap().is_empty()); - assert_eq!(plan.raw_data_retention().unwrap(), []); -} - -/// A range-evaluated query (`max_over_time(m[5m])` over the last hour at -/// 1-minute steps) installed with every node at query time (its frontend -/// DAG, nothing materialized): its oldest step reads `m` 5 minutes before an -/// hour ago, so the runtime keeps 65 minutes of `m`, the scan's window plus -/// the range span, on the scan's node. (Stage 3 prices one evaluation's -/// window, 5 minutes.) -#[test] -fn a_range_evaluated_query_time_scan_keeps_its_window_plus_the_span() { - let workload = repeating_with( - &["max_over_time(m[5m])"], - r#", "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}"#, - ); - let roots = asap_devtools::planning_stages::lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let (plan, _) = installable_plan_of(&roots, None, &Default::default(), &entries).unwrap(); - plan.validate().unwrap(); - assert!(plan.materialized_views().unwrap().is_empty()); - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new( - Source::TimeSeries { metric: "m".into() }, - Some(5 * 60_000 + 3_600_000) - )] - ); - let scan = plan - .dag - .nodes - .iter() - .find(|node| matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .unwrap(); - assert_eq!(scan.retention_ms, Some(3_900_000)); -} +use serde_json::Value; /// A plan file of wire version 10 (assembled from the parts of a plan -/// written on `main` before #970: the versioned DAG envelope, `queries`, -/// `stored_outputs`, `query_time_raw_windows`, `selection`) is refused for -/// its version, not for a missing field; a file of this version reads back. +/// written before #970: the versioned DAG envelope, `queries`, +/// `stored_outputs`, `query_time_raw_windows`, `selection`; its DAG nodes +/// carry today's node fields, e.g. `lifetime`) is refused for its version, +/// not for a missing field; its DAG with today's version reads back. #[test] fn a_plan_file_of_an_older_wire_version_is_refused_by_version() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) diff --git a/crates/integration-tests/tests/memo_enumeration.rs b/crates/integration-tests/tests/memo_enumeration.rs deleted file mode 100644 index 6566b08e1..000000000 --- a/crates/integration-tests/tests/memo_enumeration.rs +++ /dev/null @@ -1,481 +0,0 @@ -//! The memo the production planner builds, measured on the planner-layering -//! doc's example (`avg by (job) (quantile_over_time(0.5, data[5m]) / -//! quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))`, -//! the stage-pipeline workload -//! `tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json`). -//! -//! Lowering is production's: the PromQL frontend, then each root with the -//! series-identity column, as `Stage1ToStage3Pass` does. The memo is read from -//! the production run's `PlanningRun::per_computation_candidates` (after -//! Pass 1) and `PlanningRun::logical_asap_candidates` (after Pass 2). -//! -//! **The memo and the whole-workload plans.** After Pass 1 the memo's -//! choices whose candidates are all sized for the whole accuracy budget -//! (`accuracy_budget_parts` 1) and the plans `compose_logical_candidate` builds from -//! the Pass 1 inventory's assignments are not equal sets: -//! - A whole-workload plan is a memo DAG exactly when targets that compute -//! the same thing take the same choice and it takes no KLL. With series -//! identity the two `quantile 0.5` are one memo group, which takes one -//! candidate; the inventory keeps a target for each and may give them -//! different summaries. The inventory sizes a KLL per value; Pass 1 sizes -//! every memo KLL for the whole result (a failure probability below the -//! query's, divided over the source's series), so no KLL plan is in both. -//! - A whole-budget memo DAG is a whole-workload plan exactly when it takes -//! no KLL sized for the whole result, that is, no KLL. -//! -//! The memo's other choices size an operand's summary for a share of the -//! budget (a ratio's operands); the whole-workload path has no such plans. -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use std::rc::Rc; - -use asap_devtools::planning_stages::lower_promql_roots; -use asap_integration_tests::planning::expanded_memo; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, -}; -use asap_logical_asap_optimization::{ - build_logical_asap_dags_from_chosen_candidates, enumerate, ChosenCandidatePerMemoGroup, -}; -use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; -use asap_types::ir::schema::{FieldDataType, SketchParams}; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::stages::{queries_rooted_at, MemoGroupId}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::workload::{PlanningWorkload, Query, RepeatedDemand, RepetitionInterval}; - -const RATIO_OF_QUANTILES: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json" -); - -/// The example's failure probability, as in its workload file. -const FAILURE_PROBABILITY: f64 = 0.001; - -/// One root per query, in query order. -type Plan = Vec>; - -fn ratio_of_quantiles() -> PlanningWorkload { - serde_json::from_str(&std::fs::read_to_string(RATIO_OF_QUANTILES).unwrap()).unwrap() -} - -/// The example's workload with `queries` in place of its query, each with -/// its requirements, repeating every `interval_ms`. -fn with_queries(queries: &[&str], interval_ms: u32) -> PlanningWorkload { - let mut workload = ratio_of_quantiles(); - let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); - let mut entry = repeating[0].clone(); - entry.demand = RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)); - *repeating = queries - .iter() - .map(|query| { - let mut entry = entry.clone(); - entry.query = Query((*query).into()); - entry - }) - .collect(); - workload -} - -/// The frontend's roots, each with the series-identity column, as -/// `Stage1ToStage3Pass` and `asap_planner` lower a PromQL workload. -fn production_roots(workload: &PlanningWorkload) -> Vec { - lower_promql_roots(workload).unwrap() -} - -/// The production run of Stages 1 to 3 over `workload`. -fn production_run(workload: &PlanningWorkload) -> PlanningRun { - production_run_with(workload, PlanningModels::builtin()) -} - -/// [`production_run`] with `models`, keeping every stage's memo. -fn production_run_with(workload: &PlanningWorkload, models: PlanningModels<'_>) -> PlanningRun { - let search = PlanSearchSettings { inspect: true }; - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let roots = production_roots(workload); - run_planning(roots, &entries, &data, models, search).unwrap() -} - -/// A deployment model that may accept an output whatever its guarantee, so -/// Pass 1 prunes no summary for its target, and accepts none: Stage 3 -/// validates no plan, and the run still records its memo. -struct AcceptsNoOutput; - -impl asap_accuracy::AccuracyModel for AcceptsNoOutput { - fn satisfies( - &self, - _guarantee: &asap_types::ir::properties::AccuracyGuarantee, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - false - } -} - -/// The memo after Pass 2. -fn pass2_memo(run: &PlanningRun) -> &Memo { - &run.logical_asap_candidates - .as_ref() - .expect("the run recorded its Pass 2 memo") - .memo -} - -/// The memo after Pass 1. -fn pass1_memo(run: &PlanningRun) -> &Memo { - &run.per_computation_candidates - .as_ref() - .expect("Pass 1's output") - .memo -} - -/// Pass 1's target list for `workload`, pruned as the facade's Pass 1 -/// prunes it under `models`. -fn pass1_inventory( - workload: &PlanningWorkload, - models: &PlanningModels<'_>, -) -> PerComputationCandidates { - let roots = production_roots(workload); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap() - .without_unsupported_summaries( - models - .deployment_capabilities - .supported_summaries - .as_deref(), - ); - let accuracy = models.accuracy_check_inputs(); - if accuracy.model.may_accept_regardless_of_guarantee() - || accuracy.evidence.may_provide_evidence() - { - return inventory; - } - let targets: Vec<_> = workload - .query_workload - .entries() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - inventory.without_summaries_no_query_can_use(&targets, &data, accuracy.model) -} - -fn candidate_count(memo: &Memo) -> usize { - memo.candidates.iter().map(Vec::len).sum() -} - -/// Every plan the whole-workload path composes from the inventory: one per -/// assignment of each target's choices, with the assignment. -fn whole_workload_plans(inventory: &PerComputationCandidates) -> Vec<(Vec, Plan)> { - let counts: Vec<_> = inventory.targets.iter().map(|t| t.choice_count()).collect(); - let total: usize = counts.iter().product(); - (0..total) - .map(|mut n| { - counts - .iter() - .map(|&count| { - let digit = n % count; - n /= count; - digit - }) - .collect::>() - }) - .filter(|choice| !inventory.absorbed_choice_varies(choice)) - .filter_map(|choice| { - let roots = compose_logical_candidate(inventory, &choice).ok()?; - let plan = roots - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator roots only"), - }) - .collect(); - Some((choice, plan)) - }) - .collect() -} - -/// `choice`'s DAG, one root per query. -fn composed(memo: &Memo, choice: &ChosenCandidatePerMemoGroup) -> Plan { - let mut roots = build_logical_asap_dags_from_chosen_candidates(memo, choice) - .expect("an enumerated choice composes"); - roots.sort_by_key(|(group, _)| { - queries_rooted_at(&memo.query_roots, *group) - .first() - .copied() - }); - roots.into_iter().map(|(_, root)| root).collect() -} - -fn chosen<'a>( - memo: &'a Memo, - choice: &'a ChosenCandidatePerMemoGroup, -) -> impl Iterator { - choice - .iter() - .map(|(group, &index)| &memo.candidates[group.0 as usize][index]) -} - -/// The KLL parameters of the summaries `candidate` builds. -fn kll(candidate: &CandidateForMemoGroup) -> Vec<(u32, Option)> { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - SketchParams::Kll { - k, - failure_probability, - } => Some((*k, *failure_probability)), - _ => None, - }, - _ => None, - }) - .collect() -} - -/// A KLL Pass 1 sized for the whole result, at the query's failure -/// probability divided over the source's series (1e-7 here), far below what -/// a per-value KLL is sized for (the query's, or half of it). -fn sized_for_whole_result(candidate: &CandidateForMemoGroup) -> bool { - kll(candidate) - .iter() - .any(|(_, p)| p.is_some_and(|p| p < FAILURE_PROBABILITY / 10.0)) -} - -/// Checks the relation of the module doc between the Pass 1 memo's -/// whole-budget DAGs and the whole-workload plans; returns (memo choices, -/// whole-budget ones, whole-workload plans, plans in both). -fn assert_pass1_memo_and_whole_workload_plans( - run: &PlanningRun, - inventory: &PerComputationCandidates, -) -> (usize, usize, usize, usize) { - let memo = pass1_memo(run); - let enumeration = enumerate(memo, 100_000); - assert!(enumeration.complete); - let plans: Vec<_> = enumeration - .all_chosen_candidates - .iter() - .map(|choice| composed(memo, choice)) - .collect(); - for (i, plan) in plans.iter().enumerate() { - assert!(!plans[..i].contains(plan), "each choice is a different DAG"); - } - let whole_budget: Vec<_> = enumeration - .all_chosen_candidates - .iter() - .zip(&plans) - .filter(|(choice, _)| chosen(memo, choice).all(|c| c.accuracy_budget_parts == 1)) - .collect(); - let whole = whole_workload_plans(inventory); - // Targets that compute the same thing. - let same: Vec<(usize, usize)> = (0..inventory.targets.len()) - .flat_map(|a| (a + 1..inventory.targets.len()).map(move |b| (a, b))) - .filter(|&(a, b)| { - let (a, b) = (&inventory.targets[a], &inventory.targets[b]); - inventory.aggregate(a) == inventory.aggregate(b) && a.measure_index == b.measure_index - }) - .collect(); - let takes_kll = |plan: &Plan| { - plan.iter().flat_map(OperatorNode::reachable).any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.params(), SketchParams::Kll { .. })) - }) - }; - for (assignment, plan) in &whole { - assert_eq!( - whole_budget.iter().any(|(_, p)| *p == plan), - same.iter().all(|&(a, b)| assignment[a] == assignment[b]) && !takes_kll(plan), - "a whole-workload plan is a memo DAG when equal targets choose alike \ - and it takes no (per-value) KLL" - ); - assert!( - !plans.iter().any(|p| p == plan) || whole_budget.iter().any(|(_, p)| *p == plan), - "a whole-workload plan is never a budget-share DAG" - ); - } - for (choice, plan) in &whole_budget { - assert_eq!( - whole.iter().any(|(_, p)| p == *plan), - !chosen(memo, choice).any(sized_for_whole_result), - "a whole-budget memo DAG is a whole-workload plan unless it takes a \ - KLL sized for the whole result" - ); - } - let both = whole - .iter() - .filter(|(_, plan)| whole_budget.iter().any(|(_, p)| p == &plan)) - .count(); - (plans.len(), whole_budget.len(), whole.len(), both) -} - -/// The example's memo after each production step, and the `accuracy_budget_parts` -/// of the state-configuration rule's rewrites (one test: one planning run). -#[test] -fn ratio_of_quantiles_production_memo() { - // With the built-in model, Pass 1 prunes every summary of the quantile - // groups: no query has a guarantee through `/` and `+` over a KLL rank - // error or a DDSketch relative one. The memo below is planned with a - // model that prunes none and accepts no output, so its summaries stay - // and its Pass 2 memo is recorded. - let enforced = production_run(&ratio_of_quantiles()); - assert_eq!(candidate_count(pass1_memo(&enforced)), 7); - let models = PlanningModels::builtin().with_accuracy(&AcceptsNoOutput); - let run = production_run_with(&ratio_of_quantiles(), models); - // Stage 0 (Pass 1's groups): series identity makes the two - // `quantile 0.5` one group. - // Pass 1: the two quantile groups have 5 candidates each, 3 for the - // whole budget (exact, DDSketch, KLL sized for the whole result) and 2 - // for half of it; the other 5 groups one each. - assert_eq!(pass1_memo(&run).groups.len(), 7); - assert_eq!(candidate_count(pass1_memo(&run)), 15); - // 5 × 5 choices, 3 × 3 of them for the whole budget. The inventory has - // 3 × 3 assignments (one target per quantile group and measure, so the - // two `quantile 0.5` share one, 3 choices each): the 2 × 2 that take no - // KLL are memo DAGs. - assert_eq!( - assert_pass1_memo_and_whole_workload_plans( - &run, - &pass1_inventory(&ratio_of_quantiles(), &models) - ), - (25, 9, 9, 4) - ); - // The last step, over the complete tumbling-window-width domain. - let last = pass2_memo(&run); - assert_eq!(last.groups.len(), 255); - assert_eq!(candidate_count(last), 1243); - let enumeration = enumerate(last, 2_000); - assert!(!enumeration.complete); - assert_eq!(enumeration.all_chosen_candidates.len(), 2_000); - for (group, candidates) in last.candidates.iter().enumerate() { - for index in 0..candidates.len() { - let choice = [(MemoGroupId(group as u32), index)].into(); - build_logical_asap_dags_from_chosen_candidates(last, &choice) - .unwrap_or_else(|e| panic!("G{group} candidate {index}: {e}")); - } - } - // The state-configuration rule rewrites a candidate's KLL to the size - // the readers of a state share; the rewrite is a copy of the candidate - // (`derive` in `pass2/memo/rules/shared_configuration_readers.rs`) with - // the `accuracy_budget_parts` its new size was made for. In each quantile - // group the `accuracy_budget_parts` 1 KLL `k` 567 (sized for the whole - // result) gets copies at `k` 567 (parts 1) and 1184 (parts 2, the size of - // the half-budget KLL); the `accuracy_budget_parts` 2 KLL `k` 1184 keeps - // 2. The `k` 1184 copy composes the same DAG as the original - // `accuracy_budget_parts` 2 candidate: one sketch, sized for half the - // budget. - let pass1 = pass1_memo(&run); - let quantile_groups: Vec<_> = (0..pass1.groups.len()) - .filter(|&group| pass1.candidates[group].len() == 5) - .collect(); - assert_eq!(quantile_groups.len(), 2); - for group in quantile_groups { - let sizes = |candidates: &[CandidateForMemoGroup]| { - candidates - .iter() - .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Pass1) - .filter_map(|c| kll(c).first().map(|&(k, _)| (k, c.accuracy_budget_parts))) - .collect::>() - }; - assert_eq!(sizes(&pass1.candidates[group]), [(567, 1), (1184, 2)]); - let candidates = &last.candidates[group]; - assert_eq!( - sizes(candidates), - [(567, 1), (1184, 2), (567, 1), (1184, 2)] - ); - let at_k: Vec = candidates - .iter() - .enumerate() - .filter(|(_, c)| { - c.origin == OriginOfCandidateForMemoGroup::Pass1 - && kll(c).first().is_some_and(|&(k, _)| k == 1184) - }) - .map(|(index, _)| index) - .collect(); - assert_eq!(at_k.len(), 2); - let alone = |index: usize| composed(last, &[(MemoGroupId(group as u32), index)].into()); - assert_eq!(alone(at_k[0]), alone(at_k[1])); - } -} - -/// On the three-query workload, the same relation: the memo's 3 × 3 -/// whole-budget DAGs and the inventory's 3 × 3 assignments share the 2 × 2 -/// that take no KLL (the memo's is sized for the whole result, the -/// inventory's per value), with the default search. -#[test] -fn shared_quantiles_pass1_memo_and_whole_workload_plans() { - let workload = with_queries( - &[ - "quantile_over_time(0.5, data[5m])", - "quantile_over_time(0.9, data[5m])", - "sum by (job) (rate(data[1m]))", - ], - 60_000, - ); - let run = production_run(&workload); - assert_eq!( - assert_pass1_memo_and_whole_workload_plans( - &run, - &pass1_inventory(&workload, &PlanningModels::builtin()) - ), - (9, 9, 9, 4) - ); -} - -/// Pass 2 rejects a memo whose groups read each other, a group reading -/// itself included, so production's memo (coarse and fine quantiles, whose -/// rollups read the finer grouping) has a dependency order and every choice -/// `enumerate` returns composes. A cycle made by hand, a child candidate -/// reading back the group that reads it, composes no DAG: -/// `build_logical_asap_dags_from_chosen_candidates` says so. The memo is -/// built from production's roots on the seed grids, without the search. -#[test] -fn choices_whose_candidates_read_each_other_are_left_out() { - let workload = with_queries( - &[ - "quantile_over_time(0.5, data[1m])", - "quantile_over_time(0.9, data[5m])", - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ], - 10_000, - ); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let memo = expanded_memo(production_roots(&workload), &entries); - assert!(memo.topological_order().is_some()); - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - assert!(!candidate.input_groups.contains(&MemoGroupId(group as u32))); - } - } - let enumeration = enumerate(&memo, 2_000); - assert!(!enumeration.all_chosen_candidates.is_empty()); - for choice in &enumeration.all_chosen_candidates { - build_logical_asap_dags_from_chosen_candidates(&memo, choice).unwrap(); - } - - let (group, index, child) = (0..memo.groups.len()) - .flat_map(|group| (0..memo.candidates[group].len()).map(move |index| (group, index))) - .find_map(|(group, index)| { - memo.candidates[group][index] - .input_groups - .first() - .map(|&child| (MemoGroupId(group as u32), index, child)) - }) - .expect("a candidate reading another group"); - let mut cyclic = memo.clone(); - let mut back = cyclic.candidates[child.0 as usize][0].clone(); - back.root_node = Rc::clone(&cyclic.groups[group.0 as usize].logical_node); - back.input_groups = vec![group]; - cyclic.candidates[child.0 as usize].push(back); - let back_index = cyclic.candidates[child.0 as usize].len() - 1; - assert_eq!(cyclic.topological_order(), None); - assert!(matches!( - build_logical_asap_dags_from_chosen_candidates( - &cyclic, - &[(group, index), (child, back_index)].into() - ), - Err(asap_logical_asap_optimization::ErrorBuildingLogicalASAPDagsFromChosenCandidates::Cycle { .. }) - )); -} diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index 88b19291d..d11672f3d 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -1,5 +1,5 @@ //! #511 examples: source text → unified dag → summary rewrite → flat export. -use crate::physical_common; + use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; @@ -44,7 +44,7 @@ async fn sql_filter_projection_example() { assert!( matches!(root.expect_non_asap(),NonASAPOp::Project { cols,.. } if matches!(cols[0].expr,ScalarExpr::Arithmetic { .. })) ); - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), OperatorNode::reachable(&root).len()); } @@ -111,7 +111,7 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { assert!(dag .iter() .any(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))); - let wire = physical_common::compile_physical_asap_dag(&rewritten).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&rewritten).unwrap(); wire.validate().unwrap(); assert_eq!(wire.nodes.len(), dag.len()); } @@ -128,7 +128,7 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { .unwrap(); root.validate_structure().unwrap(); assert!(root.children().len() > 1); - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); assert!(wire .edges .iter() @@ -161,7 +161,7 @@ async fn sql_sum_example_executes_with_sql_null_semantics() { let NonASAPOp::Scan { source, .. } = logical_scan.expect_non_asap() else { panic!() }; - let wire = physical_common::compile_physical_asap_dag(&root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); let scan = wire .nodes .iter() @@ -236,146 +236,3 @@ async fn sql_window_and_filtered_aggregate_types() { assert_eq!(root.schema.fields[0], Field::plain("s", DataType::Int64, true), "{query}"); } } - -/// A real query batch replaces each SUM by its exact accumulator state, -/// retains two result roots, and executes both selected plans. Pass 2 -/// shares the identical state, and building the shared sub-DAG once per -/// evaluation for both queries (`MatAtQuery`, on the state or on its -/// finalized sum above it) is the cheapest plan, so both roots read one -/// state. No replacement dag is constructed by the test. -#[tokio::test] -async fn batch_planning_replaces_summary_operators() { - use asap_executor::values::Value; - use asap_planner::PlanningModels; - use asap_planner::{e2e_plan, FrontendInput, UserInput}; - use asap_types::workload::*; - let queries = [ - "SELECT SUM(bytes) + 1 AS result FROM requests", - "SELECT SUM(bytes) * 2 AS result FROM requests", - ]; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: Some( - queries - .iter() - .map(|query| BatchEntry { - query: Query((*query).into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 2, - execute_at: None, - range_evaluation: None, - }) - .collect(), - ), - repeating_queries: None, - }, - // `requests` at rest, holding a million rows. - data_workload: Some(DataWorkload { - arrival: DataArrival::AtRest, - sources: [( - "requests".to_string(), - asap_types::workload::SourceFacts { - rows_held: Some(1_000_000), - ..Default::default() - }, - )] - .into(), - }), - }; - let catalog = SqlCatalog::new().with_table( - "requests", - Schema::new(vec![Field::plain("bytes", DataType::Float64, false)]), - ); - let output = e2e_plan(UserInput::new( - &workload, - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await - .unwrap(); - assert_eq!(output.entry_indices(), [0, 1]); - assert_eq!(output.roots().len(), 2); - let states: Vec<_> = output - .operators() - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - let [state] = &states[..] else { - panic!("one SUM state for both queries: {states:?}") - }; - assert!(output.operators().into_iter().any(|n| { - matches!( - n.materialization, - asap_types::physical::Materialization::MatAtQuery(_) - ) && OperatorNode::reachable(&n) - .iter() - .any(|m| Rc::ptr_eq(m, state)) - })); - for (plan, expected) in output.plans.iter().zip([31.0, 60.0]) { - let root = &plan.root; - root.validate_structure().unwrap(); - assert!(OperatorNode::reachable(root) - .iter() - .any(|n| Rc::ptr_eq(n, state))); - // The table `requests` bound to its two rows. - let rows = physical_common::execute_raw_rows( - root, - vec![vec![Value::Float64(10.0)], vec![Value::Float64(20.0)]], - ); - assert_eq!(rows.len(), 1); - assert!( - matches!(rows[0][0], Value::Float64(v) if v == expected), - "{:?}", - rows - ); - } - // The batch exports as one physical DAG: a root per query, the one SUM - // state, and the shared node the selected plan builds once. - let workload_dag = output.physical_asap_dag().unwrap(); - assert_eq!(workload_dag.roots.len(), 2); - assert_ne!(workload_dag.roots[0], workload_dag.roots[1]); - let states: Vec<_> = workload_dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - asap_types::ir::ASAPOp::SummaryAgg { .. } - ) - ) - }) - .collect(); - let [state] = &states[..] else { - panic!("one SUM state: {states:?}") - }; - // The node built once is the state or a node above it. - let reaches_state = |from| { - let mut stack = vec![from]; - let mut seen = std::collections::HashSet::new(); - while let Some(id) = stack.pop() { - if id == state.id { - return true; - } - if seen.insert(id) { - stack.extend( - workload_dag - .edges - .iter() - .filter(|e| e.consumer == id) - .map(|e| e.producer), - ); - } - } - false - }; - assert!(workload_dag.nodes.iter().any(|n| matches!( - n.materialization, - asap_types::physical::Materialization::MatAtQuery(_) - ) && reaches_state(n.id))); -} diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 6902cd681..46fa6c962 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -25,11 +25,9 @@ async fn targets(sql: &str) -> Vec { let root = lower_sql(sql, &catalog, accuracy) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ + QueryRoot::Operator(root), + ])) .unwrap() .targets } @@ -79,7 +77,7 @@ fn reads_quantile_sketch(node: &std::rc::Rc) -> bo Some(asap_types::ir::ASAPOp::SummaryAgg { family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) + }) if matches!(kind, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) ) }) } @@ -154,7 +152,7 @@ fn reads_ddsketch(node: &std::rc::Rc) -> bool { Some(asap_types::ir::ASAPOp::SummaryAgg { family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), .. - }) if matches!(kind.algorithm(), SketchAlgorithm::DDSketch) + }) if matches!(kind, SketchAlgorithm::DDSketch) ) }) } @@ -213,12 +211,11 @@ async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ QueryRoot::Operator(root), - ]), - ) - .unwrap(); + ])) + .unwrap(); let name = |expr: &SummaryInputExpr| match expr { SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), SummaryInputExpr::Column(ColumnRef::Qualified { table, name }) => { diff --git a/crates/integration-tests/tests/per_series_exact_execution.rs b/crates/integration-tests/tests/per_series_exact_execution.rs index e20fed25b..e71cc9e35 100644 --- a/crates/integration-tests/tests/per_series_exact_execution.rs +++ b/crates/integration-tests/tests/per_series_exact_execution.rs @@ -14,13 +14,10 @@ use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_integration_tests::fixtures::lower_promql; use asap_types::ir::operator::{Reduction, Source}; -use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOperatorPayload}; +use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{Schema, PROMQL_SERIES_IDENTITY}; -use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, - OperatorNode, -}; +use asap_types::ir::{NonASAPOp, OperatorNode}; use futures::{executor::block_on, StreamExt}; /// One raw data sample: every label of its series, its time and value. @@ -96,13 +93,8 @@ fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap { /// Run `query`'s exact per-entity plan at query time over `samples`, bound /// at its `Scan` of `x`: each answered series' `instance` and value. fn execute(query: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { - let timed = apply_materialization_timings( - &per_entity_plan(query), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let plan = per_entity_plan(query); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(&plan).unwrap(); let node = |matches: fn(&PhysicalASAPOperatorPayload) -> bool| { let found: Vec<_> = dag .nodes diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index e9f16cfa9..998ee0ea0 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -33,40 +33,6 @@ pub fn execute( }) } -#[allow(dead_code)] -pub fn compile_physical_asap_dag( - root: &std::rc::Rc, -) -> Result> { - compile_with(root, &asap_types::ir::MaterializationAssignment::default()) -} - -/// As [`compile_physical_asap_dag`], with every sub-DAG `MatAtIngestion` -/// where the DAG allows it. -#[allow(dead_code)] // Not every test binary sharing this module stores states at ingestion. -pub fn compile_at_ingestion_physical_asap_dag( - root: &std::rc::Rc, -) -> Result> { - compile_with( - root, - &asap_types::ir::MaterializationAssignment::with_default( - asap_types::physical::Materialization::MatAtIngestion( - asap_types::physical::Medium::Memory, - ), - ), - ) -} - -fn compile_with( - root: &std::rc::Rc, - assignment: &asap_types::ir::MaterializationAssignment, -) -> Result> { - let root = - asap_types::ir::apply_materialization_timings(root, assignment, &mut Default::default())?; - Ok(asap_types::ir::physical_export::compile_physical_asap_dag( - &root, - )?) -} - /// Execute a relational DAG at query time against an in-memory connector /// for its one scan, scan predicates included; checks that no memory stays /// reserved afterwards. @@ -89,7 +55,7 @@ pub fn execute_raw_rows_at( use asap_executor::physical_planner::compile; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::NonASAPOp; - let wire = compile_physical_asap_dag(root).unwrap(); + let wire = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let (source, schema) = wire .nodes .iter() diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 47357ac32..a62cba38b 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -18,12 +18,12 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use asap_cost::CostInputs; use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::reference_pricer::WholePlanSelection as Selection; use asap_types::ir::flat::{flatten, FlatDag, NodeId}; use asap_types::ir::physical_export::PhysicalASAPDAG; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; -use asap_types::physical::Materialization; +use asap_types::physical::Lifetime; use asap_types::physical::PhysicalWorkloadPlan; use asap_types::workload::{PlanningWorkload, QueryRecurrence, RepeatedDemand}; @@ -31,28 +31,6 @@ pub use asap_devtools::planner_layering_examples::*; pub type Payload = Operator; -/// Lower and plan a PromQL workload. -pub fn run_promql(workload: &PlanningWorkload) -> Run { - run_stages(workload, lower_promql(workload)) -} - -/// Lower and plan a PromQL workload, selecting under `settings` (a cost -/// model other than the default). -pub fn run_promql_under(workload: &PlanningWorkload, settings: &CostInputs) -> Run { - run_stages_under(workload, lower_promql(workload), settings) -} - -/// Lower and plan a PromQL workload for a deployment with `deployment`'s -/// capabilities, under the default cost model. -pub fn run_promql_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { - run_stages_with( - workload, - lower_promql(workload), - &CostInputs::default(), - deployment, - ) -} - /// Whether `form` answers a query window by merging several summaries. pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { match form { @@ -193,42 +171,6 @@ impl Physical { } } -#[derive(Debug, Clone)] -pub struct Run { - pub stage0_query_language_frontends: Logical, - pub logical: Vec, - pub physical: Vec, - /// Logical candidates physical ASAP-aware optimization cannot implement: - /// (logical id, reason). - pub pruned: Vec<(String, String)>, - pub selection: Selection, -} - -impl Run { - pub fn invalid(&self) -> BTreeMap<&str, &str> { - self.selection - .rejected - .iter() - .filter(|r| r.cause.failed_a_check()) - .map(|r| (r.id.as_str(), r.reason.as_str())) - .collect() - } - - pub fn cost(&self, id: &str) -> Option { - self.selection.costs.get(id).and_then(|c| c.cost_per_second) - } - - pub fn physical(&self, id: &str) -> &Physical { - self.physical.iter().find(|p| p.id == id).expect("id") - } - - pub fn physical_of<'a>(&'a self, logical: &'a Logical) -> impl Iterator { - self.physical - .iter() - .filter(move |p| p.from_logical == logical.id) - } -} - fn logical(id: String, roots: Vec) -> Logical { let dag = flatten(&roots).0; let query_roots = dag @@ -247,212 +189,6 @@ fn logical(id: String, roots: Vec) -> Logical { } } -/// The bounded production stages over `roots`, retaining the selected plan -/// and search diagnostics. Tests that require a particular algorithm or window -/// use `run_explicit_candidates` instead of assuming it appears in the archive. -pub fn run_stages(workload: &PlanningWorkload, roots: Vec) -> Run { - run_stages_under(workload, roots, &CostInputs::default()) -} - -/// [`run_stages`], selecting under `settings`. -pub fn run_stages_under( - workload: &PlanningWorkload, - roots: Vec, - settings: &CostInputs, -) -> Run { - run_stages_with( - workload, - roots, - settings, - &DeploymentCapabilities::default(), - ) -} - -/// [`run_stages`], selecting under `settings` for a deployment with -/// `deployment`'s capabilities. -pub fn run_stages_with( - workload: &PlanningWorkload, - roots: Vec, - settings: &CostInputs, - deployment: &DeploymentCapabilities, -) -> Run { - let models = asap_planner::PlanningModels::builtin() - .with_cost_inputs(settings) - .with_deployment_capabilities(deployment); - run_stages_over(workload, roots, models) -} - -fn run_stages_over( - workload: &PlanningWorkload, - roots: Vec, - models: asap_planner::PlanningModels<'_>, -) -> Run { - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - // The facade's run, with the reference pricer over the Stage 2 - // memo choices: the whole-plan costs and rejections these tests read. - let run = run_planning_stages_over(roots, &entries, &data, None, models) - .expect("planning stages run"); - let reference = run.reference.expect("the memo is enumerated"); - let logical_candidates = reference - .choices - .into_iter() - .map(|c| { - let roots = c.roots.unwrap_or_else(|e| panic!("{} composes: {e}", c.id)); - logical(c.id, roots) - }) - .collect(); - let physical = reference - .physical - .into_iter() - .map(|candidate| Physical { - id: candidate.id.clone(), - from_logical: candidate.from_logical.clone(), - query_roots: candidate.dag.roots.clone(), - candidate, - }) - .collect(); - Run { - stage0_query_language_frontends: logical( - "stage0-query-language-frontends".into(), - run.stage0_query_language_frontends, - ), - logical: logical_candidates, - physical, - pruned: reference.stage2_failed, - selection: reference.selection.expect("plan selection selects"), - } -} - -/// Evaluate deliberately supplied logical configurations. This fixture helper -/// does not search or enumerate cross-query combinations; it exercises the -/// existing physical rules and evaluator on the computations named by a test. -/// Production-search tests continue to use `run_stages`. -pub fn run_explicit_candidates( - workload: &PlanningWorkload, - original_roots: Vec, - candidates: Vec>, - models: asap_planner::PlanningModels<'_>, -) -> Run { - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - // Explicit candidates are composed outside the memo: their summaries are - // sized as Pass 1 sizes a candidate, which Stage 3 only checks. - let logical: Vec<_> = candidates - .into_iter() - .enumerate() - .map(|(index, roots)| { - let roots = asap_accuracy::propagation::recompose_guarantees(&roots) - .expect("explicit fixture contracts compose"); - logical( - format!("P{}", index + 1), - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, &data) - .expect("explicit fixture summaries rebuild"), - ) - }) - .collect(); - let mut physical = Vec::new(); - for candidate in &logical { - for (index, mut plan) in asap_physical_asap_optimization::optimize_physical_asap( - &candidate.id, - &candidate.roots, - &entries, - &data, - models.deployment_capabilities.query_time_results_retained, - ) - .expect("explicit computation has executable physical configurations") - .into_iter() - .enumerate() - { - plan.id = if index == 0 { - candidate.id.clone() - } else { - format!("{}-m{}", candidate.id, index) - }; - physical.push(Physical { - id: plan.id.clone(), - from_logical: plan.from_logical.clone(), - query_roots: plan.dag.roots.clone(), - candidate: plan, - }); - } - } - // Stage 2's deployment rules (raw data retention) prune the plans the - // deployment cannot run before Stage 3 prices the rest. - let mut plans: Vec<_> = physical.iter().map(|plan| plan.candidate.clone()).collect(); - asap_devtools::memo_pricing::retain_deployable(&mut plans, models); - physical.retain(|p| plans.iter().any(|plan| plan.id == p.id)); - let selection = asap_devtools::memo_pricing::reference_select(&plans, &entries, &data, models) - .expect("explicit fixture contains a valid physical configuration"); - Run { - stage0_query_language_frontends: logical_stage0(original_roots), - logical, - physical, - pruned: Vec::new(), - selection, - } -} - -pub fn run_pattern_b_fixture( - workload: &PlanningWorkload, - settings: &CostInputs, - deployment: &DeploymentCapabilities, -) -> Run { - run_explicit_candidates( - workload, - lower_promql(workload), - pattern_b_computations(workload), - asap_planner::PlanningModels::builtin() - .with_cost_inputs(settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Three fixed Pattern-B computations used by cost/retention fixtures: -/// exact, whole-window KLL, and a KLL chain of sixty-second tumbling windows. -pub fn pattern_b_computations(workload: &PlanningWorkload) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!( - local.targets.len(), - 1, - "Pattern B has one quantile computation" - ); - let kll = local.targets[0] - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap(); - let whole: Vec<_> = compose_logical_candidate(&local, &[kll]).unwrap(); - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(60_000)]); - assert_eq!(tumbling_windows.len(), 1); - let chain = window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .expect("sixty-second tumbling windows compose"); - vec![roots, whole, chain] -} - fn logical_stage0(roots: Vec) -> Logical { logical("stage0-query-language-frontends".into(), roots) } @@ -576,27 +312,6 @@ pub fn is_merge(payload: &Payload) -> bool { matches!(payload, Operator::ASAP(ASAPOp::SummaryMerge { .. })) } -/// The sketch build nodes (`SummaryAgg` or tumbling-window `SummaryTumblingWindowChain` over a -/// sketch family) of `dag`, with their algorithm and parameters. -pub fn sketch_builds(dag: &impl ExportedDag) -> Vec<(NodeId, SketchAlgorithm, SketchParams)> { - dag.node_ids() - .into_iter() - .filter_map(|id| match dag.payload(id) { - Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some((id, *kind.algorithm(), kind.params().clone())), - _ => None, - }) - .collect() -} - /// The estimation nodes that read `build`, directly or through merges. pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, SketchStatistic)> { let mut out = Vec::new(); @@ -619,23 +334,6 @@ pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, Sketc out } -/// One query's per-computation choice: the sorted sketch algorithms built in -/// its closure, or `"exact"` when it builds none. -pub fn query_option(dag: &impl ExportedDag, query_roots: &[NodeId], query: usize) -> String { - let reach = closure(dag, query_roots[query]); - let mut names: Vec<_> = sketch_builds(dag) - .into_iter() - .filter(|(id, ..)| reach.contains(id)) - .map(|(_, algorithm, _)| format!("{algorithm:?}")) - .collect(); - names.sort(); - if names.is_empty() { - "exact".into() - } else { - names.join("+") - } -} - /// Nodes reachable from more than one query root. pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTreeSet { dag.node_ids() @@ -644,143 +342,18 @@ pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTre .collect() } -pub fn assert_valid_and_uniquely_named(run: &Run) { - let ids: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); - assert_eq!(ids.len(), run.logical.len(), "unique logical ids"); - for c in &run.logical { - for root in &c.roots { - root.validate_structure() - .unwrap_or_else(|e| panic!("{}: {e}", c.id)); - } - } -} - -/// No candidate is discarded before plan selection: physical ASAP-aware -/// optimization gives each logical candidate `L` exactly one physical -/// candidate with nothing materialized, `P`, and one `P-m` for each -/// other materialization assignment. -pub fn assert_one_unmaterialized_physical_candidate_per_logical(run: &Run) { - assert!(run.pruned.is_empty(), "pruned: {:?}", run.pruned); - let unmaterialized: Vec<_> = run - .physical - .iter() - .filter(|p| !p.id.contains("-m")) - .collect(); - let sources: BTreeSet<_> = unmaterialized - .iter() - .map(|p| p.from_logical.as_str()) - .collect(); - let logical: BTreeSet<_> = run.logical.iter().map(|c| c.id.as_str()).collect(); - assert_eq!(sources, logical); - assert_eq!(unmaterialized.len(), run.logical.len()); - for p in &run.physical { - let base = p.id.split("-m").next().unwrap(); - assert_eq!(&base[1..], &p.from_logical[1..], "{}", p.id); - let unmaterialized = !p.id.contains("-m"); - assert_eq!(unmaterialized, !p.candidate.materializes_any(), "{}", p.id); - if unmaterialized { - for n in &p.dag().nodes { - assert_eq!( - materialization(p, n.id), - MaterializationOption::NotMaterialized - ); - } - } - } - let ids: BTreeSet<_> = run.physical.iter().map(|p| p.id.as_str()).collect(); - assert_eq!(ids.len(), run.physical.len(), "unique physical ids"); -} - -/// The physical candidates with nothing materialized, one per logical -/// candidate. -pub fn unmaterialized(run: &Run) -> impl Iterator { - run.physical - .iter() - .filter(|p| !p.candidate.materializes_any()) -} - -/// One selected id; every other candidate rejected once, with a reason. -pub fn assert_selects_one_and_explains_the_rest(run: &Run) { - let all: BTreeSet<_> = run.physical.iter().map(|p| p.id.clone()).collect(); - let mut accounted: BTreeSet<_> = run - .selection - .rejected - .iter() - .map(|r| r.id.clone()) - .collect(); - assert_eq!( - accounted.len(), - run.selection.rejected.len(), - "rejected once each" - ); - assert!(run.selection.rejected.iter().all(|r| !r.reason.is_empty())); - assert!( - accounted.insert(run.selection.selected.clone()), - "selected is not rejected" - ); - assert_eq!(accounted, all); -} - -/// The selected plan costs no more than any valid candidate. -pub fn assert_selects_cheapest_valid(run: &Run) { - let invalid = run.invalid(); - let best = run - .cost(&run.selection.selected) - .expect("selected is priced"); - for p in run - .physical - .iter() - .filter(|p| !invalid.contains_key(p.id.as_str())) - { - // Totals within rounding of each other tie (plan selection's rule). - let cost = run.cost(&p.id).unwrap(); - assert!(best <= cost + cost.abs() * 1e-9, "{} is cheaper", p.id); - } -} - -/// Each valid candidate's cost has one entry per DAG node, `total` is their -/// sum, and each node's multiplier is [`computations`]. Invalid candidates -/// are not priced. -pub fn assert_each_node_charged_per_consuming_query(run: &Run) { - let invalid = run.invalid(); - for p in &run.physical { - if invalid.contains_key(p.id.as_str()) { - assert!(run.cost(&p.id).is_none(), "{} is priced", p.id); - continue; - } - let cost = &run.selection.costs[&p.id]; - let nodes: BTreeSet<_> = p.dag().nodes.iter().map(|n| n.id).collect(); - let charged: BTreeSet<_> = cost.per_node.keys().copied().collect(); - assert_eq!(charged, nodes, "{}", p.id); - let sum: f64 = cost - .per_node - .values() - .map(|c| c.cost_per_second.unwrap()) - .sum(); - assert!( - (cost.cost_per_second.unwrap() - sum).abs() <= 1e-9 * sum.abs().max(1.0), - "{}", - p.id - ); - let times = computations(p.dag()); - for (id, c) in &cost.per_node { - assert_eq!(c.multiplier, times[id], "{}: multiplier of {id:?}", p.id); - } - } -} - /// How many times each node is computed per evaluation of its consumers (#509; the /// rule of plan selection's private `computations`): 1 for a materialized -/// node; for a `NotMaterialized` one, the number of consumers that reach +/// node; for a not-materialized one, the number of consumers that reach /// it. A consumer is a query (operator or scalar root) or a materialized /// node, whose walk starts at its inputs; a walk stops at materialized -/// nodes and counts each node once. The `MatAtIngestion` consumers of one +/// nodes and counts each node once. The consumers materialized at ingestion time of one /// refresh window share one ingestion run and count once (#1151). pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { - let materialized: BTreeSet<_> = dag + let materialized_nodes: BTreeSet<_> = dag .nodes .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) + .filter(|n| n.lifetime != asap_types::physical::Lifetime::None) .map(|n| n.id) .collect(); // Each walk's start and, for a consumer refreshed at ingestion, its @@ -794,22 +367,22 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { .iter() .map(|(_, expr)| (expr.operator_refs().into_iter().copied().collect(), None)), ) - .chain(materialized.iter().map(|&id| { - let at_ingestion = matches!( - dag.nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .materialization, - Materialization::MatAtIngestion(_) - ); + .chain(materialized_nodes.iter().map(|&id| { + let at_ingestion = dag + .nodes + .iter() + .find(|n| n.id == id) + .unwrap() + .output_state + .timing + == ExecutionTiming::IngestionTime; let window = at_ingestion.then(|| dag.refresh_window(id)).flatten(); (dag.producers(id), window) })); let mut times: BTreeMap<_, _> = dag .nodes .iter() - .map(|n| (n.id, u64::from(materialized.contains(&n.id)))) + .map(|n| (n.id, u64::from(materialized_nodes.contains(&n.id)))) .collect(); // The (node, refresh window) pairs whose ingestion run is counted. let mut runs = HashSet::new(); @@ -817,7 +390,7 @@ pub fn computations(dag: &PhysicalASAPDAG) -> BTreeMap { let mut seen = HashSet::new(); let mut stack = start; while let Some(id) = stack.pop() { - if materialized.contains(&id) || !seen.insert(id) { + if materialized_nodes.contains(&id) || !seen.insert(id) { continue; } if window.is_none_or(|window| runs.insert((id, window))) { @@ -838,32 +411,30 @@ pub fn evaluation_interval_ms(workload: &PlanningWorkload, query: usize) -> Opti } } -// ── Materialization ────────────────────────────────────────────────────── +// ── Materialized outputs ───────────────────────────────────────────────────────── -/// A node's materialization option (#509 "Materialization"), without its -/// storage medium. +/// Whether a node's output is materialized and at which timing, without how long +/// (`OneEvaluation` or `AcrossEvaluations`) or its storage medium. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum MaterializationOption { - /// Runs as data arrives; output stored before any query asks. - MatAtIngestion, +pub enum Materializing { + /// Runs as data arrives; output materialized until a query reads it. + MaterializedAtIngestion, /// Runs when a query first needs it; output reused by the other - /// queries of the evaluation, or, as `MatAtQueryKeptAcrossEvaluations`, + /// queries of the evaluation, or, as query-time `AcrossEvaluations`, /// kept for later evaluations. - MatAtQuery, - /// Runs at query time for each execution; output discarded. + MaterializedAtQueryTime, + /// Output discarded after its reader (lifetime `None`). NotMaterialized, } -/// The materialization option physical ASAP-aware optimization assigned to -/// `node` (#667). -pub fn materialization(p: &Physical, node: NodeId) -> MaterializationOption { +/// Whether physical ASAP-aware optimization materializes `node`'s output, and at +/// which timing (#667). +pub fn materializing(p: &Physical, node: NodeId) -> Materializing { let n = p.dag().nodes.iter().find(|n| n.id == node).expect("node"); - match n.materialization { - Materialization::MatAtIngestion(_) => MaterializationOption::MatAtIngestion, - Materialization::MatAtQuery(_) | Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - MaterializationOption::MatAtQuery - } - Materialization::NotMaterialized => MaterializationOption::NotMaterialized, + match (n.output_state.timing, n.lifetime) { + (_, Lifetime::None) => Materializing::NotMaterialized, + (ExecutionTiming::IngestionTime, _) => Materializing::MaterializedAtIngestion, + (ExecutionTiming::QueryTime, _) => Materializing::MaterializedAtQueryTime, } } @@ -872,25 +443,6 @@ pub fn runs_at_ingestion(p: &Physical, node: NodeId) -> bool { n.output_state.timing == asap_types::ir::properties::ExecutionTiming::IngestionTime } -/// Every node upstream of an ingestion-time node also runs at ingestion time. -pub fn assert_ingestion_upstream_is_ingestion(run: &Run) { - for p in &run.physical { - for n in &p.dag().nodes { - if !runs_at_ingestion(p, n.id) { - continue; - } - for up in closure(p.dag(), n.id) { - assert!( - runs_at_ingestion(p, up), - "{}: {up:?} feeds ingestion-time {:?} at query time", - p.id, - n.id - ); - } - } - } -} - // ── Pending adapters ───────────────────────────────────────────────────── /// The window summary a summary build node maintains (#509 Pass 2, diff --git a/crates/integration-tests/tests/planner_layering_example1.rs b/crates/integration-tests/tests/planner_layering_example1.rs deleted file mode 100644 index ca983c655..000000000 --- a/crates/integration-tests/tests/planner_layering_example1.rs +++ /dev/null @@ -1,951 +0,0 @@ -//! Rule-fixture and production acceptance for Example 1: aggregation across dimensions. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet, HashSet}; - -use asap_devtools::memo_pricing::reference_select as select_plan; -use asap_planner::PlanningModels; -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::{FieldDataType, GroupingStrategy, HydraKind, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot, ScalarExpr, SortKey}; -use asap_types::physical::{Materialization, Medium}; -use planner_layering_common::*; - -// ── Example 1 workload ─────────────────────────────────────────────────── - -fn pipeline() -> Run { - rule_fixture(&example1()) -} - -// Explicit rule fixtures: vary Q2's three documented algorithms while Q1 -// stays exact, and exercise source sharing and ten-second tumbling windows. This is not -// a production workload search or an exhaustive memo archive. -fn rule_fixture(workload: &asap_types::workload::PlanningWorkload) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = Vec::new(); - for family in [ - None, - Some(SketchAlgorithm::CmsWithHeap), - Some(SketchAlgorithm::CountSketchWithHeap), - ] { - let mut choice = vec![0; local.targets.len()]; - if let Some(family) = family { - let (target, option) = local - .targets - .iter() - .enumerate() - .find_map(|(target, choices)| { - (0..choices.choice_count()) - .find(|&option| { - matches!(choices.computation_for(option), - Some((HowToComputeAggregateMeasure::Summary(algorithm), _)) if *algorithm == family) - }) - .map(|option| (target, option)) - }) - .expect("documented heap family offered locally"); - choice[target] = option; - } - let whole = unshare(&compose_logical_candidate(&local, &choice).unwrap()); - let mut windows = vec![whole.clone()]; - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(10_000), Some(10_000)]); - windows.extend( - window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .into_iter() - .map(|candidate| candidate.roots.unwrap()), - ); - // Each window form alone and with its identical input shared. The - // memo shares an identical range selector whole, never its scan - // alone; that variant reads the same rows twice and is dominated. - for window in windows { - let shared = share_identical(&window); - let sharing = distinct_nodes(&shared) < distinct_nodes(&window); - computations.push(window); - if sharing { - computations.push(shared); - } - } - } - run_explicit_candidates(workload, roots, computations, PlanningModels::builtin()) -} - -#[test] -fn production_memo_selects_an_executable_accurate_example1_plan() { - let workload = example1(); - let run = run_stages_under(&workload, lower_promql(&workload), &Default::default()); - assert_selects_cheapest_valid(&run); - compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -/// Q2's per-computation option for its top-k, read off its summary build node. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -enum Q2Option { - Exact, - CountMinHeapPerJob, - CountSketchHeapPerJob, - Hydra, -} - -const PASS1_Q2_OPTIONS: [Q2Option; 3] = [ - Q2Option::Exact, - Q2Option::CountMinHeapPerJob, - Q2Option::CountSketchHeapPerJob, -]; - -/// Sketch families built in `nodes`, as Example 1's Q2 options. -fn sketch_options(dag: &impl ExportedDag, nodes: &HashSet) -> BTreeSet { - nodes - .iter() - .filter_map(|&id| match dag.payload(id) { - Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, grouping), - .. - }, - ) => Some(match (grouping, kind.algorithm()) { - ( - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - .. - }, - _, - ) => Q2Option::Hydra, - (GroupingStrategy::PerSubpopulationInstance, SketchAlgorithm::CmsWithHeap) => { - Q2Option::CountMinHeapPerJob - } - ( - GroupingStrategy::PerSubpopulationInstance, - SketchAlgorithm::CountSketchWithHeap, - ) => Q2Option::CountSketchHeapPerJob, - (other, algorithm) => { - panic!("summary family outside Example 1: {algorithm:?} {other:?}") - } - }), - _ => None, - }) - .collect() -} - -/// Whether every summary of `dag` is built over its whole window: no -/// tumbling windows (Pass 2's window-composition rule). -fn whole_windows(dag: &impl ExportedDag) -> bool { - dag.node_ids().into_iter().all(|id| { - !matches!( - dag.payload(id), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) -} - -fn roots(query_roots: &[NodeId]) -> (NodeId, NodeId) { - assert_eq!(query_roots.len(), 2, "every candidate covers Q1 and Q2"); - (query_roots[0], query_roots[1]) -} - -/// Q2's option and whether Q1 and Q2 share any node, for one candidate. -fn classify(dag: &impl ExportedDag, query_roots: &[NodeId]) -> (Q2Option, bool) { - let (q1, q2) = roots(query_roots); - let (c1, c2) = (closure(dag, q1), closure(dag, q2)); - let options = sketch_options(dag, &c2); - assert!(options.len() <= 1, "Q2 uses one option: {options:?}"); - let option = options.into_iter().next().unwrap_or(Q2Option::Exact); - (option, !c1.is_disjoint(&c2)) -} - -/// The candidate with Q2 `option` whose queries share no node and whose -/// summaries are built over whole windows (Pass 2 adds shared-input and -/// tumbling-window variants of each). -fn physical_with(run: &Run, option: Q2Option) -> &Physical { - let mut found = run - .physical - .iter() - .filter(|p| whole_windows(p.dag()) && classify(p.dag(), &p.query_roots) == (option, false)); - let p = found.next().unwrap_or_else(|| panic!("no {option:?}")); - assert!(found.next().is_none(), "one {option:?} candidate"); - p -} - -/// Example 1's panels repeat every 10 s. -fn every_10s() -> asap_types::workload::QueryRecurrence { - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(10_000))) -} - -/// The workload entry of an exact query evaluated every 10 s. -fn exact_every_10s() -> asap_types::workload::QueryWorkloadEntry { - asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: Default::default(), - predictability: Default::default(), - recurrence: every_10s(), - range_evaluation: None, - } -} - -// ── Workload ───────────────────────────────────────────────────────────── - -/// The encoded workload is valid and normalizes to Q1 then Q2. -#[test] -fn workload_encodes_example1() { - let workload = example1(); - workload.validate().expect("valid workload"); - let entries: Vec<_> = workload.query_workload.entries().collect(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].query.0, EXAMPLE1_Q1); - assert_eq!(entries[1].query.0, EXAMPLE1_Q2); -} - -// ── Query-language frontends ────────────────────────────────────────── - -/// The PromQL frontend lowers each query to the doc's chain. -#[test] -fn stage0_query_language_frontends_lower_each_query_to_doc_chain() { - let roots = lower_promql(&example1()); - let chains: Vec<_> = roots.iter().map(frontend_operations).collect(); - assert_eq!( - chains, - [ - ["aggregate:rate", "aggregate:sum", "scan", "time_range"], - ["aggregate:sum", "aggregate:top_k", "scan", "time_range"], - ] - ); -} - -/// The frontends yield one workload DAG with one root per query and no summaries. -#[test] -fn stage0_query_language_frontends_one_summary_free_workload_dag() { - let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; - roots(&stage0_query_language_frontends.query_roots); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); -} - -/// The frontends keep Q1 and Q2 separate; sharing is a logical ASAP-aware optimization decision. -#[test] -fn stage0_query_language_frontends_queries_do_not_share_nodes() { - let stage0_query_language_frontends = pipeline().stage0_query_language_frontends; - assert!(!stage0_query_language_frontends.shared_input()); -} - -// ── Logical ASAP-aware optimization ────────────────────────────────────── - -/// Pass 1 gives 1 × 3 = 3 candidates without a shared input: Q2's top-k is -/// exact, Count-Min + heap or CountSketch + heap. (Pass 2 adds shared-input -/// and tumbling-window variants of each.) -#[test] -fn stage1_logical_asap_optimization_has_three_pass1_candidates() { - let run = pipeline(); - let found: Vec<_> = run - .logical - .iter() - .filter(|c| whole_windows(&c.dag)) - .map(|c| classify(&c.dag, &c.query_roots)) - .filter(|(_, shared)| !shared) - .collect(); - let expected: Vec<_> = PASS1_Q2_OPTIONS.into_iter().map(|o| (o, false)).collect(); - let mut sorted = found.clone(); - sorted.sort(); - assert_eq!(sorted, expected, "each Q2 option exactly once: {found:?}"); -} - -/// Q1 is exact in every candidate: no sketch build or estimate is reachable -/// from its root (its rate may be merged from exact tumbling windows). -#[test] -fn stage1_logical_asap_optimization_q1_is_always_exact() { - let run = pipeline(); - for c in &run.logical { - let (q1, _) = roots(&c.query_roots); - assert!( - closure(&c.dag, q1).iter().all(|&id| { - let payload = c.dag.payload(id); - !is_summary(payload) || is_merge(payload) - }), - "{}: Q1 reaches a summary", - c.id - ); - } -} - -/// Every Pass 1 target's exact option is its exact accumulator: Q1's rate -/// and sum and Q2's sum_over_time. Q2's heaps are whole-expression: they -/// absorb its sum_over_time, which is then not computed. -#[test] -fn stage1_logical_asap_optimization_exact_targets_use_accumulators() { - let run = pipeline(); - for c in &run.logical { - let expected = match classify(&c.dag, &c.query_roots).0 { - Q2Option::Exact => 3, - _ => 2, - }; - let accumulators = c - .dag - .nodes - .iter() - .filter(|n| { - matches!( - n.operator, - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) - }) - .count(); - assert_eq!(accumulators, expected, "{}", c.id); - } -} - -/// Q2's summary families are Count-Min + heap and CountSketch + heap, one per job. -#[test] -fn stage1_logical_asap_optimization_q2_summary_families() { - let run = pipeline(); - let families: BTreeSet<_> = run - .logical - .iter() - .flat_map(|c| { - let (_, q2) = roots(&c.query_roots); - sketch_options(&c.dag, &closure(&c.dag, q2)) - }) - .collect(); - assert_eq!( - families, - BTreeSet::from([ - Q2Option::CountMinHeapPerJob, - Q2Option::CountSketchHeapPerJob - ]) - ); -} - -/// Pass 1 also offers Hydra (HydraCms over all (job, series) keys, a heap per job) for Q2's top-k. -#[test] -#[ignore = "needs Pass 1 Hydra for grouped approximate counts (#703)"] -fn stage1_logical_asap_optimization_q2_offers_hydra() { - let run = pipeline(); - assert!(run - .logical - .iter() - .any(|c| classify(&c.dag, &c.query_roots).0 == Q2Option::Hydra)); -} - -/// Pass 2's identical-expression rule adds a shared-input variant of every Q2 option and keeps the independent one. -#[test] -fn stage1_logical_asap_optimization_keeps_independent_and_shared_variants() { - let run = pipeline(); - let found: Vec<_> = run - .logical - .iter() - .map(|c| classify(&c.dag, &c.query_roots)) - .collect(); - for option in PASS1_Q2_OPTIONS { - assert!(found.contains(&(option, false)), "{option:?} independent"); - assert!(found.contains(&(option, true)), "{option:?} shared"); - } -} - -/// Pass 2's window-composition rule builds Q1's rate and Q2's option in six -/// 10-s tumbling windows (the gcd of the 1-min window and the 10-s interval), -/// each alone and both together, next to the whole-window candidates: 27 -/// candidates in all. -#[test] -fn stage1_logical_asap_optimization_window_composition_adds_10s_tumbling_windows() { - let run = pipeline(); - let mut chains = BTreeSet::new(); - for c in &run.logical { - for id in c.dag.node_ids() { - if let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = c.dag.payload(id) - { - let state = match family { - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?}"), - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - other => panic!("{}: {other:?}", c.id), - }; - let query = readers(&c.dag, &c.query_roots, id); - chains.insert(( - state, - query, - *tumbling_window_width_ms, - *tumbling_window_count, - )); - } - } - } - let expected: BTreeSet<_> = [ - ("Rate", 0), - ("Sum", 1), - ("CmsWithHeap", 1), - ("CountSketchWithHeap", 1), - ] - .into_iter() - .map(|(state, query)| (state.to_string(), BTreeSet::from([query]), 10_000, 6)) - .collect(); - assert_eq!(chains, expected); - let whole = run.logical.iter().filter(|c| whole_windows(&c.dag)).count(); - assert!(whole > 0); -} - -/// Only the raw input is ever shared between Q1 and Q2; no summary is shared. -#[test] -fn stage1_logical_asap_optimization_shares_input_but_never_a_summary() { - let run = pipeline(); - for c in &run.logical { - for id in cross_query_nodes(&c.dag, &c.query_roots) { - let p = c.dag.payload(id); - assert!( - matches!(relational(p).as_deref(), Some("scan" | "time_range")), - "{}: shared node {id:?} is not the range selector input: {p:?}", - c.id - ); - } - } -} - -/// Logical candidates are valid DAGs with unique ids. -#[test] -fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { - assert_valid_and_uniquely_named(&pipeline()); -} - -// ── Physical ASAP-aware optimization ───────────────────────────────────── - -/// No candidate is discarded before plan selection: each logical candidate -/// keeps one physical candidate with nothing materialized, 3 of them -/// without a shared input or tumbling windows. -#[test] -fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { - let run = pipeline(); - assert_one_unmaterialized_physical_candidate_per_logical(&run); - let unshared = unmaterialized(&run) - .filter(|p| whole_windows(p.dag()) && !classify(p.dag(), &p.query_roots).1) - .count(); - assert_eq!(unshared, 3); -} - -/// Each physical candidate keeps its logical Q2 option, and one with -/// nothing materialized keeps its input sharing. Maintaining Q2's tumbling windows at -/// ingestion time splits a scan Q1 reads at query time, one copy per phase. -#[test] -fn stage2_physical_asap_optimization_preserves_logical_choices() { - let run = pipeline(); - for p in &run.physical { - let source = run.logical.iter().find(|c| c.id == p.from_logical).unwrap(); - let (option, shared) = classify(p.dag(), &p.query_roots); - let (logical_option, logical_shared) = classify(&source.dag, &source.query_roots); - assert_eq!(option, logical_option, "{} vs {}", p.id, source.id); - if !p.candidate.materializes_any() { - assert_eq!(shared, logical_shared, "{} vs {}", p.id, source.id); - } - } -} - -/// Exact TopK is implemented as a sort followed by a limit. -#[test] -fn stage2_physical_asap_optimization_exact_topk_is_sort_then_limit() { - let run = pipeline(); - let p = physical_with(&run, Q2Option::Exact); - let sort_then_limit = p.dag().edges.iter().any(|e| { - relational(p.dag().payload(e.producer)).as_deref() == Some("sort") - && relational(p.dag().payload(e.consumer)).as_deref() == Some("limit") - }); - assert!(sort_then_limit, "{}: no sort → limit", p.id); -} - -/// Exact Q2 sorts each job's series by the ranked value, descending, and a -/// `Project` over the limit restores the logical `[job, topk_10]` columns, -/// then keeps each chosen series' identity (its full label set). -#[test] -fn stage2_physical_asap_optimization_exact_topk_returns_job_and_topk_10() { - let run = pipeline(); - let p = physical_with(&run, Q2Option::Exact); - let QueryRoot::Operator(root) = &p.candidate.roots[1] else { - panic!("{}: Q2 is an operator root", p.id) - }; - let names: Vec<_> = root.schema.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!( - names, - ["job", "topk_10", "$promql_series_identity"], - "{}", - p.id - ); - let NonASAPOp::Project { child: limit, .. } = root.expect_non_asap() else { - panic!("{}: Q2's root is a Project", p.id) - }; - let NonASAPOp::Limit { - n: Some(10), - child: sort, - .. - } = limit.expect_non_asap() - else { - panic!("{}: a limit 10 under the Project", p.id) - }; - let NonASAPOp::Sort { - keys, - partition_by, - child, - } = sort.expect_non_asap() - else { - panic!("{}: a sort under the limit", p.id) - }; - let [SortKey { - expr: ScalarExpr::Column(ranked), - ascending: false, - .. - }] = keys.as_slice() - else { - panic!("{}: one descending column key: {keys:?}", p.id) - }; - assert_eq!(child.schema.fields[*ranked].name, "value", "{}", p.id); - assert_eq!(partition_by.len(), 1, "{}: partitioned by job", p.id); -} - -/// A summary Q2 over its whole window is a build node feeding a top-10 -/// estimation node, with no merge. -#[test] -fn stage2_physical_asap_optimization_summary_topk_is_build_then_estimate() { - let run = pipeline(); - for p in &run.physical { - if classify(p.dag(), &p.query_roots).0 == Q2Option::Exact || !whole_windows(p.dag()) { - continue; - } - assert!( - !p.dag().nodes.iter().any(|n| is_merge(&n.payload)), - "{}: no window summaries, so no merge", - p.id - ); - let build_to_estimate = p.dag().edges.iter().any(|e| { - matches!( - p.dag().payload(e.producer), - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) && matches!( - p.dag().payload(e.consumer), - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - ) - }); - assert!(build_to_estimate, "{}: no build → estimate", p.id); - } -} - -/// Only tumbling windows can be maintained at ingestion time: the -/// whole-window builds' fixed interval has no phase, and no summary is read -/// by both queries. Q1's rate tumbling windows are counter states, which need each -/// series' samples in time order; continuously ingesting PromQL data is -/// taken to arrive in that order, so they are maintained too. Each tumbling window -/// chain is maintained or not independently: a logical candidate with `c` -/// chains has `2^c` physical candidates. A shared input (the highest node -/// both queries read: the scan, or the time range above it) is also built -/// once per evaluation (`MatAtQuery`), which adds one candidate: the one -/// with every chain at query time, since a chain maintained at ingestion -/// time leaves one query reading the input at query time. -#[test] -fn stage2_physical_asap_optimization_materializes_only_tumbling_windows_at_ingestion() { - let run = pipeline(); - let mut counter_maintained = 0; - let mut shared_once = 0; - for p in &run.physical { - for n in &p.dag().nodes { - match materialization(p, n.id) { - MaterializationOption::NotMaterialized => {} - MaterializationOption::MatAtIngestion => { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { family, .. }) = - p.dag().payload(n.id) - else { - panic!("{}: {:?} is not a tumbling-window chain", p.id, n.id) - }; - if matches!( - family, - FieldDataType::ExactAggregate( - asap_types::ir::schema::ExactKind::Rate - | asap_types::ir::schema::ExactKind::Increase, - _ - ) - ) { - counter_maintained += 1; - } - } - MaterializationOption::MatAtQuery => { - let (q1, q2) = roots(&p.query_roots); - assert!( - closure(p.dag(), q1).contains(&n.id) - && closure(p.dag(), q2).contains(&n.id), - "{}: {:?} is built once for both queries", - p.id, - n.id - ); - assert!( - matches!( - relational(p.dag().payload(n.id)).as_deref(), - Some("scan" | "time_range") - ), - "{}: {:?} is the shared input", - p.id, - n.id - ); - shared_once += 1; - } - } - } - } - assert!( - counter_maintained > 0, - "Q1's rate tumbling windows are maintained" - ); - assert!( - shared_once > 0, - "explicit shared-source fixtures are materialized" - ); - let expected: usize = run - .logical - .iter() - .map(|c| { - let chains = c - .dag - .node_ids() - .into_iter() - .filter(|&id| { - matches!( - c.dag.payload(id), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count(); - (1 << chains) + usize::from(c.shared_input()) - }) - .sum(); - assert_eq!(run.physical.len(), expected); -} - -/// Compile `p` in the physical planner (the runtime capability check), with -/// its materialized views as view slots and every `Scan` as raw data. -fn compile_in_runtime(p: &Physical) -> Result<(), String> { - use asap_executor::physical_planner::compile; - let mut dag = p.dag().clone(); - let views = dag - .set_retention(&Default::default(), &Default::default()) - .map_err(|e| format!("{}: {e}", p.id))?; - let roots: Vec = dag.roots.iter().map(|&id| id as u64).collect(); - compile(&dag, &views, &roots) - .map(|_| ()) - .map_err(|e| format!("{}: {e}", p.id)) -} - -// ── Plan selection ─────────────────────────────────────────────────────── - -/// Plan selection selects one candidate and gives every other one a reason. -#[test] -fn stage3_plan_selection_selects_one_and_explains_the_rest() { - assert_selects_one_and_explains_the_rest(&pipeline()); -} - -/// The exact Q2 is selected. Both heaps are invalid because a top-k output -/// has no guarantee and the deployment has no evidence for it. Count-Min's -/// weights are raw data samples of `http_requests_total`, which the workload -/// declares a counter, so they are non-negative. -#[test] -fn stage3_plan_selection_selects_exact_q2_and_rejects_both_heaps() { - let run = pipeline(); - let selected = run.physical(&run.selection.selected); - assert_eq!( - classify(selected.dag(), &selected.query_roots).0, - Q2Option::Exact - ); - let invalid = run.invalid(); - // Only heap candidates are invalid (with or without a shared input). - for id in invalid.keys() { - let p = run.physical(id); - assert_ne!( - classify(p.dag(), &p.query_roots).0, - Q2Option::Exact, - "{invalid:?}" - ); - } - let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; - assert!( - invalid[cms.as_str()].contains("no guarantee"), - "{invalid:?}" - ); - let count_sketch = &physical_with(&run, Q2Option::CountSketchHeapPerJob).id; - assert!( - invalid[count_sketch.as_str()].contains("no guarantee"), - "{invalid:?}" - ); -} - -/// Without the counter declaration, Count-Min's weights are not proven -/// non-negative: plan selection reads no type from the `_total` name. -#[test] -fn stage3_plan_selection_count_min_needs_the_counter_declaration() { - let mut workload = example1(); - workload - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut(EXAMPLE1_METRIC) - .unwrap() - .metric_type = None; - let run = rule_fixture(&workload); - let invalid = run.invalid(); - let cms = &physical_with(&run, Q2Option::CountMinHeapPerJob).id; - assert!( - invalid[cms.as_str()].contains("non-negative"), - "{invalid:?}" - ); -} - -/// A shared NotMaterialized input is charged once per consumer, and both -/// queries read the same 1 m window, so the shared-input candidates cost -/// what the independent one does; their totals differ from it only by -/// floating-point rounding (the same charges summed in another grouping). -/// Built once per evaluation (`MatAtQuery`), the shared input is read once: -/// the shared time range (the doc's "Raw with a shared input") is selected, -/// at 4.38 instead of 7.01 per second. -#[test] -fn stage3_plan_selection_shared_input_ties_up_to_rounding_unless_built_once() { - let run = pipeline(); - let independent = physical_with(&run, Q2Option::Exact); - let base = run.cost(&independent.id).unwrap(); - let exact_shared = |p: &&Physical| { - whole_windows(p.dag()) && classify(p.dag(), &p.query_roots) == (Q2Option::Exact, true) - }; - let built_once = |p: &Physical| { - p.dag() - .nodes - .iter() - .any(|n| materialization(p, n.id) == MaterializationOption::MatAtQuery) - }; - let shared: Vec<_> = run - .physical - .iter() - .filter(exact_shared) - .filter(|p| !built_once(p)) - .map(|p| (p.id.clone(), run.cost(&p.id).unwrap())) - .collect(); - assert!(!shared.is_empty()); - assert!( - shared - .iter() - .any(|(_, total)| total.to_bits() != base.to_bits()), - "the rounding difference the tie tolerance absorbs: {base} vs {shared:?}" - ); - for (id, total) in &shared { - assert!( - (total - base).abs() <= 1e-9 * base, - "{id}: {total} vs {base}" - ); - } - let selected = run.physical(&run.selection.selected); - assert!(exact_shared(&selected) && built_once(selected)); - let time_range = selected - .dag() - .nodes - .iter() - .find(|n| materialization(selected, n.id) == MaterializationOption::MatAtQuery) - .unwrap(); - assert_eq!( - relational(&time_range.payload).as_deref(), - Some("time_range") - ); - let cost = run.cost(&selected.id).unwrap(); - assert!( - (cost - 4.3770).abs() < 1e-3 && (base - 7.0125).abs() < 1e-3, - "{cost} vs {base}" - ); -} - -/// Tumbling windows rebuilt at every evaluation cost more than the whole -/// window: the tumbling windows read the same rows and update the same states, and -/// merging them adds work. Heap tumbling windows are invalid like their whole-window -/// builds (and their merged top-k is only approximate). -#[test] -fn stage3_plan_selection_tumbling_windows_at_query_time_cost_more() { - let run = pipeline(); - let selected = run.cost(&run.selection.selected).unwrap(); - let invalid = run.invalid(); - let mut compared = 0; - for p in run.physical.iter().filter(|p| !whole_windows(p.dag())) { - match run.cost(&p.id) { - Some(cost) => { - assert!(cost > selected * (1.0 + 1e-9), "{}: {cost}", p.id); - compared += 1; - } - None => assert_ne!( - classify(p.dag(), &p.query_roots).0, - Q2Option::Exact, - "{}: {invalid:?}", - p.id - ), - } - } - assert!(compared > 0); -} - -/// The selected plan is the cheapest valid candidate for the whole workload. -#[test] -fn stage3_plan_selection_selects_cheapest_valid() { - assert_selects_cheapest_valid(&pipeline()); -} - -/// Every node of a valid candidate is charged once per consumer that reads -/// it: a shared `NotMaterialized` node is recomputed for each query, and a -/// materialized one is computed once. -#[test] -fn stage3_plan_selection_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&pipeline()); -} - -/// A hand-built candidate with a materialized node: Q1's plan read by two -/// queries, its `time_range` kept `MatAtQuery`. Plan selection charges the -/// two accumulators (rate, sum) and their finalizations twice, once per query; the walks from -/// both roots stop at the materialized `time_range`, which is computed once -/// and reads the scan once. -fn materialized_case() -> Run { - let mut workload = example1(); - let repeating = workload.query_workload.repeating_queries.as_mut().unwrap(); - repeating.truncate(1); - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let exact = compose_logical_candidate(&local, &vec![0; local.targets.len()]).unwrap(); - let run = run_explicit_candidates(&workload, roots, vec![exact], PlanningModels::builtin()); - let mut p = run.physical[0].clone(); - p.candidate.roots = vec![p.candidate.roots[0].clone(); 2]; - let dag = &mut p.candidate.dag; - dag.roots = vec![dag.roots[0]; 2]; - p.query_roots = dag.roots.clone(); - let time_range = dag - .nodes - .iter_mut() - .find(|n| relational(&n.payload).as_deref() == Some("time_range")) - .unwrap(); - time_range.materialization = Materialization::MatAtQuery(Medium::Memory); - let selection = select_plan( - std::slice::from_ref(&p.candidate), - &[exact_every_10s(), exact_every_10s()], - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .unwrap(); - Run { - physical: vec![p], - selection, - ..run - } -} - -/// The charging helper agrees with plan selection when a node is -/// materialized. -#[test] -fn stage3_plan_selection_charging_helper_follows_plan_selection_with_a_materialized_node() { - let run = materialized_case(); - let p = &run.physical[0]; - let mut by_kind: Vec<_> = run.selection.costs[&p.id] - .per_node - .iter() - .map(|(id, c)| (operation(p.dag().payload(*id)), c.multiplier)) - .collect(); - by_kind.sort(); - let expected = [ - ("finalize_exact_accumulator", 2), - ("finalize_exact_accumulator", 2), - ("scan", 1), - ("summary_agg", 2), - ("summary_agg", 2), - ("time_range", 1), - ]; - assert_eq!( - by_kind, - expected.map(|(kind, times)| (kind.to_string(), times)) - ); - assert_each_node_charged_per_consuming_query(&run); -} - -/// Charging the scan under a materialized node once per query is caught. -#[test] -#[should_panic(expected = "multiplier of")] -fn stage3_plan_selection_charging_helper_rejects_recounting_below_a_materialized_node() { - let mut run = materialized_case(); - let p = run.physical[0].clone(); - let scan = p - .dag() - .nodes - .iter() - .find(|n| relational(&n.payload).as_deref() == Some("scan")) - .unwrap() - .id; - run.selection - .costs - .get_mut(&p.id) - .unwrap() - .per_node - .get_mut(&scan) - .unwrap() - .multiplier = 2; - assert_each_node_charged_per_consuming_query(&run); -} - -/// The plan selection picks compiles in the physical planner (added by the -/// implementer, not part of the spec). -#[test] -fn stage3_plan_selection_selected_plan_compiles_in_the_physical_planner() { - let run = pipeline(); - compile_in_runtime(run.physical(&run.selection.selected)).unwrap(); -} - -/// Sharing the input never costs more than reading it separately. -#[test] -fn stage3_plan_selection_shared_input_is_not_costlier() { - let run = pipeline(); - let by_combo: BTreeMap<_, _> = run - .physical - .iter() - .filter(|p| whole_windows(p.dag())) - .filter_map(|p| Some((classify(p.dag(), &p.query_roots), run.cost(&p.id)?))) - .collect(); - let mut compared = 0; - for option in PASS1_Q2_OPTIONS { - if let (Some(shared), Some(separate)) = ( - by_combo.get(&(option, true)), - by_combo.get(&(option, false)), - ) { - assert!(shared <= separate, "{option:?}"); - compared += 1; - } - } - assert!(compared > 0); -} diff --git a/crates/integration-tests/tests/planner_layering_example2.rs b/crates/integration-tests/tests/planner_layering_example2.rs deleted file mode 100644 index 9c09da9fd..000000000 --- a/crates/integration-tests/tests/planner_layering_example2.rs +++ /dev/null @@ -1,1164 +0,0 @@ -//! Rule-fixture and production acceptance for Example 2: shared summaries across statistics. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::physical_common; -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::ir::properties::ProbabilityExpr; -use asap_types::ir::properties::{AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::DataType; -use asap_types::ir::schema::{Field, Schema, SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::ir::QueryRoot; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, - SourceFacts, SqlDialect, -}; -use planner_layering_common::*; - -// ── Example 2 workload ─────────────────────────────────────────────────── - -const SQL_Q1: &str = - "SELECT COUNT(DISTINCT src_ip) FROM flows WHERE ts >= now() - INTERVAL '1 minute'"; -const SQL_Q2: &str = "SELECT -SUM(p * LN(p)) FROM (\ - SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows \ - WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; -/// Q3 casts the counts to DOUBLE: the design's Int64 `SUM(c * c)` can -/// overflow, so the SQL frontend recognizes only the Float64 product. -const SQL_Q3: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) FROM (\ - SELECT src_ip, COUNT(*) AS c FROM flows \ - WHERE ts >= now() - INTERVAL '1 minute' GROUP BY src_ip)"; - -/// The metric the PromQL stand-in reads. -const STANDIN_METRIC: &str = "flows_src_ip"; -/// PromQL stand-in for Q1–Q3 (see the module docs). -const PROMQL_Q1: &str = "distinct_over_time(flows_src_ip[1m])"; -const PROMQL_Q2: &str = "entropy_over_time(flows_src_ip[1m])"; -const PROMQL_Q3: &str = "l2_over_time(flows_src_ip[1m])"; - -/// (ε, δ) of Q1, Q2, Q3. Q3 is the strictest. -const ACCURACY: [(f64, f64); 3] = [(0.02, 0.01), (0.05, 0.01), (0.01, 0.01)]; - -fn panel(query: &str, (epsilon, delta): (f64, f64), scope: AccuracyScope) -> RepeatingEntry { - RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget { - accuracy_scope: scope, - ..AccuracyTarget::max_error(epsilon).failure_probability(delta) - }, - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - } -} - -/// The shared data workload with 10,000,000 distinct source IPs, in the -/// SQL `flows` table and the PromQL stand-in's `flows_src_ip` metric -/// (sampled every 15 s; SQL needs no sample interval). -fn data_workload() -> DataWorkload { - let ten_million = SourceFacts { - series_count: Some(10_000_000), - ..SourceFacts::default() - }; - let sampled = SourceFacts { - sample_interval: Some(DurationMs(15_000)), - ..ten_million.clone() - }; - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000_000.0 / 15.0, - }, - sources: [ - ("flows".to_string(), ten_million), - (STANDIN_METRIC.to_string(), sampled), - ] - .into(), - } -} - -/// The SQL queries' targets cover their whole result, as the doc's do. The -/// PromQL stand-in states them per value: it reads one series, but the -/// series count of `flows_src_ip` (the 10M source IPs) is what plan -/// selection takes as the number of values a PromQL vector returns, so a -/// whole-result target would be checked over 10M values (#669b). -fn workload(language: QueryLanguage, queries: [&str; 3]) -> PlanningWorkload { - let sql = matches!(language, QueryLanguage::SQL(_)); - let scope = if sql { - AccuracyScope::WholeResult - } else { - AccuracyScope::PerValue - }; - PlanningWorkload { - query_workload: QueryWorkload { - language, - query_batch: None, - repeating_queries: Some( - queries - .iter() - .zip(ACCURACY) - .map(|(q, accuracy)| panel(q, accuracy, scope)) - .collect(), - ), - }, - data_workload: Some(data_workload()), - } -} - -/// Example 2 as the doc writes it. -fn sql_workload() -> PlanningWorkload { - workload( - QueryLanguage::SQL(SqlDialect::DataFusionSQL), - [SQL_Q1, SQL_Q2, SQL_Q3], - ) -} - -fn standin_workload() -> PlanningWorkload { - workload(QueryLanguage::PromQL, [PROMQL_Q1, PROMQL_Q2, PROMQL_Q3]) -} - -/// `flows(ts, src_ip)`: the doc names only these two columns. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "flows", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("src_ip", DataType::Utf8, false), - ], - 0, - vec![], - ), - ) -} - -async fn lower_sql(workload: &PlanningWorkload) -> Vec { - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -// Targeted rule fixtures: exact, each local summary in isolation, and every -// query on its own UnivMon, needed by compatibility and cost assertions. -// Shared UnivMons come from the memo (`shared_pipeline`). -// Production optimization is tested separately; this is not a Cartesian search. -fn rule_fixture(workload: &PlanningWorkload, roots: Vec) -> Run { - let computations = rule_fixture_computations(workload, &roots); - run_explicit_candidates( - workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`rule_fixture`]'s plans, before Stage 2 and plan selection. -fn rule_fixture_computations( - workload: &PlanningWorkload, - roots: &[QueryRoot], -) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(roots), - ) - .unwrap(); - let exact = vec![0; local.targets.len()]; - let mut choices = BTreeSet::from([exact.clone()]); - let mut univmon = exact.clone(); - let mut shared_targets = Vec::new(); - for (target, options) in local.targets.iter().enumerate() { - for option in 1..options.choice_count() { - let mut choice = exact.clone(); - choice[target] = option; - choices.insert(choice); - } - if let Some(option) = (0..options.choice_count()).find(|&choice| { - matches!( - options.computation_for(choice), - Some(( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::UnivMon), - _ - )) - ) - }) { - univmon[target] = option; - shared_targets.push(target); - } - } - choices.insert(univmon.clone()); - for &third in &shared_targets { - for option in 0..local.targets[third].choice_count() { - let mut choice = univmon.clone(); - choice[third] = option; - choices.insert(choice); - } - } - let grid: Vec<_> = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect(); - let mut computations = Vec::new(); - for choice in choices { - if local.absorbed_choice_varies(&choice) { - continue; - } - let whole: Vec<_> = compose_logical_candidate(&local, &choice).unwrap(); - computations.push(whole.clone()); - let shared = share_identical(&whole); - if distinct_nodes(&shared) < distinct_nodes(&whole) { - computations.push(shared); - } - let tumbling_windows = tumbling_window_candidates(&whole, &grid); - // Isolate one tumbling-window configuration at a time; no Cartesian products of - // independent window implementations are needed for the error tests. - for tumbling_window in tumbling_windows { - computations.extend( - window_composition_pass2_candidates(&whole, &[tumbling_window]) - .unwrap() - .into_iter() - .map(|candidate| candidate.roots.unwrap()), - ); - } - } - computations -} - -#[test] -fn production_memo_selects_a_validated_example2_plan() { - let workload = standin_workload(); - let run = run_stages(&workload, lower_promql(&workload)); - assert_selects_cheapest_valid(&run); - assert_eq!(run.selection.accepted[&run.selection.selected].len(), 3); -} - -fn pipeline() -> Run { - let workload = standin_workload(); - rule_fixture(&workload, lower_promql(&workload)) -} - -/// The stand-in with the deployment's bound on distinct items per -/// evaluation of its one source, when given. -fn standin_with_distinct_bound(bound: Option) -> PlanningWorkload { - let mut workload = standin_workload(); - if let Some(bound) = bound { - workload - .data_workload - .as_mut() - .expect("data workload") - .sources - .get_mut(STANDIN_METRIC) - .expect("the stand-in's source") - .max_distinct_per_evaluation = Some(bound); - } - workload -} - -fn pipeline_with_distinct_bound(bound: Option) -> Run { - let workload = standin_with_distinct_bound(bound); - rule_fixture(&workload, lower_promql(&workload)) -} - -/// Plans composed of the memo's candidates, priced by Stage 2 and plan -/// selection: every query on its group's first candidate; each query alone -/// on its own UnivMon; and, per UnivMon state group the memo shares between -/// queries, those queries reading it. -fn memo_pipeline() -> Run { - let workload = standin_workload(); - let roots = lower_promql(&workload); - let computations = memo_computations(&workload, &roots); - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`rule_fixture`]'s plans and [`memo_pipeline`]'s together, so the -/// memo's shared UnivMons compete with every query's own options. -fn shared_pipeline(workload: &PlanningWorkload, roots: Vec) -> Run { - let mut computations = rule_fixture_computations(workload, &roots); - computations.extend(memo_computations(workload, &roots)); - run_explicit_candidates( - workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -/// [`shared_pipeline`] over the stand-in. -fn standin_shared_pipeline() -> Run { - let workload = standin_workload(); - shared_pipeline(&workload, lower_promql(&workload)) -} - -/// [`memo_pipeline`]'s plans of `roots`, before Stage 2 and plan selection. -fn memo_computations(workload: &PlanningWorkload, roots: &[QueryRoot]) -> Vec> { - use asap_integration_tests::planning::{compose_memo_plan, expanded_memo, own_choices}; - use asap_types::ir::schema::FieldDataType; - use asap_types::ir::{ASAPOp, OperatorNode}; - use asap_types::stages::{CandidateForMemoGroup, OriginOfCandidateForMemoGroup}; - let entries: Vec<_> = workload.query_workload.entries().collect(); - let memo = expanded_memo(roots.to_vec(), &entries); - let queries = roots.len(); - let is_univmon = |node: &std::rc::Rc| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::UnivMon) - }; - // Query `q`'s topmost own group's first candidate `keep` accepts. - let own = |q: usize, keep: &dyn Fn(&CandidateForMemoGroup) -> bool| { - own_choices(&memo, q, keep).into_iter().next() - }; - let mut choices = vec![BTreeMap::new()]; - for q in 0..queries { - if let Some(choice) = own(q, &|c| { - c.origin == OriginOfCandidateForMemoGroup::Pass1 - && OperatorNode::reachable(&c.root_node).iter().any(is_univmon) - }) { - choices.push(BTreeMap::from([choice])); - } - } - for (state, group) in memo.groups.iter().enumerate() { - if !is_univmon(&group.logical_node) || group.reached_by_queries().len() < 2 { - continue; - } - let reading: Option> = group - .reached_by_queries() - .iter() - .map(|&q| { - own(q, &|c| { - c.input_groups.iter().any(|child| child.0 as usize == state) - }) - }) - .collect(); - choices.extend(reading); - } - choices - .iter() - .map(|choice| { - compose_memo_plan(&memo, queries, choice) - .into_iter() - .map(QueryRoot::Operator) - .collect() - }) - .collect() -} - -// ── Helpers ────────────────────────────────────────────────────────────── - -/// UnivMon build nodes of `c` with the queries that read each. -fn univmons(c: &Logical) -> Vec<(BTreeSet, SketchParams, BTreeSet)> { - sketch_builds(&c.dag) - .into_iter() - .filter(|(_, algorithm, _)| *algorithm == SketchAlgorithm::UnivMon) - .map(|(id, _, params)| { - let statistics = estimates_of(&c.dag, id) - .into_iter() - .map(|(_, s)| format!("{s:?}")) - .collect(); - (readers(&c.dag, &c.query_roots, id), params, statistics) - }) - .collect() -} - -/// The UnivMon read by every query in `queries` and by no other. -fn shared_univmon(c: &Logical, queries: &[usize]) -> Option { - let want: BTreeSet<_> = queries.iter().copied().collect(); - univmons(c) - .into_iter() - .find(|(readers, ..)| *readers == want) - .map(|(_, params, _)| params) -} - -/// Whether some summary build is read by more than one query. -fn shares_a_summary(c: &Logical) -> bool { - sketch_builds(&c.dag) - .iter() - .any(|(id, ..)| readers(&c.dag, &c.query_roots, *id).len() > 1) -} - -fn options(run: &Run, query: usize) -> BTreeSet { - run.logical - .iter() - .filter(|c| !shares_a_summary(c)) - .map(|c| query_option(&c.dag, &c.query_roots, query)) - .collect() -} - -/// UnivMon state size: heap entries plus counters over every layer. -fn footprint(params: &SketchParams) -> u64 { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - panic!("UnivMon params: {params:?}"); - }; - u64::from(*heap_size) + u64::from(*sketch_rows) * u64::from(*sketch_cols) * u64::from(*layers) -} - -/// Q`query`'s own UnivMon parameters, from an independent candidate. -fn own_univmon(run: &Run, query: usize) -> SketchParams { - run.logical - .iter() - .filter(|c| !shares_a_summary(c)) - .find_map(|c| shared_univmon(c, &[query])) - .expect("Pass 1 offers UnivMon") -} - -// ── Workload and query-language frontends ───────────────────────────────────────────────── - -/// The encoded SQL workload is valid and normalizes to Q1, Q2, Q3. -#[test] -fn workload_encodes_example2() { - let workload = sql_workload(); - workload.validate().expect("valid workload"); - let queries: Vec<_> = workload - .query_workload - .entries() - .map(|e| e.query.0) - .collect(); - assert_eq!(queries, [SQL_Q1, SQL_Q2, SQL_Q3]); - standin_workload().validate().expect("valid stand-in"); -} - -/// The SQL frontend lowers Q1 to a distinct count over `flows`. -#[tokio::test] -async fn stage0_query_language_frontends_sql_q1_lowers_to_a_distinct_count() { - let roots = lower_sql(&sql_workload()).await; - let ops = frontend_operations(&roots[0]); - assert!(ops.contains(&"scan".to_string()), "{ops:?}"); - assert!( - ops.contains(&"aggregate:cardinality".to_string()), - "{ops:?}" - ); -} - -/// The SQL frontend recognizes Q2 as `Entropy(src_ip)` and Q3 as `L2(src_ip)`. -#[tokio::test] -async fn stage0_query_language_frontends_sql_q2_q3_lower_to_entropy_and_l2() { - let roots = lower_sql(&sql_workload()).await; - assert!(frontend_operations(&roots[1]).contains(&"aggregate:frequency_entropy".to_string())); - assert!(frontend_operations(&roots[2]).contains(&"aggregate:frequency_l2".to_string())); -} - -/// Once the SQL forms are recognized, the SQL workload has the stand-in's Pass 1 options. -#[tokio::test] -async fn stage1_logical_asap_optimization_sql_has_the_standin_options() { - let workload = sql_workload(); - let sql = rule_fixture(&workload, lower_sql(&workload).await); - let standin = pipeline(); - for q in 0..3 { - assert_eq!(options(&sql, q), options(&standin, q), "Q{}", q + 1); - } -} - -/// The doc's SQL workload itself gets the memo's shared candidate: one -/// UnivMon read by Q1, Q2 and Q3. -#[tokio::test] -async fn stage1_logical_asap_optimization_sql_adds_one_univmon_for_all_three() { - let workload = sql_workload(); - let run = shared_pipeline(&workload, lower_sql(&workload).await); - assert!( - run.logical.iter().any(|c| { - let ums = univmons(c); - ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) - }), - "no SQL candidate reads one UnivMon for Q1, Q2 and Q3" - ); -} - -/// The stand-in lowers to the three statistics, each over a 1-min range of one input. -#[test] -fn stage0_query_language_frontends_standin_lowers_to_three_statistics_of_one_input() { - let roots = lower_promql(&standin_workload()); - let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); - for (ops, statistic) in ops.iter().zip([ - "aggregate:cardinality", - "aggregate:frequency_entropy", - "aggregate:frequency_l2", - ]) { - let mut want = vec!["scan".to_string(), statistic.into(), "time_range".into()]; - want.sort(); - assert_eq!(ops, &want); - } -} - -/// The frontends give one summary-free workload DAG whose queries share no node. -#[test] -fn stage0_query_language_frontends_one_summary_free_dag_without_sharing() { - let run = pipeline(); - let stage0_query_language_frontends = &run.stage0_query_language_frontends; - assert_eq!(stage0_query_language_frontends.query_roots.len(), 3); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); - assert!(cross_query_nodes( - &stage0_query_language_frontends.dag, - &stage0_query_language_frontends.query_roots - ) - .is_empty()); -} - -// ── Logical ASAP-aware optimization ────────────────────────────────────── - -/// Pass 1 offers an exact and a UnivMon option for each of the three statistics. -#[test] -fn stage1_logical_asap_optimization_pass1_offers_exact_and_univmon_for_each_statistic() { - let run = pipeline(); - for q in 0..3 { - let found = options(&run, q); - assert!(found.contains("exact"), "Q{}: {found:?}", q + 1); - assert!(found.contains("UnivMon"), "Q{}: {found:?}", q + 1); - } -} - -/// Pass 1 offers a specialized distinct-count summary for Q1. -#[test] -fn stage1_logical_asap_optimization_pass1_offers_a_distinct_count_summary() { - let found = options(&pipeline(), 0); - assert!( - found.iter().any(|o| ["Hll", "Kmv"].contains(&o.as_str())), - "{found:?}" - ); -} - -/// Pass 1 offers a specialized entropy summary for Q2 and a norm summary for Q3. -#[test] -#[ignore = "needs Pass 1 specialized entropy and L2 summary families (#444)"] -fn stage1_logical_asap_optimization_pass1_offers_specialized_entropy_and_l2_summaries() { - let run = pipeline(); - for q in [1, 2] { - let found = options(&run, q); - assert!( - found.iter().any(|o| o != "exact" && o != "UnivMon"), - "Q{}: {found:?}", - q + 1 - ); - } -} - -/// The summary-capability rule adds one UnivMon build feeding a distinct-count, an entropy and an L2 estimate. -#[test] -fn stage1_logical_asap_optimization_summary_capability_adds_one_univmon_for_all_three() { - let run = memo_pipeline(); - let shared = run.logical.iter().find(|c| { - let ums = univmons(c); - ums.len() == 1 && ums[0].0 == BTreeSet::from([0, 1, 2]) - }); - let c = shared.expect("a candidate with one UnivMon read by Q1, Q2 and Q3"); - let statistics = &univmons(c)[0].2; - for s in [ - SketchStatistic::Cardinality, - SketchStatistic::FrequencyEntropy, - SketchStatistic::FrequencyL2, - ] { - assert!(statistics.contains(&format!("{s:?}")), "{statistics:?}"); - } -} - -/// A UnivMon shared by several statistics is sized for every consumer: -/// each consumer is sized alone, and the shared build takes each -/// parameter's largest value among them. -#[test] -fn stage1_logical_asap_optimization_shared_univmon_takes_each_parameters_largest_own_size() { - let run = memo_pipeline(); - let own = |q| match own_univmon(&run, q) { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - .. - } => [heap_size, sketch_rows, sketch_cols, u32::from(layers)], - other => panic!("{other:?}"), - }; - let mut seen = 0; - for queries in [vec![0, 1, 2], vec![0, 1], vec![0, 2], vec![1, 2]] { - let mut want = [0u32; 4]; - for &q in &queries { - for (w, p) in want.iter_mut().zip(own(q)) { - *w = (*w).max(p); - } - } - let want = SketchParams::UnivMon { - heap_size: want[0], - sketch_rows: want[1], - sketch_cols: want[2], - layers: want[3] as u8, - // Ignored by equality. - distinct_failure_probability: None, - }; - for c in &run.logical { - if let Some(params) = shared_univmon(c, &queries) { - seen += 1; - assert_eq!(params, want, "{} {queries:?}", c.id); - } - } - } - assert!(seen > 0, "no shared UnivMon"); -} - -/// Pass 1 sizes UnivMon for its query's target: ε = 0.01 (Q3) needs a larger state than ε = 0.05 (Q2). -#[test] -fn stage1_logical_asap_optimization_univmon_is_sized_per_accuracy_target() { - let run = pipeline(); - assert!(footprint(&own_univmon(&run, 2)) > footprint(&own_univmon(&run, 1))); -} - -/// Only a UnivMon, never a specialized summary, is shared across the statistics. -#[test] -fn stage1_logical_asap_optimization_only_univmon_is_shared_across_statistics() { - let run = standin_shared_pipeline(); - for c in &run.logical { - for (id, algorithm, _) in sketch_builds(&c.dag) { - if readers(&c.dag, &c.query_roots, id).len() > 1 { - assert_eq!(algorithm, SketchAlgorithm::UnivMon, "{}", c.id); - } - } - } -} - -/// Logical candidates are valid DAGs with unique ids. -#[test] -fn stage1_logical_asap_optimization_candidates_are_valid_and_uniquely_named() { - assert_valid_and_uniquely_named(&pipeline()); -} - -// ── Physical ASAP-aware optimization ───────────────────────────────────── - -/// No candidate is discarded before plan selection: each logical candidate keeps one physical candidate with nothing materialized, plus one per other materialization assignment. -#[test] -fn stage2_physical_asap_optimization_keeps_every_logical_candidate() { - assert_one_unmaterialized_physical_candidate_per_logical(&pipeline()); -} - -/// Physical ASAP-aware optimization keeps the shared UnivMon as one build node read by all three queries. -#[test] -fn stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build() { - let run = memo_pipeline(); - let mut seen = 0; - for c in run - .logical - .iter() - .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) - { - for p in run.physical_of(c) { - seen += 1; - let builds: Vec<_> = sketch_builds(p.dag()) - .into_iter() - .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) - .collect(); - assert_eq!(builds.len(), 1, "{}", p.id); - assert_eq!(readers(p.dag(), &p.query_roots, builds[0].0).len(), 3); - } - } - assert!(seen > 0); -} - -// ── Plan selection ─────────────────────────────────────────────────────── - -/// Plan selection selects one candidate and gives every other one a reason. -#[test] -fn stage3_plan_selection_selects_one_and_explains_the_rest() { - assert_selects_one_and_explains_the_rest(&pipeline()); -} - -/// The selected plan is the cheapest valid candidate for the whole workload. -#[test] -fn stage3_plan_selection_selects_cheapest_valid() { - assert_selects_cheapest_valid(&pipeline()); -} - -/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. -#[test] -fn stage3_plan_selection_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&pipeline()); -} - -/// `asap_sketchlib` has no Theta sketch, so no candidate builds one. -#[test] -fn stage3_plan_selection_has_no_theta_candidate() { - let run = pipeline(); - for (id, cost) in &run.selection.costs { - assert!( - !cost - .per_node - .values() - .any(|n| n.detail.starts_with("build Theta")), - "{id} builds Theta" - ); - } -} - -/// A UnivMon estimate has a guarantee, so plan selection judges it against its target instead of rejecting it for having none. -#[test] -#[ignore = "needs a UnivMon entropy accuracy model (#788); L2 and the distinct count have one"] -fn stage3_plan_selection_judges_univmon_with_an_accuracy_model() { - let run = pipeline(); - let invalid = run.invalid(); - for c in &run.logical { - for p in run.physical_of(c) { - let Some(reason) = invalid.get(p.id.as_str()) else { - continue; - }; - for q in 0..3 { - if query_option(&c.dag, &c.query_roots, q) == "UnivMon" { - let unguaranteed = format!("q{}: the output has no guarantee", q + 1); - assert!(!reason.starts_with(&unguaranteed), "{}: {reason}", p.id); - } - } - } - } -} - -/// The physical candidates that answer Q1 with `algorithm` and Q2 and Q3 -/// exactly: their validity depends only on Q1's estimate. -fn q1_only_candidates<'a>(run: &'a Run, algorithm: &str) -> Vec<&'a str> { - let ids: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq([algorithm, "exact", "exact"]) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) - .collect(); - assert!( - !ids.is_empty(), - "a candidate answers Q1 with {algorithm} alone" - ); - ids -} - -fn hll_only_candidates(run: &Run) -> Vec<&str> { - q1_only_candidates(run, "Hll") -} - -/// With the deployment's bound on distinct items per evaluation, plan -/// selection binds classic HLL's failure probability and accepts Q1's HLL -/// estimate; without a bound it stays unknown and the target unmet. Pass 1 -/// sizes HLL from ε = 0.02 and δ = 0.01 (#694): `m ≥ 1/(2ε²δ)` gives -/// precision 17, which certifies δ = 0.01 at ε = 0.02 for bounds up to -/// 1,842 distinct items. -#[test] -fn stage3_plan_selection_accepts_hll_given_a_distinct_bound() { - for bound in [9, 1_842] { - let bounded = pipeline_with_distinct_bound(Some(bound)); - for id in hll_only_candidates(&bounded) { - assert!( - bounded.cost(id).is_some(), - "bound {bound}: {id}: {:?}", - bounded.invalid().get(id) - ); - } - } - let unbounded = pipeline_with_distinct_bound(None); - let invalid = unbounded.invalid(); - for id in hll_only_candidates(&unbounded) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - reason.contains("failure probability None"), - "{id}: {reason}" - ); - } -} - -/// A bound too large for the precision sized for ε alone (17 certifies at -/// most 1,842 distinct items) is met by a larger precision: Pass 1's sizing -/// for the failure probability raises HLL's precision until its failure -/// probability over the bound at ε = 0.02 meets the target, up to precision -/// 18 (#669b), which certifies up to 3,791; Stage 3 only checks it. Past -/// that, up to Example 2's 10M source IPs, classic HLL's linear-counting -/// contract certifies nothing. -#[test] -fn pass1_sizing_enlarges_hll_for_a_larger_distinct_bound() { - let run = pipeline_with_distinct_bound(Some(3_000)); - for id in hll_only_candidates(&run) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } - for bound in [10_000, 10_000_000] { - let run = pipeline_with_distinct_bound(Some(bound)); - let invalid = run.invalid(); - for id in hll_only_candidates(&run) { - let reason = invalid - .get(id) - .unwrap_or_else(|| panic!("bound {bound}: {id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - !reason.contains("failure probability None"), - "{id}: {reason}" - ); - } - } -} - -/// KMV certifies Q1 over the 10M source IPs without a distinct bound -/// (#694): its bound holds for any number of distinct items, and Pass 1 -/// sizes `k` for the requested δ (#700), so a stricter δ = 10⁻⁴ is met -/// too, by a larger `k`. -#[test] -fn stage3_plan_selection_accepts_kmv_for_any_failure_probability_without_a_bound() { - let mut ks = Vec::new(); - for delta in [0.01, 1e-4] { - let mut workload = standin_workload(); - let q1 = &mut workload - .query_workload - .repeating_queries - .as_mut() - .expect("repeating queries")[0]; - q1.requirements.accuracy = AccuracyTarget { - accuracy_scope: AccuracyScope::PerValue, - ..AccuracyTarget::max_error(0.02).failure_probability(delta) - }; - let run = rule_fixture(&workload, lower_promql(&workload)); - for id in q1_only_candidates(&run, "Kmv") { - assert!( - run.cost(id).is_some(), - "δ = {delta}: {id}: {:?}", - run.invalid().get(id) - ); - let k: BTreeSet<_> = sketch_builds(run.physical(id).dag()) - .into_iter() - .filter_map(|(_, _, params)| match params { - SketchParams::Kmv { k, .. } => Some(k), - _ => None, - }) - .collect(); - ks.push((delta, k)); - } - } - assert!( - ks.iter().any(|(d, k)| *d == 0.01 && k.contains(&26_656)), - "{ks:?}" - ); - assert!( - ks.iter() - .any(|(d, k)| *d == 1e-4 && k.iter().all(|k| *k > 26_656)), - "{ks:?}" - ); -} - -/// The physical candidates among `ids` that build their sketch as a -/// tumbling-window `SummaryTumblingWindowChain`. -fn tumbling<'a>(run: &Run, ids: Vec<&'a str>) -> Vec<&'a str> { - ids.into_iter() - .filter(|id| { - let dag = run.physical(id).dag(); - sketch_builds(dag).iter().any(|(build, _, _)| { - matches!(window_form(dag, *build), WindowForm::Tumbling { .. }) - }) - }) - .collect() -} - -/// The enlarged precision reaches a sketch carried by a tumbling-window -/// `SummaryTumblingWindowChain` too (#713): Q1's HLL over a bound of 3,000 is valid only at -/// precision 18, above the 17 sized for ε alone, so a valid tumbling-window -/// HLL candidate carries precision 18 in its chain. -#[test] -fn pass1_sizing_enlarges_tumbling_window_hll() { - let run = pipeline_with_distinct_bound(Some(3_000)); - let hll = tumbling(&run, hll_only_candidates(&run)); - assert!( - !hll.is_empty(), - "a tumbling-window HLL candidate answers Q1" - ); - for id in hll { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// Stage 1 does not read the distinct bound: its candidates are the same -/// with and without one, and Q1's HLL estimate leaves its failure -/// probability symbolic in the source's bound. -#[test] -fn stage1_logical_asap_optimization_leaves_the_hll_failure_probability_symbolic() { - let bounded = pipeline_with_distinct_bound(Some(9)); - let unbounded = pipeline_with_distinct_bound(None); - let stage1 = |run: &Run| { - run.logical - .iter() - .map(|c| format!("{} {:?}", c.id, c.roots)) - .collect::>() - }; - assert_eq!(stage1(&bounded), stage1(&unbounded)); - let mut seen = 0; - for c in unbounded - .logical - .iter() - .filter(|c| !shares_a_summary(c) && query_option(&c.dag, &c.query_roots, 0) == "Hll") - { - let QueryRoot::Operator(q1) = &c.roots[0] else { - panic!("operator root") - }; - let guarantee = q1.guarantee.as_ref().expect("Q1 has a guarantee"); - assert_eq!( - guarantee.spec.failure_probability, - ProbabilityExpr::Unknown { - statistic: - "classic_hll_failure_probability[p=17,eps=0.02](max_distinct[flows_src_ip])" - .into() - }, - "{}", - c.id - ); - seen += 1; - } - assert!(seen > 0, "a candidate answers Q1 with HLL"); -} - -/// The physical candidates that answer Q`query` with UnivMon and the other -/// two exactly: their validity depends only on that UnivMon estimate. -fn univmon_only_candidates(run: &Run, query: usize) -> Vec<&str> { - let ids: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3).all(|q| { - query_option(&c.dag, &c.query_roots, q) - == if q == query { "UnivMon" } else { "exact" } - }) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.id.as_str())) - .collect(); - assert!( - !ids.is_empty(), - "a candidate answers Q{} with UnivMon alone", - query + 1 - ); - ids -} - -/// Q1's UnivMon is sized for (0.02, 0.01): a 337,500-key heap on 19 -/// layers. Its distinct count is certified once the deployment bounds the -/// source's distinct items, both at 9 (the top layer is always complete) -/// and at 10M (7 layers suffice, where classic HLL certifies nothing); -/// without a bound its failure probability stays unknown. -#[test] -fn stage3_plan_selection_accepts_univmon_distinct_given_a_distinct_bound() { - for bound in [9, 10_000_000] { - let run = pipeline_with_distinct_bound(Some(bound)); - for id in univmon_only_candidates(&run, 0) { - assert!( - run.cost(id).is_some(), - "{bound} {id}: {:?}", - run.invalid().get(id) - ); - } - } - let unbounded = pipeline_with_distinct_bound(None); - let invalid = unbounded.invalid(); - for id in univmon_only_candidates(&unbounded, 0) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - assert!( - reason.contains("failure probability None"), - "{id}: {reason}" - ); - } -} - -/// A distinct bound past what 19 layers cover for Q1's heap -/// (337,500 · 2^17) certifies nothing. -#[test] -fn stage3_plan_selection_rejects_univmon_distinct_past_its_layers() { - let run = pipeline_with_distinct_bound(Some(337_500 << 17 | 1)); - let invalid = run.invalid(); - for id in univmon_only_candidates(&run, 0) { - let reason = invalid.get(id).unwrap_or_else(|| panic!("{id} is invalid")); - assert!(reason.starts_with("q1:"), "{id}: {reason}"); - } -} - -/// Q3's L2 at (0.01, 0.01) is certified from layer 0 alone: a 5 × 56,817 -/// Count Sketch on one layer, with no data-workload statistic. -#[test] -fn stage3_plan_selection_accepts_univmon_l2() { - let run = pipeline(); - assert_eq!( - own_univmon(&run, 2), - SketchParams::UnivMon { - heap_size: 1, - sketch_rows: 5, - sketch_cols: 56_817, - layers: 1, - distinct_failure_probability: None, - } - ); - for id in univmon_only_candidates(&run, 2) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// SQL Q3's L2 through UnivMon keeps its guarantee: the exact population -/// count joined beside it (a cross join, no key) and the `CASE` deciding the -/// empty-input NULL on that exact count keep the estimate's guarantee, so -/// plan selection accepts it. -#[tokio::test] -async fn stage3_plan_selection_accepts_sql_univmon_l2_beside_its_exact_population_count() { - let workload = sql_workload(); - let run = rule_fixture(&workload, lower_sql(&workload).await); - for id in univmon_only_candidates(&run, 2) { - assert!(run.cost(id).is_some(), "{id}: {:?}", run.invalid().get(id)); - } -} - -/// One materialized UnivMon for all three costs no more than three separate UnivMons. -#[test] -#[ignore = "needs a UnivMon entropy accuracy model (#788): without one, the shared UnivMon candidate is left unpriced"] -fn stage3_plan_selection_shared_univmon_costs_no_more_than_three() { - let run = standin_shared_pipeline(); - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let separate: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3).all(|q| query_option(&c.dag, &c.query_roots, q) == "UnivMon") - }) - .map(|c| cost(c).unwrap_or_else(|| panic!("{} is not priced", c.id))) - .collect(); - assert!(!separate.is_empty(), "independent all-UnivMon candidate"); - let mut compared = 0; - for shared in run - .logical - .iter() - .filter(|c| shared_univmon(c, &[0, 1, 2]).is_some()) - { - let shared_cost = cost(shared).unwrap_or_else(|| panic!("{} is not priced", shared.id)); - for separate_cost in &separate { - assert!( - shared_cost <= *separate_cost, - "{shared_cost} vs {separate_cost}" - ); - } - compared += 1; - } - assert!(compared > 0); -} - -/// The selected plan updates at most one UnivMon per flow record: separate UnivMons are dominated by the shared one. -#[test] -fn stage3_plan_selection_selected_plan_has_at_most_one_univmon() { - let run = standin_shared_pipeline(); - let selected = run.physical(&run.selection.selected); - let count = sketch_builds(selected.dag()) - .iter() - .filter(|(_, a, _)| *a == SketchAlgorithm::UnivMon) - .count(); - assert!(count <= 1, "{}: {count} UnivMons", selected.id); -} - -/// Stage 3 over the candidates that answer Q1 with KMV and Q2 and Q3 -/// exactly selects one, and the executor runs its Q1: the distinct count of -/// the window's sample values. Below KMV's `k` distinct values its estimate -/// is exact. Only Q1 is compiled: the exact entropy and L2 of Q2 and Q3 do -/// not compile yet ("per-entity aggregate requires an explicit entity -/// binding"). -#[test] -fn stage3_plan_selection_selected_kmv_plan_executes_end_to_end() { - use asap_devtools::memo_pricing::reference_select as select_plan; - use asap_executor::physical_planner::{compile, promql_rows}; - use asap_executor::runtime::Scope; - use asap_executor::values::Value; - use asap_planner::PlanningModels; - - let workload = standin_workload(); - let roots = lower_promql(&workload); - let run = rule_fixture(&workload, roots.clone()); - let kmv: Vec<_> = run - .logical - .iter() - .filter(|c| { - !shares_a_summary(c) - && (0..3) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .eq(["Kmv", "exact", "exact"]) - }) - .flat_map(|c| run.physical_of(c).map(|p| p.candidate.clone())) - .collect(); - assert!(!kmv.is_empty(), "a candidate answers Q1 with KMV"); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let selection = select_plan( - &kmv, - &entries, - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .expect("Stage 3 selects a KMV plan"); - let p = run.physical(&selection.selected); - assert_eq!(query_option(p.dag(), &p.query_roots, 0), "Kmv"); - - // 1,000 flow records over 50 s with 300 distinct source IPs. - let distinct = 300u32; - let samples: Vec<(i64, f64)> = (0..1_000u32) - .map(|i| (i64::from(i) * 50, f64::from(i * 7 % distinct))) - .collect(); - let labels = BTreeMap::from([("__name__".to_string(), "flows_src_ip".to_string())]); - // The whole plan runs at query time over the samples bound at its `Scan`. - let dag = p.dag(); - let store = physical_common::raw_data_store(dag, |_, schema| { - samples - .iter() - .map(|(ts, v)| promql_rows::series_row(schema, &labels, *ts, *v).unwrap()) - .collect() - }); - let q1 = p.query_roots[0] as u64; - let plan = compile(dag, &[], &[q1]).unwrap_or_else(|e| panic!("{}: {e}", p.id)); - let result = physical_common::execute( - &plan, - store, - Scope::Query { - evaluation_time_ms: 55_000, - }, - ); - let values: Vec = result[0] - .iter() - .flat_map(|batch| batch.to_rows()) - .flatten() - .filter_map(|value| match value { - Value::Float64(v) => Some(v), - _ => None, - }) - .collect(); - assert_eq!(values, [f64::from(distinct)], "{}", p.id); -} diff --git a/crates/integration-tests/tests/planner_layering_example3.rs b/crates/integration-tests/tests/planner_layering_example3.rs deleted file mode 100644 index c0c8ab6ed..000000000 --- a/crates/integration-tests/tests/planner_layering_example3.rs +++ /dev/null @@ -1,635 +0,0 @@ -//! Rule-fixture and production acceptance for Example 3: window implementations. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::BTreeSet; - -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; -use asap_types::workload::DataArrival; - -use planner_layering_common::*; - -// Fixed algorithm/window fixtures keep numerical evaluator checks independent -// of a time-bounded production search's archived incumbents. -fn run_a() -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ); - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = Vec::new(); - for kll in [false, true] { - let choices: Vec<_> = local - .targets - .iter() - .map(|target| { - if kll { - (0..target.choice_count()) - .find(|&choice| { - matches!( - target.computation_for(choice), - Some(( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - _ - )) - ) - }) - .unwrap() - } else { - 0 - } - }) - .collect(); - let whole = unshare(&compose_logical_candidate(&local, &choices).unwrap()); - computations.push(whole.clone()); - computations.push(share_identical(&whole)); - } - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -fn run_b() -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = pattern_b(); - let roots = lower_promql(&workload); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut computations = vec![roots.clone()]; - for option in 0..local.targets[0].choice_count() { - if !matches!( - local.targets[0].computation_for(option), - Some((HowToComputeAggregateMeasure::Summary(_), _)) - ) { - continue; - } - let whole: Vec<_> = compose_logical_candidate(&local, &[option]).unwrap(); - let tumbling_windows = tumbling_window_candidates(&whole, &[Some(PATTERN_B_INTERVAL_MS)]); - let chain = window_composition_pass2_candidates(&whole, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - computations.extend([whole, chain]); - } - run_explicit_candidates( - &workload, - roots, - computations, - asap_planner::PlanningModels::builtin(), - ) -} - -#[test] -fn production_memo_covers_pattern_b_with_validated_accuracy() { - let workload = pattern_b(); - let run = run_stages_under(&workload, lower_promql(&workload), &Default::default()); - assert_selects_cheapest_valid(&run); - assert_eq!(run.selection.accepted[&run.selection.selected].len(), 1); -} - -/// The per-computation options of `query` across all candidates. -fn options(run: &Run, query: usize) -> BTreeSet { - run.logical - .iter() - .map(|c| query_option(&c.dag, &c.query_roots, query)) - .collect() -} - -fn op(name: &str) -> String { - name.to_string() -} - -/// The summary builds of `c`: (build, algorithm, window form, readers). -fn windowed_builds(c: &Logical) -> Vec<(NodeId, SketchAlgorithm, WindowForm, BTreeSet)> { - sketch_builds(&c.dag) - .into_iter() - .map(|(id, algorithm, _)| { - let form = window_form(&c.dag, id); - (id, algorithm, form, readers(&c.dag, &c.query_roots, id)) - }) - .collect() -} - -// ── Workloads ──────────────────────────────────────────────────────────── - -/// Pattern A is one ad hoc batch of five queries run once at T; Pattern B one repeating query. -#[test] -fn workload_encodes_example3() { - let a = pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ); - a.validate().expect("valid Pattern A"); - let queries: Vec<_> = a.query_workload.entries().map(|e| e.query.0).collect(); - assert_eq!(queries, PATTERN_A.map(|pattern| pattern.promql)); - let b = pattern_b(); - b.validate().expect("valid Pattern B"); - assert_eq!(evaluation_interval_ms(&b, 0), Some(PATTERN_B_INTERVAL_MS)); -} - -// ── Pattern A: query-language frontends ─────────────────────────────── - -/// Each Pattern A query lowers to scan → (time shift) → range → quantile. -#[test] -fn stage0_query_language_frontends_a_lowers_each_query_to_its_interval() { - let roots = lower_promql(&pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - )); - let ops: Vec<_> = roots.iter().map(frontend_operations).collect(); - let plain = vec![op("aggregate:quantile"), op("scan"), op("time_range")]; - let shifted = vec![ - op("aggregate:quantile"), - op("scan"), - op("time_range"), - op("time_shift"), - ]; - assert_eq!( - ops, - [ - plain.clone(), - plain, - shifted.clone(), - shifted.clone(), - shifted - ] - ); -} - -/// The frontends give one summary-free DAG whose five queries share no node. -#[test] -fn stage0_query_language_frontends_a_one_summary_free_dag_without_sharing() { - let run = run_a(); - let stage0_query_language_frontends = &run.stage0_query_language_frontends; - assert_eq!(stage0_query_language_frontends.query_roots.len(), 5); - assert!(stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); - assert!(cross_query_nodes( - &stage0_query_language_frontends.dag, - &stage0_query_language_frontends.query_roots - ) - .is_empty()); -} - -// ── Pattern A: logical ASAP-aware optimization ─────────────────────────── - -/// Pass 1 offers each query an exact and a KLL candidate over its own interval. -#[test] -fn stage1_logical_asap_optimization_a_pass1_offers_exact_and_kll_per_query() { - let run = run_a(); - for q in 0..5 { - let found = options(&run, q); - assert!( - found.contains("exact") && found.contains("Kll"), - "q{}: {found:?}", - q + 1 - ); - } -} - -/// The five independent KLLs: one per query, each read by its query only, all sized for ε = 0.005. -#[test] -fn stage1_logical_asap_optimization_a_keeps_five_independent_klls() { - let run = run_a(); - let all_kll = run - .logical - .iter() - .find(|c| { - !c.shared_input() && (0..5).all(|q| query_option(&c.dag, &c.query_roots, q) == "Kll") - }) - .expect("independent all-KLL candidate"); - let builds = sketch_builds(&all_kll.dag); - assert_eq!(builds.len(), 5); - let readers: BTreeSet<_> = builds - .iter() - .map(|(id, ..)| readers(&all_kll.dag, &all_kll.query_roots, *id)) - .collect(); - assert_eq!(readers, (0..5).map(|q| BTreeSet::from([q])).collect()); - let sizes: BTreeSet<_> = builds - .iter() - .map(|(_, _, params)| match params { - SketchParams::Kll { k, .. } => *k, - other => panic!("{other:?}"), - }) - .collect(); - assert_eq!(sizes.len(), 1, "one ε, one KLL size: {sizes:?}"); -} - -/// The identical-expression rule shares only raw input (scan, range, shift), never a quantile or summary, and keeps the unshared variant. -#[test] -fn stage1_logical_asap_optimization_a_identical_expression_rule_shares_only_raw_input() { - let run = run_a(); - assert!(run.logical.iter().any(|c| c.shared_input())); - assert!(run.logical.iter().any(|c| !c.shared_input())); - for c in &run.logical { - for id in cross_query_nodes(&c.dag, &c.query_roots) { - let kind = relational(c.dag.payload(id)); - assert!( - matches!(kind.as_deref(), Some("scan" | "time_range" | "time_shift")), - "{}: shared {kind:?}", - c.id - ); - } - } -} - -/// One Exponential Histogram of KLLs over [T − 5y, T] serves all five queries, each through its own merge and p99 estimate. -#[test] -#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] -fn stage1_logical_asap_optimization_a_window_composition_adds_one_eh_for_all_five() { - let run = run_a(); - let found = run.logical.iter().find(|c| { - windowed_builds(c).iter().any(|(_, a, form, readers)| { - *a == SketchAlgorithm::Kll - && matches!(form, WindowForm::ExponentialHistogram { horizon_ms } if *horizon_ms >= 5 * YEAR_MS) - && readers.len() == 5 - }) - }); - let c = found.expect("a candidate with one EH of KLLs read by all five queries"); - let (eh, ..) = windowed_builds(c) - .into_iter() - .find(|(_, _, form, _)| matches!(form, WindowForm::ExponentialHistogram { .. })) - .unwrap(); - let estimates = estimates_of(&c.dag, eh); - assert_eq!(estimates.len(), 5, "one estimate per query"); - for (estimate, statistic) in estimates { - assert_eq!(statistic, SketchStatistic::Quantile { q: 0.99 }); - let merged = c - .dag - .producers(estimate) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))); - assert!( - merged, - "{}: estimate {estimate:?} does not read a merge", - c.id - ); - } -} - -/// The shared EH candidate is added next to the independent KLL candidates, not instead of them. -#[test] -#[ignore = "needs Pass 2 window composition across queries: one window summary for all five (#628, #634)"] -fn stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries() { - let run = run_a(); - let shared = run.logical.iter().any(|c| { - windowed_builds(c).iter().any(|(_, _, f, r)| { - matches!(f, WindowForm::ExponentialHistogram { .. }) && r.len() == 5 - }) - }); - let independent = run.logical.iter().any(|c| { - let builds = windowed_builds(c); - builds.len() == 5 - && builds.iter().all(|(_, a, f, r)| { - *a == SketchAlgorithm::Kll && *f == WindowForm::None && r.len() == 1 - }) - }); - assert!(shared && independent); -} - -/// Pass 2 adds a candidate for each way of grouping two queries onto one shared window summary. -#[test] -#[ignore = "needs Pass 2 window composition across queries: partial groupings (#628, #634)"] -fn stage1_logical_asap_optimization_a_window_composition_groups_every_pair() { - let run = run_a(); - let groups: BTreeSet<_> = run - .logical - .iter() - .flat_map(windowed_builds) - .filter(|(_, _, f, r)| *f != WindowForm::None && r.len() > 1) - .map(|(.., r)| r) - .collect(); - for a in 0..5 { - for b in a + 1..5 { - assert!( - groups.contains(&BTreeSet::from([a, b])), - "pair q{} q{}", - a + 1, - b + 1 - ); - } - } -} - -// ── Pattern A: plan selection ──────────────────────────────────────────── - -/// Plan selection selects one Pattern A candidate, explains the rest, and picks the cheapest valid one. -#[test] -fn stage3_plan_selection_a_selects_cheapest_valid() { - let run = run_a(); - assert_selects_one_and_explains_the_rest(&run); - assert_selects_cheapest_valid(&run); -} - -/// Every node of a valid candidate is charged once per query that reads it: nothing is materialized, so a shared node is recomputed for each. -#[test] -fn stage3_plan_selection_a_charges_each_node_per_consuming_query() { - assert_each_node_charged_per_consuming_query(&run_a()); -} - -/// Sharing the scan never costs more than scanning separately, for the same per-computation choices. -#[test] -#[ignore = "needs plan selection to charge a shared scan that is not materialized per query by the \ - window each query reads, not by the longest window it serves (#632)"] -fn stage3_plan_selection_a_shared_scan_is_not_costlier() { - let run = run_a(); - let key = |c: &Logical| { - (0..5) - .map(|q| query_option(&c.dag, &c.query_roots, q)) - .collect::>() - }; - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let mut compared = 0; - for shared in run.logical.iter().filter(|c| c.shared_input()) { - let separate = run - .logical - .iter() - .find(|c| !c.shared_input() && key(c) == key(shared)) - .expect("independent counterpart"); - if let (Some(s), Some(i)) = (cost(shared), cost(separate)) { - assert!(s <= i, "{} {s} vs {} {i}", shared.id, separate.id); - compared += 1; - } - } - assert!(compared > 0); -} - -// ── Pattern B: query-language frontends and Pass 1 ──────────────────── - -/// Pattern B lowers to scan → range 5m → quantile, with no summary. -#[test] -fn stage0_query_language_frontends_b_lowers_to_one_range_quantile() { - let run = run_b(); - let roots = lower_promql(&pattern_b()); - assert_eq!( - frontend_operations(&roots[0]), - [op("aggregate:quantile"), op("scan"), op("time_range")] - ); - assert!(run - .stage0_query_language_frontends - .dag - .nodes - .iter() - .all(|n| !is_summary(&n.operator))); -} - -/// Pass 1 offers an exact and a KLL candidate for the 5-min window. -#[test] -fn stage1_logical_asap_optimization_b_pass1_offers_exact_and_kll() { - let found = options(&run_b(), 0); - assert!( - found.contains("exact") && found.contains("Kll"), - "{found:?}" - ); -} - -// ── Pattern B: window composition ──────────────────────────────────────── - -/// Each Pass 1 option gets three window forms: none, a 5-min sliding window with a 1-min slide, and 1-min tumbling windows. -#[test] -#[ignore = "needs Pass 2 window composition: sliding windows (#522); tumbling windows pass in \ - stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option"] -fn stage1_logical_asap_optimization_b_window_composition_adds_sliding_and_tumbling_per_option() { - let run = run_b(); - let summaries: BTreeSet<_> = options(&run, 0) - .into_iter() - .filter(|o| o != "exact") - .collect(); - let found: BTreeSet<_> = run - .logical - .iter() - .flat_map(windowed_builds) - .map(|(_, a, f, _)| (format!("{a:?}"), f)) - .collect(); - let forms = [ - WindowForm::None, - WindowForm::Sliding { - length_ms: PATTERN_B_WINDOW_MS, - slide_ms: PATTERN_B_INTERVAL_MS, - }, - WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }, - ]; - for option in &summaries { - for form in forms { - assert!(found.contains(&(option.clone(), form)), "{option} {form:?}"); - } - } -} - -/// Each summary option of Pass 1 (KLL and DDSketch, both mergeable) also -/// comes as 1-min tumbling windows, gcd(5 min, 1 min), merged five at a time -/// before its p99 estimate; the whole-window builds are kept and read no -/// merge. Exact has no summary to split: 2 options × 2 forms + exact = 5. -#[test] -fn stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option() { - let run = run_b(); - assert_eq!(run.logical.len(), 5); - let mut found = BTreeSet::new(); - for c in &run.logical { - for (build, algorithm, form, _) in windowed_builds(c) { - let merged = estimates_of(&c.dag, build).iter().all(|(e, _)| { - c.dag - .producers(*e) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))) - }); - assert_eq!(merged, form != WindowForm::None, "{}: {form:?}", c.id); - if let asap_types::ir::Operator::ASAP( - asap_types::ir::ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }, - ) = c.dag.payload(build) - { - assert_eq!(*tumbling_window_count, 5, "{}", c.id); - } - found.insert((format!("{algorithm:?}"), form)); - } - } - let tumbling = WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }; - let expected: BTreeSet<_> = ["DDSketch", "Kll"] - .into_iter() - .flat_map(|a| [(a.to_string(), WindowForm::None), (a.to_string(), tumbling)]) - .collect(); - assert_eq!(found, expected); -} - -/// Window parameters are legal: L divides W, s divides L and the evaluation interval, a tumbling length divides both. -#[test] -fn stage1_logical_asap_optimization_b_window_parameters_are_legal() { - let tumbling = run_b() - .logical - .iter() - .flat_map(windowed_builds) - .filter(|(_, _, form, _)| matches!(form, WindowForm::Tumbling { .. })) - .count(); - assert!(tumbling > 0, "not vacuous"); - let (w, every) = (PATTERN_B_WINDOW_MS, PATTERN_B_INTERVAL_MS); - for c in &run_b().logical { - for (_, _, form, _) in windowed_builds(c) { - match form { - WindowForm::Sliding { - length_ms, - slide_ms, - } => { - assert_eq!(w % length_ms, 0, "{}: L | W", c.id); - assert_eq!(length_ms % slide_ms, 0, "{}: s | L", c.id); - assert_eq!(every % slide_ms, 0, "{}: s | interval", c.id); - } - WindowForm::Tumbling { length_ms } => { - assert_eq!(w % length_ms, 0, "{}: length | W", c.id); - assert_eq!(every % length_ms, 0, "{}: length | interval", c.id); - } - WindowForm::ExponentialHistogram { .. } => { - panic!("{}: Pattern B has no EH", c.id) - } - WindowForm::None => {} - } - } - } -} - -/// A sliding window with L = W reads one completed window (no merge); tumbling windows merge W / length of them. -#[test] -#[ignore = "needs Pass 2 window composition: sliding windows (#522); tumbling merges pass in \ - stage1_logical_asap_optimization_b_window_composition_adds_tumbling_per_mergeable_option"] -fn stage1_logical_asap_optimization_b_merge_only_where_the_window_form_needs_it() { - let run = run_b(); - let mut seen = BTreeSet::new(); - for c in &run.logical { - for (build, _, form, _) in windowed_builds(c) { - let merged = estimates_of(&c.dag, build).iter().any(|(e, _)| { - c.dag - .producers(*e) - .into_iter() - .any(|p| is_merge(c.dag.payload(p))) - }); - assert_eq!( - merged, - needs_merge(form, PATTERN_B_WINDOW_MS), - "{}: {form:?}", - c.id - ); - if form != WindowForm::None { - seen.insert(merged); - } - } - } - assert_eq!( - seen, - BTreeSet::from([false, true]), - "both a sliding and a tumbling candidate" - ); -} - -/// A window form that merges is used only with a summary whose states merge (KLL, DDSketch). -#[test] -fn stage1_logical_asap_optimization_window_merges_use_mergeable_summaries() { - for (run, window) in [(run_a(), 5 * YEAR_MS), (run_b(), PATTERN_B_WINDOW_MS)] { - for c in &run.logical { - for (_, algorithm, form, _) in windowed_builds(c) { - if needs_merge(form, window) { - assert!( - matches!(algorithm, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch), - "{}: {algorithm:?} {form:?}", - c.id - ); - } - } - } - } -} - -// ── Pattern B: plan selection ──────────────────────────────────────────── - -/// Plan selection selects one Pattern B candidate, explains the rest, and picks the cheapest valid one. -#[test] -fn stage3_plan_selection_b_selects_cheapest_valid() { - let run = run_b(); - assert_selects_one_and_explains_the_rest(&run); - assert_selects_cheapest_valid(&run); - assert_each_node_charged_per_consuming_query(&run); -} - -/// With nothing materialized, the 1-min KLL tumbling windows are all rebuilt at every -/// evaluation: they read the same 5 minutes of samples and update the same -/// states as one whole-window KLL, and merging five tumbling windows per series adds -/// 1M × 5 operations a minute. Each evaluation also holds its states while -/// it runs: the whole window's 1M KLLs, or the five tumbling windows' 5M smaller ones, -/// for the evaluation's latency. So whole-window KLL is selected at 2.363 per -/// second and the tumbling windows cost 3.611; exact costs 2.458. DDSketch, whole or in -/// tumbling windows, misses the rank target. -#[test] -fn stage3_plan_selection_b_tumbling_windows_rebuilt_at_query_time_add_the_merge() { - let run = run_b(); - let cost = |c: &Logical| run.cost(&run.physical_of(c).next().unwrap().id); - let with = |algorithm: &str, tumbling_windows: bool| { - run.logical - .iter() - .find(|c| { - query_option(&c.dag, &c.query_roots, 0) == algorithm - && windowed_builds(c) - .iter() - .any(|(_, _, f, _)| (*f != WindowForm::None) == tumbling_windows) - }) - .unwrap_or_else(|| panic!("{algorithm} tumbling_windows={tumbling_windows}")) - }; - let whole = cost(with("Kll", false)).unwrap(); - let tumbling_windows = cost(with("Kll", true)).unwrap(); - assert!((whole - 2.362_500_025_2).abs() < 1e-9, "{whole}"); - assert!( - (tumbling_windows - 3.611_233_350_8).abs() < 1e-9, - "{tumbling_windows}" - ); - assert_eq!( - run.selection.selected, - run.physical_of(with("Kll", false)).next().unwrap().id - ); - for tumbling_windows in [false, true] { - assert_eq!( - cost(with("DDSketch", tumbling_windows)), - None, - "tumbling_windows={tumbling_windows}" - ); - } -} diff --git a/crates/integration-tests/tests/planner_layering_example4.rs b/crates/integration-tests/tests/planner_layering_example4.rs deleted file mode 100644 index a47c5fbfa..000000000 --- a/crates/integration-tests/tests/planner_layering_example4.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! Rule-fixture and production acceptance for Example 4: materialization and objective models. -//! -//! Numerical, accuracy, and operator assertions use explicitly supplied -//! configurations of the existing rules and physical evaluator. They do not -//! assume a bounded MILP run archives every supported workload plan. Separate -//! production checks exercise memo search and validated output selection. -//! Unsupported algorithms retain their existing ignored tests. - -use crate::planner_layering_common; - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_cost::{ClusterLimits, ClusterObjective, CostInputs, CostModel, ResourceUnitCost}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_types::ir::flat::NodeId; -use asap_types::ir::schema::SketchAlgorithm; -use asap_types::workload::{DataArrival, PlanningWorkload}; -use planner_layering_common::*; - -/// Pattern A as given: one ad hoc batch at T over mixed data. -fn once() -> PlanningWorkload { - pattern_a( - PatternARecurrence::OnceAdHoc, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ) -} - -fn monthly() -> PlanningWorkload { - pattern_a( - PatternARecurrence::MonthlyPredictable, - DataArrival::Mixed { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - ) -} - -fn at_rest() -> PlanningWorkload { - pattern_a(PatternARecurrence::OnceAdHoc, DataArrival::AtRest) -} - -/// The window-summary build of `c` with form `want` (`Some(form)` matches -/// exactly; `None` matches any EH) read by `readers` queries. -fn window_build(c: &Logical, want: Option, readers_count: usize) -> Option { - sketch_builds(&c.dag) - .into_iter() - .find_map(|(id, algorithm, _)| { - let form = window_form(&c.dag, id); - let matches = match want { - Some(want) => form == want, - None => matches!(form, WindowForm::ExponentialHistogram { .. }), - }; - (algorithm == SketchAlgorithm::Kll - && matches - && readers(&c.dag, &c.query_roots, id).len() == readers_count) - .then_some(id) - }) -} - -/// The physical candidates of the logical candidate with a KLL window -/// summary of `form` read by `readers_count` queries, keyed by the -/// materialization of that build. Several logical candidates may match (for -/// example with and without the shared input); the first is used. -fn options_of( - run: &Run, - form: Option, - readers_count: usize, -) -> BTreeMap { - let logical = run - .logical - .iter() - .find(|c| window_build(c, form, readers_count).is_some()) - .unwrap_or_else(|| { - let form = form.map_or("EH".to_string(), |f| format!("{f:?}")); - panic!("no logical candidate with a {form} KLL window summary") - }); - let build = window_build(logical, form, readers_count).unwrap(); - // Physical ASAP-aware optimization renumbers nodes; find the build again - // in each physical DAG. - run.physical_of(logical) - .map(|p| { - let id = sketch_builds(p.dag()) - .into_iter() - .find(|(id, a, _)| { - *a == SketchAlgorithm::Kll - && window_form(p.dag(), *id) == window_form(&logical.dag, build) - }) - .map(|(id, ..)| id) - .expect("the window summary survives physical ASAP-aware optimization"); - (materialization(p, id), (p.id.clone(), id)) - }) - .collect() -} - -fn eh_options(run: &Run) -> BTreeMap { - options_of(run, None, 5) -} - -fn tumbling() -> Option { - Some(WindowForm::Tumbling { - length_ms: PATTERN_B_INTERVAL_MS, - }) -} - -fn sliding() -> Option { - Some(WindowForm::Sliding { - length_ms: PATTERN_B_WINDOW_MS, - slide_ms: PATTERN_B_INTERVAL_MS, - }) -} - -fn cost(run: &Run, id: &str) -> f64 { - run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) -} - -use MaterializationOption::{MatAtIngestion, MatAtQuery, NotMaterialized}; - -// ── Constraints over every candidate ───────────────────────────────────── - -/// Every node upstream of an ingestion-time node also runs at ingestion time, in every workload variant. -/// The monthly variant runs in -/// `stage2_physical_asap_optimization_monthly_ingestion_time_upstream_is_ingestion_time`. -#[test] -fn stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time() { - for workload in [once(), at_rest(), pattern_b()] { - assert_ingestion_upstream_is_ingestion(&run_promql(&workload)); - } -} - -/// The monthly variant of -/// `stage2_physical_asap_optimization_ingestion_time_upstream_is_ingestion_time`: -/// it plans 67,228 candidates, too slow for a debug build in CI. -#[test] -#[ignore = "slow: run in release by hand"] -fn stage2_physical_asap_optimization_monthly_ingestion_time_upstream_is_ingestion_time() { - assert_ingestion_upstream_is_ingestion(&run_promql(&monthly())); -} - -/// Data at rest has no ingestion, so no candidate runs anything at ingestion time. -#[test] -fn stage2_physical_asap_optimization_at_rest_runs_nothing_at_ingestion_time() { - let run = run_promql(&at_rest()); - for p in &run.physical { - for n in &p.dag().nodes { - assert!(!runs_at_ingestion(p, n.id), "{}: {:?}", p.id, n.id); - } - } -} - -/// Each Pattern A query's range window and offset, as its PromQL text states them. -fn pattern_a_windows_and_offsets() -> (Vec, Vec) { - let windows = PATTERN_A.iter().map(|p| p.window_ms).collect(); - let offsets = PATTERN_A.iter().map(|p| p.offset_before_t_ms).collect(); - (windows, offsets) -} - -/// Every materialized output of every candidate of `workload` is kept for at -/// least its longest reader's `windows[q] + offsets[q]`. -fn assert_materialized_outputs_cover_their_consumers( - workload: &PlanningWorkload, - windows: &[u64], - offsets: &[u64], -) { - let run = run_promql(workload); - for p in &run.physical { - for n in &p.dag().nodes { - // A shared scan or range built once per evaluation holds its - // rows within that evaluation; plan selection reads it over the - // longest window above it. - let shared_rows = materialization(p, n.id) == MatAtQuery - && matches!( - relational(&n.payload).as_deref(), - Some("scan" | "time_range" | "time_shift") - ); - if materialization(p, n.id) == NotMaterialized || shared_rows { - continue; - } - let need = readers(p.dag(), &p.query_roots, n.id) - .into_iter() - .map(|q| windows[q] + offsets[q]) - .max() - .unwrap_or(0); - let kept = retention_ms(p, n.id).unwrap_or_else(|| { - panic!("{}: {:?} is materialized without a retention", p.id, n.id) - }); - assert!(kept >= need, "{}: {:?} kept {kept} < {need}", p.id, n.id); - } - } -} - -/// A materialized output is kept for as long as its consumers read: it covers every reader's lookback and offset. -/// The monthly variant runs in -/// `stage2_physical_asap_optimization_monthly_materialized_output_covers_its_consumers`. -#[test] -fn stage2_physical_asap_optimization_materialized_output_covers_its_consumers() { - let (windows, offsets) = pattern_a_windows_and_offsets(); - for workload in [once(), at_rest()] { - assert_materialized_outputs_cover_their_consumers(&workload, &windows, &offsets); - } - assert_materialized_outputs_cover_their_consumers(&pattern_b(), &[PATTERN_B_WINDOW_MS], &[0]); -} - -/// The monthly variant of -/// `stage2_physical_asap_optimization_materialized_output_covers_its_consumers`: -/// it plans 67,228 candidates, too slow for a debug build in CI. -#[test] -#[ignore = "slow: run in release by hand"] -fn stage2_physical_asap_optimization_monthly_materialized_output_covers_its_consumers() { - let (windows, offsets) = pattern_a_windows_and_offsets(); - assert_materialized_outputs_cover_their_consumers(&monthly(), &windows, &offsets); -} - -// ── Pattern A ──────────────────────────────────────────────────────────── - -/// The shared EH candidate yields A1 (query time, kept), A2 (ingestion time) and A3 (not materialized). -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options() { - let found: BTreeSet<_> = eh_options(&run_promql(&once())).into_keys().collect(); - assert_eq!( - found, - BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) - ); -} - -/// With data at rest, A2 is not generated; A1 and A3 remain. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option() { - let found: BTreeSet<_> = eh_options(&run_promql(&at_rest())).into_keys().collect(); - assert_eq!(found, BTreeSet::from([MatAtQuery, NotMaterialized])); -} - -/// A materialized shared EH is one build node read by all five queries, charged once. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers() { - let run = run_promql(&once()); - for (m, (id, build)) in eh_options(&run) { - if m == NotMaterialized { - continue; - } - let p = run.physical(&id); - let ehs = sketch_builds(p.dag()) - .into_iter() - .filter(|(b, ..)| { - matches!( - window_form(p.dag(), *b), - WindowForm::ExponentialHistogram { .. } - ) - }) - .count(); - assert_eq!(ehs, 1, "{id}"); - assert_eq!(readers(p.dag(), &p.query_roots, build).len(), 5, "{id}"); - assert!( - run.selection.costs[&id].per_node.contains_key(&build), - "{id}" - ); - } -} - -/// A3 rebuilds the EH for each of the five queries (plan selection charges a -/// `NotMaterialized` node once per consuming query), so it costs more than A1, -/// which builds it once. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once() { - let run = run_promql(&once()); - let options = eh_options(&run); - let (a1, a3) = (&options[&MatAtQuery].0, &options[&NotMaterialized].0); - assert!( - cost(&run, a3) > cost(&run, a1), - "A3 {} vs A1 {}", - cost(&run, a3), - cost(&run, a1) - ); -} - -/// As given (run once, ad hoc), A1 is the cheapest of the three: A2 maintains years of history for one batch. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh() { - let run = run_promql(&once()); - let options = eh_options(&run); - let a1 = cost(&run, &options[&MatAtQuery].0); - for (m, (id, _)) in &options { - assert!(a1 <= cost(&run, id), "A1 {a1} vs {m:?} {}", cost(&run, id)); - } -} - -/// Repeated monthly and predictable, A2's maintenance is shared by many batches, so A2 gains on A1. -#[test] -#[ignore = "needs Pass 2 window segments shared across queries (#628, #634); Stage 2 materialization (MatAtQuery within one batch) is in place"] -fn stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance() { - let ratio = |workload: PlanningWorkload| { - let run = run_promql(&workload); - let options = eh_options(&run); - cost(&run, &options[&MatAtIngestion].0) / cost(&run, &options[&MatAtQuery].0) - }; - let (once, monthly) = (ratio(once()), ratio(monthly())); - assert!(monthly < once, "A2/A1: monthly {monthly} vs once {once}"); -} - -// ── Pattern B ──────────────────────────────────────────────────────────── - -/// A deployment that keeps query-time results across evaluations, which B3 -/// needs; the default deployment keeps none. -fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } -} - -/// The 1-min tumbling KLL candidate yields B1 (ingestion time), B2 (not materialized) and B3 (query time, kept). -#[test] -fn stage2_physical_asap_optimization_b_tumbling_kll_has_three_materialization_options() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &keeping_query_time_results(), - ); - let found: BTreeSet<_> = options_of(&run, tumbling(), 1).into_keys().collect(); - assert_eq!( - found, - BTreeSet::from([MatAtIngestion, MatAtQuery, NotMaterialized]) - ); -} - -/// B1 builds the tumbling KLLs at ingestion time and merges and estimates at query time. -#[test] -fn stage2_physical_asap_optimization_b_b1_builds_at_ingestion_and_merges_at_query_time() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ); - let (id, build) = options_of(&run, tumbling(), 1)[&MatAtIngestion].clone(); - let p = run.physical(&id); - assert!(runs_at_ingestion(p, build)); - let estimates = estimates_of(p.dag(), build); - assert!(!estimates.is_empty()); - for (estimate, _) in estimates { - assert!( - !runs_at_ingestion(p, estimate), - "{id}: estimate at ingestion" - ); - let merge = p - .dag() - .producers(estimate) - .into_iter() - .find(|&m| is_merge(p.dag().payload(m))); - let merge = merge.unwrap_or_else(|| panic!("{id}: no merge before the estimate")); - assert!(!runs_at_ingestion(p, merge), "{id}: merge at ingestion"); - } -} - -/// Implementer test: for the default deployment, which keeps no query-time -/// result across evaluations, the tumbling KLL candidate yields B1 -/// (maintained at ingestion time) and B2 (rebuilt at every evaluation), not -/// B3; both are priced, and B2's evaluation stays within the 200 ms bound. -#[test] -fn stage2_physical_asap_optimization_b_tumbling_kll_has_b1_and_b2() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ); - let options = options_of(&run, tumbling(), 1); - assert_eq!( - options.keys().copied().collect::>(), - BTreeSet::from([MatAtIngestion, NotMaterialized]) - ); - for (id, _) in options.values() { - assert!(run.cost(id).is_some(), "{id} is priced"); - let latency = run.selection.costs[id].usage.latency_per_query[0].total_ms(); - assert!(latency <= 200.0, "{id}: {latency} ms"); - } -} - -/// B3 runs nothing at ingestion time; it keeps the tumbling KLLs it builds at query time. -/// Implementer addition: kept for the readers' 5-min look-back, it is -/// priced as building one 1-min tumbling window per evaluation, so it reads a fifth of -/// B2's raw data and keeps a fifth of it on disk. -#[test] -fn stage2_physical_asap_optimization_b_b3_keeps_query_time_windows() { - let run = run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &keeping_query_time_results(), - ); - let options = options_of(&run, tumbling(), 1); - let (id, build) = options[&MatAtQuery].clone(); - let p = run.physical(&id); - assert!( - p.dag().nodes.iter().all(|n| !runs_at_ingestion(p, n.id)), - "{id}" - ); - assert_eq!(retention_ms(p, build), Some(PATTERN_B_WINDOW_MS)); - let disk = |id: &str| run.selection.costs[id].usage.disk_bytes; - assert_eq!(disk(&id) * 5, disk(&options[&NotMaterialized].0)); -} - -/// The sliding-window KLL is kept from ingestion time or from query time; not materializing it is the no-window plan. -#[test] -#[ignore = "needs sliding windows (#522)"] -fn stage2_physical_asap_optimization_b_sliding_kll_has_two_materialization_options() { - let found: BTreeSet<_> = options_of( - &run_pattern_b_fixture( - &pattern_b(), - &CostInputs::default(), - &DeploymentCapabilities::default(), - ), - sliding(), - 1, - ) - .into_keys() - .collect(); - assert_eq!(found, BTreeSet::from([MatAtIngestion, MatAtQuery])); -} - -// ── Cost models (#683, #660) ───────────────────────────────────────────── - -/// Pattern B under `cost_model`, with the default resource speed and horizon. -fn pattern_b_under(cost_model: CostModel) -> Run { - run_pattern_b_fixture( - &pattern_b(), - &CostInputs { - cost_model, - ..Default::default() - }, - &DeploymentCapabilities::default(), - ) -} - -/// The candidates the cost models choose between: the exact plan, the KLL -/// rebuilt over the whole window at query time and B1 (the tumbling windows maintained at ingestion time). -struct PatternBCandidates { - exact: String, - whole_window_kll: String, - b1: String, -} - -fn pattern_b_candidates(run: &Run) -> PatternBCandidates { - let tumbling_windows = options_of(run, tumbling(), 1); - PatternBCandidates { - exact: run - .physical - .iter() - .find(|p| sketch_builds(p.dag()).is_empty()) - .expect("an exact candidate") - .id - .clone(), - whole_window_kll: options_of(run, Some(WindowForm::None), 1)[&NotMaterialized] - .0 - .clone(), - b1: tumbling_windows[&MatAtIngestion].0.clone(), - } -} - -/// A cluster with `memory_gb` GB of memory and 1 TB of disk. -fn cluster(memory_gb: u64, objective: ClusterObjective) -> CostModel { - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: memory_gb * 1_000_000_000, - disk_bytes: 1_000_000_000_000, - cpu_cores: None, - objective, - }) -} - -/// Each cost model chooses what its deployment pays for. Cost model 1 -/// (usage over time) picks the KLL rebuilt every minute: it uses memory only -/// while it runs, and B1 pays for 5.2 GB of tumbling windows every second. Cost model 2 -/// (provisioned capacity) picks the exact plan: every candidate needs one -/// core for its evaluation, and the exact plan's peak memory (192 MB of -/// per-series values) is the smallest, while the KLL's 1M states, each -/// grown for its 20 samples, take 1 GB at once. Cost model 3 (own cluster, -/// lowest latency) picks B1 when its 6.1 GB fit, since merging maintained -/// tumbling windows takes 7 ms against 126 ms for rebuilding; in 4 GB, B1 is invalid -/// and the KLL is fastest. -#[test] -fn stage3_plan_selection_b_cost_models_choose_differently() { - let usage_over_time = - pattern_b_under(CostModel::UsageOverTime(ResourceUnitCost::illustrative())); - let ids = pattern_b_candidates(&usage_over_time); - assert_eq!(usage_over_time.selection.selected, ids.whole_window_kll); - - let provisioned = pattern_b_under(CostModel::ProvisionedCapacity( - ResourceUnitCost::illustrative(), - )); - assert_eq!(provisioned.selection.selected, ids.exact); - let peak = |run: &Run, id: &str| run.selection.costs[id].usage.peak_memory_bytes; - assert!(peak(&provisioned, &ids.exact) < peak(&provisioned, &ids.whole_window_kll)); - - let large = pattern_b_under(cluster(256, ClusterObjective::LowestLatency)); - assert_eq!(large.selection.selected, ids.b1); - assert!(large - .selection - .costs - .values() - .all(|c| c.cost_per_second.is_none())); - - let small = pattern_b_under(cluster(4, ClusterObjective::LowestLatency)); - assert_eq!(small.selection.selected, ids.whole_window_kll); - let invalid = small.invalid(); - let b1 = invalid[ids.b1.as_str()]; - assert!(b1.starts_with("peak memory"), "{b1}"); -} diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs deleted file mode 100644 index 9db3c4947..000000000 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ /dev/null @@ -1,641 +0,0 @@ -//! Planner-selected summaries over raw data samples compile, bound to raw data -//! at their `Scan`, and produce the same estimates as feeding their kernel -//! sample by sample. -use crate::physical_common; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPOperatorPayload}; -use asap_types::ir::ASAPOp; -use asap_types::ir::OperatorNode; -use physical_common::compile_at_ingestion_physical_asap_dag; -use std::{collections::BTreeMap, collections::BTreeSet, rc::Rc, sync::Arc}; - -use asap_executor::{ - factory::create_planner_accumulator, - physical_planner::{compile, promql_rows, CompiledPhysicalDAG}, - runtime::{Limits, RunContext, Scope}, - summary_kernels::{exact::ExactAccumulator, weighted_frequency::WeightedFrequency}, - values::Value, - AggregateCore, -}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::Reduction; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ColumnRef; -use asap_types::ir::schema::{ - FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, -}; -use futures::{executor::block_on, StreamExt}; - -type Series = BTreeMap; - -/// A series label set; `None` omits the label. An empty value is present -/// in the input but is not part of the series identity. -fn series(service: Option<&str>, instance: &str) -> Series { - [ - ("__name__", Some("m")), - ("service", service), - ("instance", Some(instance)), - ] - .into_iter() - .filter_map(|(k, v)| Some((k.to_owned(), v?.to_owned()))) - .collect() -} - -fn canonical(labels: &Series) -> Series { - labels - .iter() - .filter(|(_, v)| !v.is_empty()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() -} - -/// Exact and explicit single-rule candidates for `query`, with -/// `m` declared a counter: plan selection's answer to a heap's open -/// requirement that its weights are counter samples, which the executor -/// needs before it builds a Count-Min heap. -fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { - use asap_types::ir::QueryRoot; - use asap_types::workload::bounds::with_confirmed_weights; - use asap_types::workload::{DataWorkload, PrometheusMetricType, SourceFacts}; - let root = lower_promql(query, accuracy.clone()).expect("lowering failed"); - let root = promql_rows::with_series_identity(&root).unwrap(); - let counter = SourceFacts { - metric_type: Some(PrometheusMetricType::Counter), - ..SourceFacts::default() - }; - let data = DataWorkload { - sources: [("m".to_string(), counter)].into(), - ..Default::default() - }; - explicit_local_candidates(root) - .into_iter() - .map(|candidate| { - match &with_confirmed_weights(&[QueryRoot::Operator(candidate)], &data)[..] { - [QueryRoot::Operator(confirmed)] => Rc::clone(confirmed), - _ => unreachable!("one operator root"), - } - }) - .collect() -} - -/// Raw-input summary nodes: `(dag, raw source id, summary id)`. -fn raw_summaries(dag: &PhysicalASAPDAG) -> Vec<(u64, u64)> { - dag.nodes - .iter() - .filter(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - ) - }) - .filter_map(|node| { - let inputs = dag - .edges - .iter() - .filter(|edge| edge.consumer == node.id) - .collect::>(); - let [edge] = inputs.as_slice() else { - return None; - }; - let source = dag.nodes.iter().find(|n| n.id == edge.producer)?; - matches!( - source.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::TimeRange { .. }) - ) - .then_some((source.id as u64, node.id as u64)) - }) - .collect() -} - -fn samples() -> Vec<(Series, i64, f64)> { - let mut rows = Vec::new(); - let series_set = [ - (Some("a"), "1"), - (Some("a"), "2"), - (Some("b"), "1"), - (None, "3"), - (Some("b"), ""), - ]; - for (index, (service, instance)) in series_set.iter().enumerate() { - for step in 1..=5i64 { - let value = (index as f64 + 1.0) * step as f64 + (step % 2) as f64; - rows.push((series(*service, instance), step * 1000, value)); - } - } - rows -} - -/// Compile the summary `root` alone, its raw data read at the `Scan`. -fn compile_summary(dag: &PhysicalASAPDAG, root: u64) -> Result { - compile(dag, &[], &[root]).map_err(|error| error.to_string()) -} - -/// Run the summary `root` in one ingestion run over `rows`, bound as raw -/// data at its `Scan`: each population's labels and state. -fn execute( - dag: &PhysicalASAPDAG, - root: u64, - rows: &[(Series, i64, f64)], -) -> Vec<(Series, Arc)> { - let program = compile_summary(dag, root).unwrap_or_else(|error| { - panic!("raw summary {root} does not compile: {error}"); - }); - let store = physical_common::raw_data_store(dag, |_, schema| { - rows.iter() - .map(|(labels, time, value)| { - promql_rows::series_row(schema, &canonical(labels), *time, *value).unwrap() - }) - .collect() - }); - let output = program.output_schema(root).unwrap(); - let physical_dag = program.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Ingestion { - window_start_ms: 0, - window_end_ms: 6000, - }, - Limits::default(), - ) - .unwrap(); - block_on(async { - let mut stream = physical_dag - .execute(program.roots(), context) - .unwrap() - .remove(0); - let mut result = Vec::new(); - while let Some(batch) = stream.next().await { - for row in &batch.unwrap().to_rows() { - let mut labels = Series::new(); - let mut state = None; - for (field, value) in output.fields.iter().zip(row) { - match value { - Value::Utf8(identity) - if field.name == promql_rows::SERIES_IDENTITY_COLUMN => - { - labels.extend(promql_rows::decode_series_identity(identity).unwrap()) - } - Value::Utf8(label) => { - labels.insert(field.name.clone(), label.to_string()); - } - Value::Summary { state: s, .. } => state = Some(s.clone()), - Value::Timestamp(6000) => {} - other => panic!("unexpected population column {other:?} in {row:?}"), - } - } - result.push((canonical(&labels), state.expect("a summary state"))); - } - } - result - }) -} - -/// Each series' latest sample, moved to the ingestion run's end (6000 ms). -fn latest_per_series(rows: &[(Series, i64, f64)]) -> Vec<(Series, i64, f64)> { - let mut latest = BTreeMap::::new(); - for (labels, time, value) in rows { - let entry = latest.entry(canonical(labels)).or_insert((*time, *value)); - if *time >= entry.0 { - *entry = (*time, *value); - } - } - latest - .into_iter() - .map(|(labels, (_, value))| (labels, 6000, value)) - .collect() -} - -/// PromQL grouping of a canonical label set: `by` keeps the named labels, -/// `without` drops them and `__name__`. -fn population(reduction: &Reduction, dag_labels: &[String], labels: &Series) -> Series { - let labels = canonical(labels); - match reduction { - Reduction::PerEntity => labels, - Reduction::Reduce(keys) => labels - .into_iter() - .filter(|(k, _)| { - if keys.is_without() { - k != "__name__" && !dag_labels.contains(k) - } else { - dag_labels.contains(k) - } - }) - .collect(), - } -} - -/// Item identity used by a keyed summary: labels by name, the sample value, -/// or the canonical label-set identity. -fn item(expr: &SummaryInputExpr, labels: &Series, value: f64, out: &mut Vec) { - match expr { - SummaryInputExpr::Column(ColumnRef::SampleValue) => out.push(Value::Float64(value)), - SummaryInputExpr::Column(ColumnRef::Named(name)) if name == "value" => { - out.push(Value::Float64(value)) - } - SummaryInputExpr::Column(ColumnRef::Named(name)) - if name == promql_rows::SERIES_IDENTITY_COLUMN => - { - out.push(Value::Utf8( - promql_rows::encode_series_identity(&canonical(labels)) - .unwrap() - .into(), - )) - } - SummaryInputExpr::Column(ColumnRef::Named(name)) => out.push(Value::Utf8( - canonical(labels) - .get(name) - .cloned() - .unwrap_or_default() - .into(), - )), - SummaryInputExpr::Tuple(items) => items.iter().for_each(|i| item(i, labels, value, out)), - other => panic!("unsupported fixture item {other:?}"), - } -} - -fn weight(update: &SummaryUpdate, value: f64) -> f64 { - match update.weight { - SummaryInputExpr::Constant(weight) => weight, - _ => value, - } -} - -/// Estimates that identify a state's content for comparison. -fn evaluations(state: &dyn AggregateCore, family: &FieldDataType) -> Vec { - if let Some(exact) = state.as_any().downcast_ref::() { - let FieldDataType::ExactAggregate(kind, _) = family else { - unreachable!() - }; - return vec![exact.evaluation(*kind, None, None).unwrap().unwrap()]; - } - let FieldDataType::Sketch(kind, _) = family else { - panic!("sketch state for exact family") - }; - match kind.algorithm() { - SketchAlgorithm::Kll | SketchAlgorithm::DDSketch => [0.1, 0.5, 0.9] - .into_iter() - .map(|q| state.estimate(&SketchStatistic::Quantile { q }).unwrap()) - .collect(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv => { - vec![state.estimate(&SketchStatistic::Cardinality).unwrap()] - } - SketchAlgorithm::UnivMon => [ - SketchStatistic::Cardinality, - SketchStatistic::FrequencyL2, - SketchStatistic::FrequencyEntropy, - ] - .iter() - .map(|statistic| state.estimate(statistic).unwrap()) - .collect(), - other => panic!("unexpected unkeyed sketch {other:?}"), - } -} - -/// Compile one raw-input summary, execute it over `rows`, and compare each -/// population with its kernel fed sample by sample. Returns the family label, -/// or the family when it has no native state. -fn check( - query: &str, - dag: &PhysicalASAPDAG, - source: u64, - root: u64, - rows: &[(Series, i64, f64)], -) -> Result { - let node = dag.nodes.iter().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family, - input, - reduction, - grouping, - .. - }) = &node.payload - else { - unreachable!() - }; - let source_node = dag.nodes.iter().find(|n| n.id as u64 == source).unwrap(); - // An instant selector keeps each series' latest sample within its - // lookback before the run's end: feed one such sample per series. - let latest_rows; - let rows = if matches!( - source_node.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::TimeRange { - kind: asap_types::ir::TimeRangeKind::Instant, - .. - }) - ) { - latest_rows = latest_per_series(rows); - &latest_rows[..] - } else { - rows - }; - let keys = match reduction { - Reduction::Reduce(keys) => keys - .keys() - .iter() - .map(|i| source_node.output_schema.fields[*i].name.clone()) - .collect(), - Reduction::PerEntity => vec![], - }; - let stored_only = matches!(family, FieldDataType::Sketch(kind, _) - if kind.algorithm() == &asap_types::ir::schema::SketchAlgorithm::Cms); - if stored_only || asap_executor::capability::validate_native_family(family).is_err() { - // Families without a native state, or with native stored state only (plain CMS), are outside raw summary execution; - // their compile must fail. - assert!(compile_summary(dag, root).is_err()); - return Err(format!("{family:?}")); - } - let actual = execute(dag, root, rows); - let label = match family { - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?}"), - FieldDataType::Sketch(kind, _) => format!("{:?}", kind.algorithm()), - other => format!("{other:?}"), - }; - if let FieldDataType::Sketch(kind, _) = family { - if let (Some(keyed), false) = ( - &input.item, - matches!( - kind.algorithm(), - SketchAlgorithm::Hll | SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon - ), - ) { - // Keyed heaps: every item's estimated weight is its exact - // total at this scale (no collisions in the fixture). - let mut expected = BTreeMap::>::new(); - for (labels, _, value) in rows { - let mut items = Vec::new(); - item(keyed, labels, *value, &mut items); - *expected - .entry(population(reduction, &keys, labels)) - .or_default() - .entry(format!("{items:?}")) - .or_default() += weight(input, *value); - } - assert_eq!(actual.len(), expected.len(), "{query}"); - for (labels, state) in &actual { - let heap = state.as_any().downcast_ref::().unwrap(); - let got = heap - .rows(usize::MAX >> 1) - .into_iter() - .map(|mut row| { - let Some(Value::Float64(score)) = row.pop() else { - panic!("heap score") - }; - (format!("{row:?}"), score) - }) - .collect::>(); - assert_eq!(&got, &expected[labels], "{query}"); - } - return Ok(label); - } - } - let mut expected = - BTreeMap::>::new(); - for (labels, time, value) in rows { - let updater = expected - .entry(population(reduction, &keys, labels)) - .or_insert_with(|| create_planner_accumulator(family, input, grouping).unwrap()); - let unit = input.item.is_some(); - updater - .update_single(if unit { *value } else { weight(input, *value) }, *time) - .unwrap(); - } - assert_eq!(actual.len(), expected.len(), "{query}"); - for (labels, state) in actual { - let reference = expected[&labels].snapshot_accumulator(); - assert_eq!( - evaluations(state.as_ref(), family), - evaluations(reference.as_ref(), family), - "{query}: {labels:?}" - ); - } - Ok(label) -} - -// Every raw-input summary Planner composes compiles over raw data sample rows, -// and each population's estimates equal feeding its kernel sample by sample. -#[test] -fn raw_data_sample_summaries_compile_and_match_their_kernels() { - let exact = AccuracyTarget::exact(); - let sketch = AccuracyTarget::max_error(0.02); - // A UnivMon sized for a distinct count at ε = 0.02 holds a heap of - // millions of keys on each of its 19 layers, past the default 64 MiB. - let distinct = AccuracyTarget::max_error(0.2).failure_probability(0.1); - let queries = [ - ("sum_over_time(m[5m])", &exact), - ("count_over_time(m[5m])", &exact), - ("min_over_time(m[5m])", &exact), - ("max_over_time(m[5m])", &exact), - ("rate(m[5m])", &exact), - ("increase(m[5m])", &exact), - ("sum by (service) (sum_over_time(m[5m]))", &exact), - ("sum by (service) (rate(m[5m]))", &exact), - ("topk(2, sum_over_time(m[5m]))", &exact), - ("quantile_over_time(0.9, m[5m])", &sketch), - ("sum by (service) (quantile_over_time(0.9, m[5m]))", &sketch), - ("quantile by (service) (0.9, m)", &sketch), - ("distinct_over_time(m[5m])", &distinct), - ("count(m)", &sketch), - ("topk(2, m)", &sketch), - ( - "topk(2, sum by (service) (count_over_time(m[5m])))", - &sketch, - ), - ("topk(2, sum_over_time(m[5m]))", &sketch), - ("topk by (service) (2, sum_over_time(m[5m]))", &sketch), - ]; - let rows = samples(); - let mut families = BTreeSet::new(); - let mut unsupported = BTreeSet::new(); - let mut checked = BTreeMap::new(); - for (query, accuracy) in queries { - for candidate in candidates(query, accuracy.clone()) { - let dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - for (source, root) in raw_summaries(&dag) { - match check(query, &dag, source, root, &rows) { - Ok(family) => { - families.insert(family); - *checked.entry(query).or_insert(0) += 1; - } - Err(family) => { - unsupported.insert(family); - } - } - } - } - } - println!("checked {checked:?}; families {families:?}; without native state {unsupported:?}"); - for query in [ - "topk by (service) (2, sum_over_time(m[5m]))", - "quantile by (service) (0.9, m)", - "distinct_over_time(m[5m])", - ] { - assert!( - checked.contains_key(query), - "{query} has no checked raw summary" - ); - } - for family in [ - "Sum", - "Count", - "Min", - "Max", - "Rate", - "Increase", - "Kll", - "DDSketch", - "Hll", - "Kmv", - "UnivMon", - // The whole-expression heaps over raw data samples. - "CmsWithHeap", - "CountSketchWithHeap", - ] { - assert!( - families.contains(family), - "no {family} fixture: {families:?}" - ); - } -} - -/// Replace the raw summary of `sum by (service) (sum_over_time(m{instance!="none"}[5m]))` with -/// another update, keeping its raw input and reduction. -fn grouped_raw_summary(family: FieldDataType, input: SummaryUpdate) -> (PhysicalASAPDAG, u64, u64) { - // The matcher, true of every sample, makes the `Scan` read `instance`. - let candidate = candidates( - r#"sum by (service) (sum_over_time(m{instance!="none"}[5m]))"#, - AccuracyTarget::exact(), - ) - .pop() - .unwrap(); - let mut dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - let (source, root) = raw_summaries(&dag)[0]; - let node = dag.nodes.iter_mut().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: old, - input: update, - .. - }) = &mut node.payload - else { - unreachable!() - }; - for field in &mut node.output_schema.fields { - if field.dtype == *old { - field.dtype = family.clone(); - } - } - *old = family; - *update = input; - let schema = node.output_schema.clone(); - for edge in dag.edges.iter_mut().filter(|e| e.producer as u64 == root) { - *edge.intermediate_schema_mut() = schema.clone(); - } - (dag, source, root) -} - -// Keyed heaps over raw data samples resolve items from the series label set and -// estimate each item's exact total; invalid weight contracts do not compile. -#[test] -fn raw_data_sample_heaps_resolve_items_from_labels() { - use asap_types::ir::schema::GroupingStrategy::PerSubpopulationInstance; - use asap_types::ir::schema::{NonNegativeWeightProof, SketchKind, SketchParams, WeightDomain}; - let heap = |algorithm, params| { - FieldDataType::Sketch(SketchKind::new(algorithm, params), PerSubpopulationInstance) - }; - let cms = heap( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ); - let count_sketch = heap( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: 64, - depth: 3, - heap_size: 8, - }, - ); - // The series label set, read from the `Scan`'s series identity column. - let identity = SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::Named( - promql_rows::SERIES_IDENTITY_COLUMN.into(), - ))), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }; - let by_instance = SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( - ColumnRef::Named("instance".into()), - )])), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }; - let rows = samples(); - for (family, input) in [ - (cms.clone(), identity.clone()), - (count_sketch.clone(), identity), - (count_sketch, by_instance.clone()), - ] { - let unit_count = input.weight == SummaryInputExpr::Constant(1.0); - let (dag, source, root) = grouped_raw_summary(family, input); - assert!(check("heap", &dag, source, root, &rows).is_ok()); - // A unit count adds the constant 1 per row; no column of ones is - // projected for it. - let compiled = serde_json::to_value(compile_summary(&dag, root).unwrap()).unwrap(); - assert_eq!( - compiled - .to_string() - .contains(r#""row_weight":{"Constant":1}"#), - unit_count, - "{compiled}" - ); - } - let signed_cms = grouped_raw_summary(cms, by_instance.clone()); - assert!(compile_summary(&signed_cms.0, signed_cms.2).is_err()); -} - -// End to end: `topk(2, count by (service) (m))` at ε = 0.02 has a Count-Min -// heap candidate whose update is the unit count (`Constant(1.0)`). Placed at -// ingestion, its raw-data-sample precompute adds the constant 1 per sample to each -// service's count, with no column of ones projected, and each heap reads the -// exact per-service sample counts of the fixture. -#[test] -fn count_ranked_heap_precompute_adds_a_constant_row_weight() { - let query = "topk(2, count by (service) (m))"; - let rows = samples(); - let mut checked = 0; - for candidate in candidates(query, AccuracyTarget::max_error(0.02)) { - let dag = compile_at_ingestion_physical_asap_dag(&candidate).unwrap(); - for (source, root) in raw_summaries(&dag) { - let node = dag.nodes.iter().find(|n| n.id as u64 == root).unwrap(); - let PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - input, - .. - }) = &node.payload - else { - continue; - }; - if kind.algorithm() != &SketchAlgorithm::CmsWithHeap { - continue; - } - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - let compiled = serde_json::to_value(compile_summary(&dag, root).unwrap()) - .unwrap() - .to_string(); - assert!( - compiled.contains(r#""row_weight":{"Constant":1}"#), - "{compiled}" - ); - assert_eq!( - check(query, &dag, source, root, &rows), - Ok("CmsWithHeap".into()) - ); - checked += 1; - } - } - assert!(checked > 0, "{query} has no Count-Min heap candidate"); -} diff --git a/crates/integration-tests/tests/production_rollup_workloads.rs b/crates/integration-tests/tests/production_rollup_workloads.rs index c44f7724a..5cad992ea 100644 --- a/crates/integration-tests/tests/production_rollup_workloads.rs +++ b/crates/integration-tests/tests/production_rollup_workloads.rs @@ -181,96 +181,6 @@ async fn production_sql_distinct_region_service_rollup() { check(&sql(&queries, accuracy).await, 2, 1, "Kmv", None); } -#[test] -// The earlier controlled fixture omitted raw retention. This production -// workload explicitly holds one current sample for each of its 10,000 series. -fn production_promql_compares_rollup_with_exact() { - let accuracy = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - let roots: Vec<_> = [ - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ] - .iter() - .map(|query| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()) - }) - .collect(); - let data: DataWorkload = serde_json::from_value(serde_json::json!({"arrival":{"ingesting":{"rows_per_second":666.67}},"sources":{"data":{"series_count":10000,"metric_type":"gauge","sample_interval":15000,"rows_held":10000}}})).unwrap(); - let entries = [entry(&accuracy, 10000), entry(&accuracy, 10000)]; - let separate = independent_kll_cost(roots.clone(), &entries, &data); - println!("independent KLL cost={separate}"); - check( - &production(roots, &entries, &data), - 2, - 1, - "Kll", - Some(separate), - ); -} - -// Price the same workload with independently built, unmaterialized KLL states. -fn independent_kll_cost( - roots: Vec, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, -) -> f64 { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary( - asap_types::ir::schema::SketchAlgorithm::Kll - ) - ) - }) - .unwrap() - }) - .collect(); - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - // Composed outside the memo: sized as Pass 1 sizes a candidate. - let targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let roots = - asap_accuracy::sizing::with_summaries_sized_for_targets(&roots, &targets, data).unwrap(); - let mut settings = CostInputs::default(); - settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - settings.resource_speed, - )); - let models = PlanningModels::builtin().with_cost_inputs(&settings); - let plan = asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "independent", - &roots, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |_, _, _| Some(asap_types::physical::Materialization::NotMaterialized), - ) - .unwrap() - .unwrap(); - let result = - asap_devtools::memo_pricing::reference_select(&[plan], entries, data, models).unwrap(); - result.costs[&result.selected].cost_per_second.unwrap() -} - // Exercise multi-input operators through the production route under more than // three heterogeneous query schedules, not just the local pricing kernel. #[test] diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs index 0e8469f03..5626606a4 100644 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ b/crates/integration-tests/tests/promql_numeric_regressions.rs @@ -120,7 +120,7 @@ fn checked_ratio_must_not_certify_cross_zero_interpolation() { ) .unwrap(); let node = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch); - assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if *kind.algorithm() == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, + assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if *kind == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, "the signed interpolation counterexample requires two explicit DDSketch builds"); assert!( matches!(node.operator, Operator::NonASAP(NonASAPOp::BinaryOp { .. })) @@ -273,7 +273,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { .find_map(|node| { let state = heap(node)?; let (family, _, _) = aggregate(&state); - matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &wanted) + matches!(family, FieldDataType::Sketch(kind, _) if kind == &wanted) .then_some((node, state)) }) .expect("weighted sketch candidate"); @@ -295,7 +295,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { // so plan selection rejects it. for state in candidates.iter().filter_map(heap) { let (family, update, _) = aggregate(&state); - if matches!(family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind, _) if kind == &SketchAlgorithm::CmsWithHeap) { assert!(!matches!( update.weight_domain, diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index a2e6670d3..398ee91de 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -6,29 +6,23 @@ //! the summary-bound shape node by node, including the family `(Kind, //! Params)` committed on each edge's schema. -use asap_types::stages::HowToComputeAggregateMeasure; use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::physical_export::{ - at_ingestion, at_ingestion_physical_asap_dag, physical_asap_dag, -}; +use asap_integration_tests::physical_export::physical_asap_dag; use asap_integration_tests::planning::{ logical_candidates, logical_candidates_ignoring_guarantees, }; -use asap_plan_selection::reference_pricer::WholePlanSelection as Selection; use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyMetric; + use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::DataType; use asap_types::ir::schema::{ EntityIdentity, ExactKind, ExactParams, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryInputExpr, SummaryUpdate, + SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::physical::PhysicalWorkloadPlan; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; /// The first logical candidate (Stage 1, in enumeration order) of `root` as /// a query at `accuracy` that uses an ASAP operator, or the plain exact one @@ -52,7 +46,7 @@ fn uses_sketch(node: &OperatorNode, algorithm: &SketchAlgorithm) -> bool { Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm() == algorithm => true, + }) if kind == algorithm => true, _ => node .children() .iter() @@ -254,90 +248,6 @@ fn promql_binary_arithmetic_preserves_nested_structure_and_vector_matching() { } } -#[test] -fn planner_only_e2e_temporal_topk_preserves_query_update_and_evaluation_contract() { - // Self-contained Planner E2E: each case starts from PromQL text and ends - // at the post-ASAP summary DAG. No controller/backend types, - // fixtures, configuration, or runtime are involved. - let cases = [ - ( - "topk by (service) (5, count_over_time(requests[1m]))", - SummaryInputExpr::Constant(1.0), - SketchAlgorithm::CmsWithHeap, - vec![ColumnRef::Named("service".into())], - ), - ( - "topk(5, sum_over_time(requests[1m]))", - SummaryInputExpr::Column(ColumnRef::SampleValue), - SketchAlgorithm::CountSketchWithHeap, - vec![], - ), - ]; - for (source, expected_update, expected_family, excluded_labels) in cases { - let pre = lower_promql( - source, - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ) - .expect("lower temporal Top-K"); - let candidates = heap_topk_candidates_using( - &pre, - AccuracyTarget::max_error(0.01).failure_probability(0.01), - expected_family, - ); - assert!( - !candidates.is_empty(), - "heap-backed temporal Top-K candidate" - ); - // The search archive may visit tighter local error allocations first. - // Every visited configuration must preserve the same update contract. - for candidate in candidates { - let Some(ASAPOp::SummaryEstimate { - summary_input, - query: SketchStatistic::TopK { k, .. }, - }) = candidate.asap() - else { - panic!("expected Top-K estimate, got {:?}", candidate.operator) - }; - assert_eq!( - *k, 5, - "the requested Top-K cardinality must survive binding" - ); - let Some(ASAPOp::SummaryAgg { - input: state_input, - family, - child, - .. - }) = summary_input.asap() - else { - panic!("expected structured Top-K state input") - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("expected a heap-backed sketch family, got {family:?}") - }; - assert_eq!(kind.algorithm(), &expected_family); - let heap_size = match kind.params() { - SketchParams::CmsWithHeap { heap_size, .. } - | SketchParams::CountSketchWithHeap { heap_size, .. } => *heap_size, - params => panic!("expected heap-bearing Top-K parameters, got {params:?}"), - }; - assert!( - heap_size >= 100u32.max(*k as u32), - "a tighter local error budget may increase, never reduce, capacity" - ); - assert_eq!( - state_input.item.as_ref(), - Some(&SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { - excluding: excluded_labels.clone() - } - )) - ); - assert_eq!(state_input.weight, expected_update); - assert!(!child.contains_asap()); - } - } -} - /// Execute the ungrouped temporal TopK subset with exact state. This tests /// the emitted update contract, not sketch approximation or backend execution. fn execute_topk_reference(plan: &OperatorNode) -> Vec<(String, f64)> { @@ -533,16 +443,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { }; assert_eq!( family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); assert_eq!(input, &SummaryUpdate::column(ColumnRef::SampleValue)); assert_eq!( @@ -552,16 +453,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); assert_eq!( dtype(&summary_input.schema, "state"), - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); let Some(ASAPOp::FinalizeExactAccumulator { child }) = child.asap() else { @@ -622,255 +514,6 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); } -#[test] -fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { - // Real workload selection must expose the state-to-value edge; an outer - // sketch must not interpret exact accumulator bytes as input samples. - let pre = lower_promql( - "quantile(0.9, sum_over_time(m[1m]))", - AccuracyTarget::max_error(0.05), - ) - .unwrap(); - // The KLL candidate over the inner exact Sum accumulator. - let plan = candidates_using(&pre, AccuracyTarget::max_error(0.05), SketchAlgorithm::Kll) - .into_iter() - .find(|plan| { - matches!(plan.asap(), Some(ASAPOp::SummaryEstimate { summary_input, .. }) - if matches!(summary_input.asap(), Some(ASAPOp::SummaryAgg { child, .. }) - if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) - }) - .expect("a KLL over the finalized Sum accumulator"); - // Stored timings are gone: time the plan with its outer summary - // maintained and read the timed copy. - let timed_plan = at_ingestion(&plan); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = timed_plan.asap() else { - panic!("expected selected quantile summary"); - }; - let Some(ASAPOp::SummaryAgg { child, .. }) = summary_input.asap() else { - panic!("expected maintained outer summary"); - }; - let Some(ASAPOp::FinalizeExactAccumulator { child: source }) = child.asap() else { - panic!( - "missing explicit accumulator finalization: {:?}", - child.operator - ); - }; - assert_eq!( - child.timing, - Some(asap_types::ir::properties::ExecutionTiming::IngestionTime) - ); - assert!(matches!( - source.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }) - )); - assert!(child - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - assert!(child - .schema - .fields - .iter() - .any(|field| matches!(field.dtype, FieldDataType::Plain(DataType::Float64)))); - // Explicit boundary is a valid post-ASAP DAG. - physical_asap_dag(&plan); -} - -#[test] -fn physical_node_owns_phase_independently_of_binary_payload() { - use asap_types::ir::properties::ExecutionTiming; - // The one-selector case (both operands over the same series, so the - // BinaryOp may run at ingestion) is `arithmetic_over_one_selector_executes` - // in materialization_e2e.rs, ignored until #580. - let query = "sum_over_time(m[1m]) + sum_over_time(n[1m])"; - let expected = ExecutionTiming::QueryTime; - let input = lower_promql(query, AccuracyTarget::max_error(0.05)).unwrap(); - let plan = realize(&input, AccuracyTarget::max_error(0.05)); - let dag = at_ingestion_physical_asap_dag(&plan); - let node = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::BinaryOp { .. }) - ) - }) - .unwrap(); - assert_eq!(node.output_state.timing, expected); - let wire = serde_json::to_value(&node.payload).unwrap(); - assert!(wire.get("timing").is_none()); - let mut obsolete = wire.clone(); - obsolete["timing"] = serde_json::json!(expected.as_str()); - assert!(serde_json::from_value::(obsolete).is_err()); - let restored: PhysicalASAPOperatorPayload = serde_json::from_value(wire).unwrap(); - assert_eq!(restored, node.payload); -} - -/// `query` lowered at `planned` and selected at `accuracy`: the physical -/// candidates whose two summaries are both DDSketch, and plan selection over -/// all of them. -fn ddsketch_ratio_selection( - query: &str, - planned: AccuracyTarget, - accuracy: AccuracyTarget, - non_negative: bool, -) -> (Vec, Selection) { - let root = lower_promql(query, planned).unwrap(); - let mut data = asap_integration_tests::planning::declared_data(&root); - for facts in data.sources.values_mut() { - facts.values_non_negative = non_negative; - } - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let exact = vec![0; inventory.targets.len()]; - let dds = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .expect("quantile supports DDS") - }) - .collect(); - let entries = [asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy, - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }]; - // Pass 1 choices composed and implemented by Stage 2; one query, so no - // sharing rule applies. - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::compose_logical_candidate; - let models = asap_planner::PlanningModels::builtin(); - let mut physical = Vec::new(); - for choice in [exact, dds] { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let roots = recompose_guarantees(&roots).unwrap(); - physical.extend( - asap_physical_asap_optimization::optimize_physical_asap( - &format!("L{}", physical.len() + 1), - &roots, - &entries, - &data, - models.deployment_capabilities.query_time_results_retained, - ) - .unwrap(), - ); - } - for (index, plan) in physical.iter_mut().enumerate() { - plan.id = format!("fixture-{index}"); - } - let selection = - asap_devtools::memo_pricing::reference_select(&physical, &entries, &data, models).unwrap(); - let ratios = physical - .into_iter() - .filter(|candidate| { - let sketches: Vec<_> = candidate - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(*kind.algorithm()), - _ => None, - }) - .collect(); - sketches == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch] - }) - .collect(); - (ratios, selection) -} - -/// Declared nonnegative inputs permit formal ratio admission; an unknown -/// sign does not. The same explicitly sized DDS fixture is also evaluated -/// numerically on all-positive and all-negative samples without claiming -/// a general bound for arbitrary signed interpolation. -#[test] -fn ddsketch_ratio_checks_sign_contract_and_one_sided_sketch_evaluations() { - for sign in [-1., 1.] { - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let (ratios, selection) = ddsketch_ratio_selection( - "quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])", - AccuracyTarget::max_error(0.01), - target.clone(), - sign > 0.0, - ); - let accepted: Vec<_> = ratios - .iter() - .filter(|candidate| selection.costs.contains_key(&candidate.id)) - .collect(); - assert!( - !ratios.is_empty(), - "explicit DDS fixture exists regardless of admission" - ); - assert_eq!(!accepted.is_empty(), sign > 0.0, "{selection:?}"); - // Both explicit operands use the planner's composed-budget alpha. - let alphas: Vec = ratios[0] - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - SketchParams::DDSketch { alpha } => Some(*alpha), - _ => None, - }, - _ => None, - }) - .collect(); - assert_eq!(alphas.len(), 2); - assert_eq!(alphas[0], alphas[1]); - let alpha = alphas[0]; - let bound = target.max_error; - for values in [ - vec![sign; 30], - (1..=100).map(|i| sign * i as f64).collect(), - vec![sign, sign, sign, sign * 30., sign * 100.], - ] { - let mut sorted = values.clone(); - sorted.sort_by(f64::total_cmp); - let exact = |q: f64| { - let r = q * (sorted.len() - 1) as f64; - sorted[r.floor() as usize] * (1. - r.fract()) - + sorted[r.ceil() as usize] * r.fract() - }; - let mut sketch = asap_sketchlib::DdSketch::new(alpha); - for v in values { - sketch.try_add(&v).unwrap(); - } - // PromQL's quantile: interpolated between neighbouring order - // statistics. - let interpolated = |q: f64| sketch.quantile_interpolated(q).unwrap().unwrap(); - let want = exact(0.9) / exact(0.5); - let got = interpolated(0.9) / interpolated(0.5); - assert!((got - want).abs() / want.abs() <= bound + 1e-12); - } - } -} - // Every `without` aggregation candidate exports a valid DAG: its summary state // column carries the family instead of the evaluation's Float64 value. #[test] diff --git a/crates/integration-tests/tests/range_evaluation.rs b/crates/integration-tests/tests/range_evaluation.rs deleted file mode 100644 index 2f97eeb72..000000000 --- a/crates/integration-tests/tests/range_evaluation.rs +++ /dev/null @@ -1,423 +0,0 @@ -//! #757: a query answered as a range request (`RangeEvaluation`), e.g. a -//! dashboard panel that evaluates the last hour at 1-minute steps on every -//! refresh. Stage 1 and Stage 2 keep tumbling-window widths on the step grid, and -//! Stage 3 prices `span_ms / step_ms + 1` evaluations per request, credits -//! the tumbling windows kept across steps and requests, and retains kept tumbling windows for -//! the span plus the window. - -use crate::planner_layering_common; - -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_plan_selection::RejectionCause; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::ASAPOp; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RangeEvaluation, RepeatedDemand, RepeatingEntry, - RepetitionInterval, SourceFacts, -}; -use planner_layering_common::*; - -/// The dashboard's range: the last hour at 1-minute steps, 61 evaluations -/// per request. -const LAST_HOUR_BY_MINUTE: RangeEvaluation = RangeEvaluation { - step_ms: 60_000, - span_ms: 3_600_000, -}; - -/// Price explicitly named computations under the same models as production. -/// This fixture controls its tumbling-window width; a timed MILP archive is not a catalog -/// of every legal materialization or a stable source of P1/P2 identifiers. -fn run_explicit_for(workload: &PlanningWorkload, deployment: &DeploymentCapabilities) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::ir::schema::SketchAlgorithm; - use asap_types::stages::HowToComputeAggregateMeasure; - let roots = lower_promql(workload); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 1); - let compose = |choice| compose_logical_candidate(&inventory, &[choice]).unwrap(); - let mut candidates = vec![compose(0)]; - if let Some(choice) = inventory.targets[0] - .ways_to_compute - .iter() - .position(|candidate| { - matches!( - candidate, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - { - candidates.push(compose(choice)); - } - let build = candidates.last().unwrap(); - let grids = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(build, &grids); - assert_eq!( - tumbling_windows.len(), - 1, - "one explicit tumbling window computation" - ); - candidates.push( - window_composition_pass2_candidates(build, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(), - ); - let settings = asap_cost::CostInputs::default(); - run_explicit_candidates( - workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -fn run_explicit(workload: &PlanningWorkload) -> Run { - run_explicit_for(workload, &DeploymentCapabilities::default()) -} - -/// Pattern B (#509 Example 3: `quantile_over_time(0.99, latency_ms[5m])` -/// every minute over 1M series) with each request answered as `range`. -fn pattern_b_with(range: Option) -> PlanningWorkload { - let mut workload = pattern_b(); - for entry in workload - .query_workload - .repeating_queries - .iter_mut() - .flatten() - { - entry.range_evaluation = range; - } - workload -} - -/// A deployment that keeps query-time results across evaluations (B3). -fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } -} - -/// `query` every `interval_ms` over 1M series of `m` sampled every 1.5 s, -/// arriving continuously, exact, answered as `range`. -fn exact_panel(query: &str, interval_ms: u32, range: Option) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(interval_ms)), - requirements: QueryRequirements::default(), - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: range, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - sources: [( - "m".to_string(), - SourceFacts { - series_count: Some(1_000_000), - sample_interval: Some(DurationMs(1_500)), - ..SourceFacts::default() - }, - )] - .into(), - }), - } -} - -fn cost(run: &Run, id: &str) -> f64 { - run.cost(id).unwrap_or_else(|| panic!("{id} is not priced")) -} - -fn close(actual: f64, expected: f64) -> bool { - (actual - expected).abs() <= 1e-6 * expected.abs().max(1.0) -} - -/// Each physical candidate's tumbling-window chains: (tumbling-window width, materialization). -fn tumbling_window_chains(run: &Run) -> Vec<(String, u64, Materialization)> { - run.physical - .iter() - .flat_map(|p| { - p.dag().nodes.iter().filter_map(|n| match &n.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some((p.id.clone(), *tumbling_window_width_ms, n.materialization)), - _ => None, - }) - }) - .collect() -} - -/// The candidate whose single tumbling-window chain is materialized as `wanted`. -fn with_chain(run: &Run, wanted: Materialization) -> String { - tumbling_window_chains(run) - .into_iter() - .find(|(_, _, m)| *m == wanted) - .unwrap_or_else(|| panic!("no candidate with a {wanted:?} chain")) - .0 -} - -/// (a) The issue's acceptance example: Pattern B as a dashboard over the -/// last hour at 1-minute steps prices 61 evaluations per request. Every -/// query-time computation runs at every step, so the whole-window KLL -/// (2.3625 → 144.1125 cost/s), the tumbling windows rebuilt per step (3.6112 → -/// 220.2852) and the exact plan (2.4576 → 149.9136) cost 61 times as much, -/// the memory each step holds while it runs included. The kept tumbling windows (B3) -/// are built once per request (one new 1-min tumbling window) but merged at every -/// step, and keep 64 older tumbling windows instead of 4 (1 h + 5 min of tumbling windows): -/// 436.59 → 6,987.05. B1 keeps 66 instead of 6: 654.54 → 7,202.31. The -/// whole-window KLL is still selected. -#[test] -fn stage3_plan_selection_prices_61_evaluations_per_dashboard_request() { - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&pattern_b_with(None), &deployment); - let range = run_explicit_for(&pattern_b_with(Some(LAST_HOUR_BY_MINUTE)), &deployment); - assert_eq!(LAST_HOUR_BY_MINUTE.evaluations_per_request(), 61); - for (id, before, after) in [ - ("P1", 2.457_600_019_2, 149.913_601_171_2), - ("P2", 2.362_500_025_2, 144.112_501_537_2), - ("P3", 3.611_233_350_8, 220.285_234_398_8), - ("P3-m1", 654.542_718_733_8, 7_202.305_718_761_8), - ("P3-m2", 436.587_770_004_24, 6_987.053_970_131_44), - ] { - assert!(close(cost(&none, id), before), "{id}: {}", cost(&none, id)); - assert!(close(cost(&range, id), after), "{id}: {}", cost(&range, id)); - } - // Every query-time computation that is not kept runs once per step. - for id in ["P1", "P2", "P3"] { - assert!(close(cost(&range, id), 61.0 * cost(&none, id)), "{id}"); - } - // B3: 4 → 64 tumbling windows kept between requests; B1: 6 → 66 tumbling windows kept. - let keeps = |run: &Run, id: &str, bytes: &str| { - assert!( - run.selection.costs[id] - .per_node - .values() - .any(|n| n.detail.contains(&format!("; keeps {bytes} bytes"))), - "{id} keeps {bytes} bytes" - ) - }; - keeps(&none, "P3-m2", "3488000000"); - keeps(&range, "P3-m2", "55808000000"); - keeps(&none, "P3-m1", "5232000000"); - keeps(&range, "P3-m1", "57552000000"); - assert_eq!(range.selection.selected, "P2"); -} - -/// (b) For this explicit fixture, refreshing every minute at 30-second steps -/// supplies tumbling windows of -/// `gcd(5 min, 1 min, 30 s) = 30 s`, on the step grid, and Stage 2 keeps -/// them; without the range, the fixture uses 1 min tumbling windows. Both configurations -/// must divide the evaluation step; production also offers finer legal widths. -#[test] -fn explicit_tumbling_window_candidates_follow_the_evaluation_step_grid() { - let step_30s = RangeEvaluation { - step_ms: 30_000, - span_ms: 3_600_000, - }; - let range = run_explicit(&exact_panel("max_over_time(m[5m])", 60_000, Some(step_30s))); - let widths: Vec = tumbling_window_chains(&range) - .into_iter() - .map(|(_, w, _)| w) - .collect(); - assert!(!widths.is_empty(), "tumbling windows are offered"); - assert!(widths.iter().all(|&w| w == 30_000), "{widths:?}"); - assert!(range.pruned.is_empty(), "{:?}", range.pruned); - - let none = run_explicit(&exact_panel("max_over_time(m[5m])", 60_000, None)); - let widths: Vec = tumbling_window_chains(&none) - .into_iter() - .map(|(_, w, _)| w) - .collect(); - assert!( - !widths.is_empty() && widths.iter().all(|&w| w == 60_000), - "{widths:?}" - ); -} - -/// (c) A plan that keeps tumbling windows reports how long the runtime must keep -/// them: the window plus the range-evaluation span, 5 min + 1 h = -/// 3,900,000 ms, so the next refresh's earliest step finds its tumbling windows. -/// Without the range it is the window, 300,000 ms. -#[test] -fn stage3_plan_selection_reports_the_retention_of_kept_tumbling_windows() { - let deployment = keeping_query_time_results(); - let query = "max_over_time(m[5m])"; - let range = run_explicit_for( - &exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE)), - &deployment, - ); - let kept = with_chain( - &range, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - assert_eq!(range.selection.selected, kept); - let retention: Vec = range - .selection - .tumbling_window_retention_ms - .values() - .copied() - .collect(); - assert_eq!(retention, [3_900_000]); - - // Pattern B without a range selects the whole-window KLL, which keeps - // nothing; B1 reports its window when it is the plan (no raw data). - let none = run_explicit(&pattern_b_with(None)); - assert!(none.selection.tumbling_window_retention_ms.is_empty()); - let b1 = run_explicit_for( - &pattern_b_with(None), - &DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }, - ); - let retention: Vec = b1 - .selection - .tumbling_window_retention_ms - .values() - .copied() - .collect(); - assert_eq!(retention, [PATTERN_B_WINDOW_MS]); -} - -/// (d) Without a range evaluation nothing changes: Pattern B's pinned -/// costs (#509 Example 3 and 4: whole-window KLL 2.3625, tumbling windows 3.6112, B3 -/// 436.59, B1 5.232 GB kept) are as before, and a range of one step per -/// request (`span_ms: 0`) prices exactly as no range. -#[test] -fn no_range_evaluation_leaves_every_price_unchanged() { - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&pattern_b_with(None), &deployment); - assert!(close(cost(&none, "P2"), 2.362_500_025_2)); - assert!(close(cost(&none, "P3"), 3.611_233_350_8)); - assert!(close(cost(&none, "P3-m2"), 436.587_770_004_24)); - assert_eq!( - none.selection.costs["P3-m1"].usage.average_memory_bytes, - 5_232_101_733.337_067 - ); - let one_step = run_explicit_for( - &pattern_b_with(Some(RangeEvaluation { - step_ms: 60_000, - span_ms: 0, - })), - &deployment, - ); - assert_eq!(one_step.selection.costs, none.selection.costs); - assert_eq!(one_step.selection.selected, none.selection.selected); -} - -/// The example where declaring the range makes Stage 3 prefer sharing: -/// `max_over_time(m[5m])` every minute over 1M series. As one evaluation -/// per minute, rebuilding the window (P1, 20.8281 cost/s) beats the kept -/// tumbling windows (P2-m2, 32.2630: 224 MB of 56-byte per-series maxima kept) and the -/// tumbling windows maintained at ingestion (P2-m1, 46.2340). As a dashboard over the -/// last hour at 1-minute steps, rebuilding costs 61 times as much -/// (1,270.5151), while the kept tumbling windows add only the per-step merges and 1 h -/// of tumbling windows (460.0410), and B1 472.2760: the kept tumbling windows are selected, and without -/// query-time retention, the ingestion-time tumbling windows. -#[test] -fn stage3_plan_selection_prefers_kept_tumbling_windows_once_the_range_is_declared() { - let query = "max_over_time(m[5m])"; - let deployment = keeping_query_time_results(); - let none = run_explicit_for(&exact_panel(query, 60_000, None), &deployment); - let range = run_explicit_for( - &exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE)), - &deployment, - ); - let kept = with_chain( - &range, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - let ingestion = with_chain(&range, Materialization::MatAtIngestion(Medium::Memory)); - for (run, id, expected) in [ - (&none, "P1", 20.828_117), - (&none, kept.as_str(), 32.262_967), - (&none, ingestion.as_str(), 46.234_035), - (&range, "P1", 1_270.515_127), - (&range, kept.as_str(), 460.040_967), - (&range, ingestion.as_str(), 472.276_035), - ] { - assert!( - (cost(run, id) - expected).abs() < 1e-5, - "{id}: {}", - cost(run, id) - ); - } - assert_eq!(none.selection.selected, "P1"); - assert_eq!(range.selection.selected, kept); - let p1 = range - .selection - .rejected - .iter() - .find(|r| r.id == "P1") - .unwrap(); - assert_eq!(p1.cause, RejectionCause::Costlier); - - let without_retention = run_explicit(&exact_panel(query, 60_000, Some(LAST_HOUR_BY_MINUTE))); - assert_eq!(without_retention.selection.selected, ingestion); -} - -/// (e) A chain over an offset input reaches back its offset further: with -/// no raw data kept, `max_over_time(m[60m] offset 10m)` every 10 minutes -/// keeps six 10-minute tumbling windows ending 10 minutes before each evaluation, so -/// its tumbling windows must be kept for 10 min + 60 min = 4,200,000 ms, and the kept -/// memory Stage 3 prices covers that seventh tumbling window; without the offset the -/// retention is the window, 3,600,000 ms. -#[test] -fn an_offset_chain_is_retained_and_priced_for_its_offset() { - let no_raw_data = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plain = run_explicit_for( - &exact_panel("max_over_time(m[60m])", 600_000, None), - &no_raw_data, - ); - let offset = run_explicit_for( - &exact_panel("max_over_time(m[60m] offset 10m)", 600_000, None), - &no_raw_data, - ); - let retention = |run: &Run| -> Vec { - run.selection - .tumbling_window_retention_ms - .values() - .copied() - .collect() - }; - assert_eq!(retention(&plain), [3_600_000]); - assert_eq!(retention(&offset), [4_200_000]); - let memory = |run: &Run| { - run.selection.costs[&run.selection.selected] - .usage - .average_memory_bytes - }; - assert!( - memory(&offset) > memory(&plain), - "{} vs {}", - memory(&offset), - memory(&plain) - ); -} diff --git a/crates/integration-tests/tests/reuse_rules.rs b/crates/integration-tests/tests/reuse_rules.rs deleted file mode 100644 index 2c8d5a4ad..000000000 --- a/crates/integration-tests/tests/reuse_rules.rs +++ /dev/null @@ -1,589 +0,0 @@ -//! Reuse in Stage 1 (#680): the nested-sum merge (a Pass 1 candidate) is -//! selected over the nested sums and answers the same; the rollup and exact -//! top-k reuse candidates (Pass 2 memo rules) are selected once the finer -//! computation they read is materialized for all its readers, and answer as -//! the computation they replace. Selection prices fixed plans made of the -//! memo's candidates, so no assertion depends on a timed search. -use crate::physical_common; - -use std::collections::{BTreeMap, BTreeSet}; -use std::rc::Rc; - -use asap_devtools::planner_layering_examples::{shared_data_workload, SHARED_ROWS_PER_SECOND}; -use asap_executor::physical_planner::{compile, promql_rows}; -use asap_executor::runtime::Scope; -use asap_executor::values::Value; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{ - cheapest_of, compose_memo_plan, expanded_memo, select_among, -}; -use asap_types::ir::physical_export::compile_physical_asap_dag; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, QueryRoot, -}; -use asap_types::physical::Materialization; -use asap_types::stages::{root_groups, MemoGroupId}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use asap_types::workload::{ - DataArrival, DataWorkload, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, - RepetitionInterval, -}; - -const EVALUATION_MS: i64 = 300_000; - -/// `queries` lowered at `accuracy`, with series identity, as the planner -/// receives them. -fn roots(queries: &[(&str, AccuracyTarget)]) -> Vec { - queries - .iter() - .map(|(query, accuracy)| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - QueryRoot::Operator(root) - }) - .collect() -} - -fn exact_queries<'a>(queries: &[&'a str]) -> Vec<(&'a str, AccuracyTarget)> { - queries - .iter() - .map(|query| (*query, AccuracyTarget::exact())) - .collect() -} - -/// The memo Stage 1 builds for `queries`, each with `recurrence`. -fn memo(queries: &[(&str, AccuracyTarget)], recurrence: QueryRecurrence) -> Memo { - expanded_memo(roots(queries), &vec![exact(recurrence); queries.len()]) -} - -/// The workload entry of an exact query evaluated as `recurrence` says. -fn exact(recurrence: QueryRecurrence) -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Default::default(), - requirements: Default::default(), - predictability: Default::default(), - recurrence, - range_evaluation: None, - } -} - -/// The #509 shared data workload: 1,000,000 series sampled every 15 s. -fn data() -> DataWorkload { - let mut data = shared_data_workload( - DataArrival::Ingesting { - rows_per_second: SHARED_ROWS_PER_SECOND, - }, - "m", - ); - // 100 jobs: each `by (job)` group holds many series, so a ranking - // shared between two top-k saves work. - data.sources.get_mut("m").unwrap().distinct_values_per_label = - [("job".to_string(), 100)].into(); - data -} - -/// Every minute. -fn repeating() -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))) -} - -/// Two jobs of two series each, a sample every 15 s over [0, 5m). -fn samples() -> Vec<(&'static str, &'static str, i64, f64)> { - let mut rows = vec![]; - for (s, (job, series)) in [("a", "a1"), ("a", "a2"), ("b", "b1"), ("b", "b2")] - .into_iter() - .enumerate() - { - for step in 0..20i64 { - rows.push(( - job, - series, - step * 15_000, - (s as i64 * 100 + step) as f64 + 0.5, - )); - } - } - rows -} - -/// Execute `root` at query time over the `samples` bound as raw data at its -/// `Scan`; each `TimeRange` keeps its own window (a tumbling window's moved -/// back by the `TimeShift` below it). -fn execute(root: &Rc) -> Vec { - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - // Tumbling windows run as their tumbling windows, each over its own window. - let timed = - asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(&timed) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let store = physical_common::raw_data_store(&dag, |_, schema| { - samples() - .into_iter() - .map(|(job, series, ts, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("job".to_string(), job.to_string()), - ("series".to_string(), series.to_string()), - ]); - promql_rows::series_row(schema, &labels, ts, value).unwrap() - }) - .collect() - }); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let mut rows: Vec = physical_common::execute( - &plan, - store, - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - ) - .remove(0) - .iter() - .flat_map(|batch| batch.to_rows().to_vec()) - .map(|row: Vec| format!("{row:?}")) - .collect(); - rows.sort(); - rows -} - -fn summary_builds(root: &Rc) -> usize { - OperatorNode::reachable(root) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .count() -} - -/// Every candidate of every group of `memo` with origin in `origins`, with -/// the group it implements; groups that answer a query or aggregate only. -fn candidates_of<'a>( - memo: &'a Memo, - origins: &'a [OriginOfCandidateForMemoGroup], -) -> impl Iterator)> + 'a { - let root_groups = root_groups(&memo.query_roots); - memo.candidates - .iter() - .enumerate() - .filter(move |(group, _)| { - let is_root = root_groups.contains(&MemoGroupId(*group as u32)); - let group = &memo.groups[*group]; - is_root - || matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) - }) - .flat_map(move |(group, candidates)| { - candidates - .iter() - .filter(move |c| origins.contains(&c.origin)) - .map(move |c| (group, &c.root_node)) - }) -} - -/// Plan selection's choice among fixed plans of `memo`'s `queries`: every -/// group on its first candidate (`group`), and `variant{i}` per entry of -/// `variants`, a group's candidate index each. Returns the selected plan's -/// name, its roots, whether it builds some computation once per evaluation -/// for all its readers (`MatAtQuery`), and the selection's lowest prices of -/// `group` and of every variant. -fn select( - memo: &Memo, - queries: usize, - entry: QueryWorkloadEntry, - variants: &[BTreeMap], -) -> (String, Vec>, bool, f64, f64) { - let mut plans = vec![( - "group".to_string(), - compose_memo_plan(memo, queries, &BTreeMap::new()), - )]; - for (i, choice) in variants.iter().enumerate() { - plans.push(( - format!("variant{i}"), - compose_memo_plan(memo, queries, choice), - )); - } - let named: Vec<(&str, Vec)> = plans - .iter() - .map(|(name, roots)| { - ( - name.as_str(), - roots.iter().cloned().map(QueryRoot::Operator).collect(), - ) - }) - .collect(); - let (physical, selection) = select_among(&named, &vec![entry; queries], &data()); - let selected = physical - .iter() - .find(|plan| plan.id == selection.selected) - .expect("the selected plan"); - let name = selected.from_logical.clone(); - let built_once = selected - .materialization - .values() - .any(|m| matches!(m, Materialization::MatAtQuery(_))); - let roots = plans - .iter() - .find(|(n, _)| *n == name) - .map(|(_, roots)| roots.clone()) - .unwrap(); - let variants = (0..variants.len()) - .map(|i| cheapest_of(&selection, &format!("variant{i}"))) - .fold(f64::INFINITY, f64::min); - let own = cheapest_of(&selection, "group"); - // A rejected baseline would make "cheaper than the baseline" hold trivially. - assert!(own.is_finite(), "the plan without reuse is valid"); - (name, roots, built_once, own, variants) -} - -/// `(group, candidate index)` of every candidate [`candidates_of`] yields. -fn choices(memo: &Memo, origins: &[OriginOfCandidateForMemoGroup]) -> Vec<(usize, usize)> { - candidates_of(memo, origins) - .map(|(group, node)| { - let index = memo.candidates[group] - .iter() - .position(|c| Rc::ptr_eq(&c.root_node, node)) - .unwrap(); - (group, index) - }) - .collect() -} - -/// [`candidates_of`], without the candidates that are their group's own -/// node. -fn own_candidates<'a>( - memo: &'a Memo, - origins: &'a [OriginOfCandidateForMemoGroup], -) -> impl Iterator)> + 'a { - candidates_of(memo, origins) - .filter(|(group, node)| !Rc::ptr_eq(node, &memo.groups[*group].logical_node)) -} - -/// `sum by (job) (sum_over_time(m[5m]))`: the nested sums and their merge -/// into one accumulator over the raw data samples. Plan selection picks the -/// merge, which builds no per-series state, and every candidate answers the -/// same per-job sums. -#[test] -fn nested_sum_merge_is_selected_and_answers_as_the_nested_sums() { - // Evaluated once, so no tumbling-window candidates. - let query = "sum by (job) (sum_over_time(m[5m]))"; - let memo = memo(&exact_queries(&[query]), QueryRecurrence::Unknown); - let variants: Vec<_> = choices(&memo, &[OriginOfCandidateForMemoGroup::Pass1]) - .into_iter() - .map(|(group, index)| BTreeMap::from([(group, index)])) - .collect(); - let (_, roots, ..) = select(&memo, 1, exact(QueryRecurrence::Unknown), &variants); - assert_eq!(summary_builds(&roots[0]), 1); - let answer = execute(&roots[0]); - assert_eq!(answer.len(), 2, "one sum per job: {answer:?}"); - every_candidate_answers_alike(query); -} - -/// Kinds of the nodes, other than the source chain, that query 0's and -/// query 1's roots share: what one query reads of the other's computation. -fn shared_kinds(roots: &[Rc]) -> BTreeSet { - let first: BTreeSet<_> = OperatorNode::reachable(&roots[0]) - .iter() - .map(Rc::as_ptr) - .collect(); - OperatorNode::reachable(&roots[1]) - .into_iter() - .filter(|node| first.contains(&Rc::as_ptr(node))) - .map(|node| node.operator.kind_name().to_string()) - .filter(|kind| !["Scan", "TimeRange", "TimeShift", "Project"].contains(&kind.as_str())) - .collect() -} - -/// The nodes the plan selected for `queries` shares between its two -/// queries, among the queries' own roots and the plans reading `origin` -/// candidates (the `i`-th of each group that has some, together). Stage 2 -/// offers each computation two queries read built once per evaluation -/// (`MatAtQuery`); the reuse plan is selected and is cheaper than without it. -fn selected_reuse(queries: &[&str], origin: OriginOfCandidateForMemoGroup) -> BTreeSet { - let memo = memo(&exact_queries(queries), repeating()); - let mut by_group: BTreeMap> = BTreeMap::new(); - for (group, index) in choices(&memo, std::slice::from_ref(&origin)) { - by_group.entry(group).or_default().push(index); - } - let most = by_group.values().map(Vec::len).max().unwrap_or(0); - assert!(most > 0, "{queries:?}: some {origin:?} candidate"); - let variants: Vec> = (0..most) - .map(|i| { - by_group - .iter() - .map(|(group, indices)| (*group, indices[i.min(indices.len() - 1)])) - .collect() - }) - .collect(); - let (name, roots, built_once, own, reuse) = - select(&memo, queries.len(), exact(repeating()), &variants); - assert!(name.starts_with("variant"), "{queries:?}: selected {name}"); - assert!( - built_once, - "{queries:?}: {name} computes its shared state once" - ); - assert!(reuse < own, "{queries:?}: reuse {reuse} vs own {own}"); - shared_kinds(&roots) -} - -/// `sum by (job)` beside `sum by (job, instance)`: with the finer sums -/// computed once, plan selection answers the coarse sum from them. -#[test] -fn rollup_is_selected_when_the_finer_sums_are_computed_once() { - let shared = selected_reuse( - &["sum by (job) (m)", "sum by (job, instance) (m)"], - OriginOfCandidateForMemoGroup::Rollup, - ); - // The coarse sum reads the finer sums' state or finalized values. - assert!( - shared.contains("Aggregate") || shared.contains("SummaryAgg"), - "{shared:?}" - ); -} - -/// Two exact top-k of the per-series rates: with the shared ranking -/// computed once, plan selection reads both from one ranking instead of -/// sorting every series twice. -#[test] -fn exact_topk_reuse_is_selected_when_the_ranking_is_computed_once() { - let shared = selected_reuse( - &[ - "topk by (job) (5, rate(m[1m]))", - "topk by (job) (10, rate(m[1m]))", - ], - OriginOfCandidateForMemoGroup::TopKReuse, - ); - assert!(shared.contains("Sort"), "{shared:?}"); -} - -/// `node` as Stage 2 implements it for `entry` (its first -/// materialization), executed. -fn execute_implemented(node: &Rc, entry: &QueryWorkloadEntry) -> Vec { - let plans = asap_physical_asap_optimization::optimize_physical_asap( - "candidate", - &[QueryRoot::Operator(Rc::clone(node))], - std::slice::from_ref(entry), - &data(), - false, - ) - .expect("Stage 2 implements the candidate"); - match &plans[0].roots[0] { - QueryRoot::Operator(root) => execute(root), - QueryRoot::Scalar(_) => panic!("operator root"), - } -} - -/// Every memo candidate of `query`'s aggregate and root groups other than -/// the group's own node executes to the same rows as that node. -fn every_candidate_answers_alike(query: &str) { - let memo = memo(&exact_queries(&[query]), QueryRecurrence::Unknown); - let entry = exact(QueryRecurrence::Unknown); - let mut checked = 0; - for (group, node) in own_candidates(&memo, &[OriginOfCandidateForMemoGroup::Pass1]) { - assert_eq!( - execute_implemented(node, &entry), - execute_implemented(&memo.groups[group].logical_node, &entry), - "{query}: a candidate of group {group} differs" - ); - checked += 1; - } - assert!( - checked > 0, - "{query}: some candidate other than the group's node" - ); -} - -/// A per-series sum drops the metric name, so grouping by `__name__` over -/// the nested sums groups every series under the empty name; a merge over -/// the raw data samples would group them under `m`. Every candidate answers as -/// the nested sums. -#[test] -fn nested_sum_by_the_metric_name_answers_as_the_nested_sums() { - every_candidate_answers_alike("sum by (__name__) (sum_over_time(m[5m]))"); - every_candidate_answers_alike("sum by (job) (sum_over_time(m[5m]))"); -} - -/// Every rollup or top-k reuse candidate in `memo`, of a group only the -/// queries before `first_approximate_query` read, executes to the rows of -/// the computation it replaces (its group's logical node). -fn reuse_answers_alike_in( - memo: &Memo, - queries: &[(&str, AccuracyTarget)], - first_approximate_query: usize, -) { - let entry = exact(repeating()); - let mut checked = 0; - for (group, node) in candidates_of( - memo, - &[ - OriginOfCandidateForMemoGroup::Rollup, - OriginOfCandidateForMemoGroup::TopKReuse, - ], - ) { - if memo.groups[group] - .reached_by_queries() - .iter() - .any(|&q| q >= first_approximate_query) - { - continue; - } - assert_eq!( - execute_implemented(node, &entry), - execute_implemented(&memo.groups[group].logical_node, &entry), - "{queries:?}: a reuse candidate of group {group} differs" - ); - checked += 1; - } - assert!(checked > 0, "{queries:?}: some reuse candidate"); -} - -/// [`reuse_answers_alike_in`] the memo of the exact `queries`. -fn reuse_answers_alike(queries: &[(&str, AccuracyTarget)]) { - reuse_answers_alike_in(&memo(queries, repeating()), queries, queries.len()); -} - -/// Coarse sums (over merged nested sums), minima and maxima rolled up from -/// the finer ones answer as computed from the raw data samples. -#[test] -fn rollups_answer_as_the_coarse_computations() { - for queries in [ - [ - "sum by (job) (sum_over_time(m[5m]))", - "sum by (job, series) (sum_over_time(m[5m]))", - ], - ["min by (job) (m)", "min by (job, series) (m)"], - ["max by (job) (m)", "max by (job, series) (m)"], - ] { - reuse_answers_alike(&exact_queries(&queries)); - } -} - -/// Top-k read from one larger ranking answer as computed alone. -#[test] -fn exact_topk_reuse_answers_as_each_topk() { - reuse_answers_alike(&exact_queries(&[ - "topk by (job) (1, sum_over_time(m[5m]))", - "topk by (job) (2, sum_over_time(m[5m]))", - ])); -} - -/// Query `q`'s rows when Stage 2 implements `roots` together (its first -/// materialization) for `entries`. -fn execute_plan( - roots: &[Rc], - entries: &[QueryWorkloadEntry], - q: usize, -) -> Vec { - let roots: Vec<_> = roots.iter().cloned().map(QueryRoot::Operator).collect(); - let plans = asap_physical_asap_optimization::optimize_physical_asap( - "plan", - &roots, - entries, - &data(), - false, - ) - .expect("Stage 2 implements the plan"); - match &plans[0].roots[q] { - QueryRoot::Operator(root) => execute(root), - QueryRoot::Scalar(_) => panic!("operator root"), - } -} - -/// A fine and a coarse sum beside two differently grouped quantiles: a plan -/// that shares the quantiles' summary build (summary-capability rule) and -/// rolls the coarse sum up from the fine one (rollup rule) answers each sum -/// as its own computation does (from the raw data samples), and each -/// quantile as the same shared build without the rollup. -#[test] -fn a_rollup_over_a_shared_summary_candidate_answers_as_without_it() { - let quantile = AccuracyTarget::max_error(0.05).failure_probability(0.3); - let queries = [ - ( - "sum by (job) (sum_over_time(m[5m]))", - AccuracyTarget::exact(), - ), - ( - "sum by (job, series) (sum_over_time(m[5m]))", - AccuracyTarget::exact(), - ), - ( - "avg by (job, series) (quantile_over_time(0.99, m[5m]))", - quantile.clone(), - ), - ("avg by (job) (quantile_over_time(0.95, m[5m]))", quantile), - ]; - let memo = memo(&queries, repeating()); - let shared = memo - .groups - .iter() - .find(|group| { - group.reached_by_queries().len() > 1 - && group.reached_by_queries().iter().all(|&q| q >= 2) - && matches!(group.logical_node.asap(), Some(ASAPOp::SummaryAgg { .. })) - }) - .map(|group| Rc::clone(&group.logical_node)) - .expect("the quantiles share a summary build"); - // Each quantile's own candidate that reads the shared build. - let mut sharing: BTreeMap = [2, 3] - .into_iter() - .map(|q| { - memo.candidates - .iter() - .enumerate() - .filter(|(group, _)| memo.groups[*group].reached_by_queries() == [q]) - .flat_map(|(group, candidates)| { - candidates - .iter() - .enumerate() - .map(move |(i, c)| (group, i, c)) - }) - .find(|(_, _, c)| { - OperatorNode::reachable(&c.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &shared)) - }) - .map(|(group, i, _)| (group, i)) - .unwrap_or_else(|| panic!("Q{} reads the shared build", q + 1)) - }) - .collect(); - let without = compose_memo_plan(&memo, queries.len(), &sharing); - let (group, index) = choices(&memo, &[OriginOfCandidateForMemoGroup::Rollup]) - .into_iter() - .find(|(group, _)| memo.groups[*group].reached_by_queries() == [0]) - .expect("a rollup of the coarse sum"); - sharing.insert(group, index); - let with = compose_memo_plan(&memo, queries.len(), &sharing); - let own = compose_memo_plan(&memo, queries.len(), &BTreeMap::new()); - assert!( - OperatorNode::reachable(&with[2]) - .iter() - .any(|node| OperatorNode::reachable(&with[3]) - .iter() - .any(|other| Rc::ptr_eq(node, other) - && matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })))), - "both quantiles read one summary build" - ); - let entries = vec![exact(repeating()); queries.len()]; - for q in 0..2 { - assert_eq!( - execute_plan(&with, &entries, q), - execute_plan(&own, &entries, q), - "Q{}", - q + 1 - ); - } - for q in 2..4 { - assert_eq!( - execute_plan(&with, &entries, q), - execute_plan(&without, &entries, q), - "Q{}", - q + 1 - ); - } -} diff --git a/crates/integration-tests/tests/rollup_controlled.rs b/crates/integration-tests/tests/rollup_controlled.rs index d646ded6b..1178513c4 100644 --- a/crates/integration-tests/tests/rollup_controlled.rs +++ b/crates/integration-tests/tests/rollup_controlled.rs @@ -1,18 +1,13 @@ //! Controlled plans for the viewer's actual distinct-count rollup workload. use asap_cost::{CostInputs, CostModel, ResourceUnitCost}; -use asap_integration_tests::planning::{builds_and_rollups, expanded_memo, select_rollup}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, rollup_candidates, - rollup_pass2_candidates, -}; + use asap_planner::PlanningModels; -use asap_types::stages::HowToComputeAggregateMeasure; + use asap_types::{ ir::{ - schema::{DataType, Field, Schema, SketchAlgorithm}, + schema::{DataType, Field, Schema}, ASAPOp, OperatorNode, QueryRoot, }, - physical::{Materialization, Medium}, workload::PlanningWorkload, }; use std::{collections::HashSet, rc::Rc}; @@ -46,122 +41,6 @@ fn nodes(roots: &[QueryRoot]) -> Vec> { .collect() } -#[test] -fn controlled_distinct_rollup_prices_the_original_workload() { - let (workload, roots) = fixture(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let generated = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut settings = CostInputs::default(); - settings.cost_model = CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - settings.resource_speed, - )); - let models = PlanningModels::builtin().with_cost_inputs(&settings); - let mut accepted = Vec::new(); - for algorithm in [None, Some(SketchAlgorithm::Hll), Some(SketchAlgorithm::Kmv)] { - let choices: Vec<_> = generated - .targets - .iter() - .map(|target| match &algorithm { - None => 0, - Some(a) => target - .ways_to_compute - .iter() - .position( - |c| matches!(c,HowToComputeAggregateMeasure::Summary(found) if found==a), - ) - .expect("family generated"), - }) - .collect(); - let roots: Vec<_> = compose_logical_candidate(&generated, &choices).unwrap(); - // Identical sub-DAGs made one node, as the memo's groups are. - let roots: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let mut variants = vec![("separate", roots.clone())]; - if algorithm.is_some() { - let rules = rollup_candidates(&roots); - assert_eq!(rules.len(), 1, "one service rollup for {algorithm:?}"); - for rolled in rollup_pass2_candidates(&roots, &rules).unwrap() { - variants.push(("rollup", rolled.roots.unwrap())); - } - } - for (kind, roots) in variants { - let family = algorithm - .as_ref() - .map_or("exact".to_string(), |a| format!("{a:?}")); - let id = format!("{family}-{kind}"); - let builds: HashSet<_> = nodes(&roots) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .map(Rc::as_ptr) - .collect(); - let mut plan = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - &id, - &roots, - &entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |node, _, options| { - let mat = Materialization::MatAtQuery(Medium::Memory); - Some(if builds.contains(&node) && options.contains(&mat) { - mat - } else { - Materialization::NotMaterialized - }) - }, - ) - .unwrap() - .expect("fixed query-time implementation"); - plan.id = id.clone(); - let result = asap_devtools::memo_pricing::reference_select( - &[plan.clone()], - &entries, - data, - models, - ); - match result { - Ok(selection) => { - println!( - "{id}: builds={} cost={:?} accepted={:?}", - builds.len(), - selection.costs[&id].cost_per_second, - selection.accepted - ); - accepted.push(plan); - } - Err(error) => println!("{id}: REJECTED {error}"), - } - for n in nodes(&roots) { - if let Some(ASAPOp::SummaryAgg { family, .. }) = n.asap() { - println!("{id} family={family:?}"); - } - } - } - } - let chosen = - asap_devtools::memo_pricing::reference_select(&accepted, &entries, data, models).unwrap(); - println!("CONTROLLED WINNER {}", chosen.selected); - assert_eq!( - chosen.selected, "Kmv-rollup", - "shared KMV must win the controlled cost comparison" - ); -} - // The production entrypoint must find a sketch rollup under the viewer budget. #[test] fn production_distinct_rollup_finds_shared_kmv() { @@ -210,18 +89,3 @@ fn production_distinct_rollup_finds_shared_kmv() { "production search must select the cheaper legal rollup" ); } - -// The production shape without a timed search: among fixed plans of the -// memo (a KMV per query, or both reading one KMV state group), plan -// selection with the built-in models picks one build and one rollup, and -// it costs less than a valid build per query. -#[test] -fn memo_offers_one_kmv_build_and_a_rollup() { - let (workload, roots) = fixture(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.as_ref().unwrap(); - let memo = expanded_memo(roots, &entries); - let (selected, separate, shared) = select_rollup(&memo, 2, &entries, data); - assert!(shared < separate, "{shared} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} diff --git a/crates/integration-tests/tests/runtime_handoff.rs b/crates/integration-tests/tests/runtime_handoff.rs deleted file mode 100644 index cadfe8788..000000000 --- a/crates/integration-tests/tests/runtime_handoff.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! What `plan_installable` returns for a deployment runtime (issue #749), -//! checked against the Planner's internals; the runtime's own end-to-end -//! run, with public APIs only, is `crates/planner/tests/runtime_install.rs`. - -use std::collections::BTreeSet; - -use asap_executor::physical_planner::compile; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_planner::{ - e2e_plan, plan_installable, test_support::query_time_reference, FrontendInput, - InstallableDagError, InstallablePlan, InstallablePlanError, PlanError, PlanOutput, - PlanningModels, SelectedPlanReport, UserInput, -}; -use asap_types::ir::physical_export::{ - compile_physical_asap_workload, PhysicalASAPDAG, PhysicalASAPNodeId, -}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::physical::Materialization; -use asap_types::workload::PlanningWorkload; - -/// #509 Example 4b: one repeating `quantile_over_time` over a 60-minute -/// window, evaluated every 10 minutes, over ingesting data. -fn example4b() -> PlanningWorkload { - serde_json::from_str( - r#"{ - "query_workload": { - "language": "promql", - "repeating_queries": [{ - "query": "quantile_over_time(0.99, latency_ms[60m])", - "demand": {"fixed_interval": 600000}, - "requirements": { - "accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}, - "max_latency_ms": 200.0 - }, - "predictability": {"predictable": {"known_at": null}} - }] - }, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 1000.0}}, - "sources": {"latency_ms": {"series_count": 1000, "sample_interval": 1000}} - } - }"#, - ) - .expect("Example 4b parses") -} - -/// A deployment that keeps no raw data samples: every summary must be built at -/// ingestion time. -fn without_raw_data() -> DeploymentCapabilities { - DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - } -} - -async fn plan(workload: &PlanningWorkload, capabilities: &DeploymentCapabilities) -> PlanOutput { - let models = PlanningModels::builtin().with_deployment_capabilities(capabilities); - e2e_plan(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .expect("Example 4b plans") -} - -fn materialized_count(dag: &PhysicalASAPDAG) -> usize { - dag.nodes - .iter() - .filter(|n| n.materialization != Materialization::NotMaterialized) - .count() -} - -/// Without raw data Stage 3 selects ingestion-time tumbling windows; the facade's -/// export carries that selection rather than resetting every node to -/// query time. -#[tokio::test] -async fn export_carries_the_selected_materialization() { - let capabilities = without_raw_data(); - let output = plan(&example4b(), &capabilities).await; - let dag = output.physical_asap_dag().expect("exports"); - assert!( - materialized_count(&dag) > 0, - "the selected tumbling windows are stored" - ); - assert!(dag - .nodes - .iter() - .any(|n| n.output_state.timing == ExecutionTiming::IngestionTime)); - // Exactly Stage 3's selected plan, as Stage 3 timed it. - assert!(output.selection.is_some(), "Stage 3 selected"); - assert_eq!( - dag, - compile_physical_asap_workload(&output.roots()).expect("the selected plan exports") - ); -} - -/// What a runtime that keeps no raw data samples gets from one facade call: -/// the plan and Stage 3's report of the selection. -async fn installable_with_report( - workload: &PlanningWorkload, -) -> (InstallablePlan, Option) { - let capabilities = without_raw_data(); - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - plan_installable(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .expect("Example 4b plans") -} - -async fn installable(workload: &PlanningWorkload) -> InstallablePlan { - installable_with_report(workload).await.0 -} - -fn tumbling_window_chain_count(dag: &PhysicalASAPDAG) -> usize { - dag.nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .count() -} - -/// The installable plan expands its selected tumbling-window width into contiguous -/// tumbling windows, each a materialized view refreshed at ingestion time, and -/// the executor compiles it. The stage export keeps the chain for costing -/// and display. Its one query is answered by the DAG's one root. -#[tokio::test] -async fn installable_plan_has_its_tumbling_window_chains_expanded() { - let workload = example4b(); - let (installed, report) = installable_with_report(&workload).await; - installed.validate().unwrap(); - let plan = &installed; - let dag = &installed.dag; - let views = plan.materialized_views().unwrap(); - assert_eq!(tumbling_window_chain_count(dag), 0); - assert_eq!(materialized_count(dag), views.len()); - let roots: Vec = dag.roots.iter().map(|&r| r as u64).collect(); - let compiled = compile(dag, &views, &roots).expect("the executor compiles it"); - assert_eq!(plan.query_texts.len(), 1); - assert_eq!( - plan.query_texts[0], - "quantile_over_time(0.99, latency_ms[60m])" - ); - // The query reads exactly the materialized views, and no raw data. - let refreshed: BTreeSet = views.iter().map(|v| v.node_id as u64).collect(); - let slots: BTreeSet = compiled.input_slots().map(|(id, _)| id).collect(); - assert_eq!(slots, refreshed); - assert_tumbling_window_coverage(plan, dag.roots[0], 0, 3_600_000); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw data kept" - ); - let report = report.as_ref().expect("Stage 3 selected"); - assert!(report.cost_per_second.is_some()); - - let staged = plan_for_stages(&workload).await; - assert_eq!( - tumbling_window_chain_count(&staged), - 1, - "the stage export keeps the chain" - ); -} - -async fn plan_for_stages(workload: &PlanningWorkload) -> PhysicalASAPDAG { - plan(workload, &without_raw_data()) - .await - .physical_asap_dag() - .unwrap() -} - -/// Each summary build of `dag` (`SummaryAgg` or `SummaryTumblingWindowChain`) as its JSON -/// payload, sorted: which families with which parameters it builds. -fn summary_builds(dag: &PhysicalASAPDAG) -> Vec { - let mut builds: Vec = dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) - }) - .map(|n| serde_json::to_string(&n.payload).unwrap()) - .collect(); - builds.sort(); - builds -} - -/// The correctness reference of the installable plan: the same summaries -/// with the same parameters, every node at query time and nothing materialized, -/// with the same queries bound to its roots. -#[tokio::test] -async fn reference_plan_runs_the_installable_plan_at_query_time() { - let plan = installable(&example4b()).await; - let reference = query_time_reference(&plan).expect("re-times"); - reference.validate().unwrap(); - assert_eq!(materialized_count(&reference.dag), 0); - assert!(reference.materialized_views().unwrap().is_empty()); - // The reference reads the raw data the plan's views were refreshed from. - let [retention] = reference.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(3_600_000)); - assert!(reference - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - assert_eq!(summary_builds(&reference.dag), summary_builds(&plan.dag)); - assert_eq!(reference.query_texts, plan.query_texts); -} - -/// A repeating PromQL workload over ingesting `latency_ms`: `queries`, each -/// every 10 minutes at rank error 0.01, δ 0.01. -fn repeating(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|q| { - format!( - r#"{{"query": {q:?}, "demand": {{"fixed_interval": 600000}}, - "requirements": {{"accuracy": {{"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}, - "max_latency_ms": 200.0}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// p99 and p50 over the same 60 minutes share one chain of tumbling windows: the -/// installable plan materializes one shared set of tumbling windows. -#[tokio::test] -async fn two_queries_on_one_chain_store_its_tumbling_windows_once() { - let plan = installable(&repeating(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.5, latency_ms[60m])", - ])) - .await; - assert_eq!(plan.query_texts.len(), 2); - let views = plan.materialized_views().unwrap(); - let views_below = |root| { - let below = nodes_below(&plan.dag, root); - views - .iter() - .filter(|view| below.contains(&view.node_id)) - .map(|view| view.node_id) - .collect::>() - }; - assert_eq!( - views_below(plan.dag.roots[0]), - views_below(plan.dag.roots[1]) - ); - assert_eq!(views_below(plan.dag.roots[0]).len(), views.len()); - assert_tumbling_window_coverage(&plan, plan.dag.roots[0], 0, 3_600_000); -} - -/// Planned with raw data kept (everything at query time), so each query's -/// range is a query-time raw-data window. -async fn installable_at_query_time(query: &str) -> Result { - let workload = repeating(&[query]); - plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin(), - )) - .await - .map(|(plan, _)| plan) -} - -/// A raw-data window whose position is not a fixed offset from the -/// evaluation time (a subquery or an offset above the range, an `@` -/// anchor below it), or that ends after the evaluation (a negative offset, -/// which no ingestion run can build in time), is refused with its reason, -/// not passed on with a wrong window or dropped; a plain range is listed. -#[tokio::test] -#[ignore = "fails on main too: the `@` query's selection MILP is infeasible (its scan has no rows_held to price); not a wall-clock issue; #1015"] -async fn raw_data_windows_of_unsupported_shapes_are_refused() { - for (query, reason) in [ - ( - "max_over_time(rate(latency_ms[5m])[1h:1m])", - "a subquery or a TimeShift above the range", - ), - ( - "max_over_time(latency_ms[10m] @ 1000)", - "an @ anchor below the range", - ), - ( - "max_over_time(latency_ms[60m] offset -10m)", - "a negative offset below the range", - ), - ] { - match installable_at_query_time(query).await { - Err(PlanError::Installable(InstallablePlanError::Uninstallable( - InstallableDagError::UnsupportedRawDataWindow { - reason: refused, .. - }, - ))) => assert_eq!(refused, reason, "{query}"), - Err(error) => { - // When planning fails before installation, its no-plan - // diagnostic must keep the concrete refusal reason. - let diagnostic = error.to_string(); - assert!(diagnostic.contains(reason), "{query}: {diagnostic}"); - assert!(diagnostic.contains("raw-data window"), "{diagnostic}"); - } - Ok(plan) => panic!( - "{query}: unexpectedly installed {:?}", - plan.raw_data_retention().unwrap() - ), - } - } - let plain = installable_at_query_time("max_over_time(latency_ms[10m] offset 5m)") - .await - .unwrap(); - // 10 minutes of data ending 5 minutes back reaches 15 minutes back. - let [retention] = plain.raw_data_retention().unwrap().try_into().unwrap(); - assert_eq!(retention.lookback_ms(), Some(900_000)); -} - -/// Example 4b answered as the last hour at 10-minute steps (a range -/// evaluation): each selected tumbling window view is kept for the hour its window reads -/// plus the hour the range spans, 7,200,000 ms, as Stage 3 records for the -/// chain (`Selection::tumbling_window_retention_ms`); without the range, for the hour. -#[tokio::test] -async fn stored_tumbling_windows_carry_stage3_tumbling_window_retention() { - let mut workload = example4b(); - let plain = installable(&workload).await; - assert!(plain - .materialized_views() - .unwrap() - .iter() - .all(|v| v.retention_ms() == 3_600_000)); - let mut json = serde_json::to_value(&workload).unwrap(); - json["query_workload"]["repeating_queries"][0]["range_evaluation"] = - serde_json::json!({"step_ms": 600000, "span_ms": 3600000}); - workload = serde_json::from_value(json).unwrap(); - let ranged = installable(&workload).await; - assert_tumbling_window_coverage(&ranged, ranged.dag.roots[0], 0, 3_600_000); - assert!(ranged - .materialized_views() - .unwrap() - .iter() - .all(|v| v.retention_ms() == 7_200_000)); - let output = plan(&workload, &without_raw_data()).await; - assert_eq!( - output - .tumbling_window_retention_ms - .values() - .copied() - .collect::>(), - [7_200_000] - ); - // A plan file that keeps a tumbling window for less than it reads is refused. - let mut short = ranged.clone(); - let view = ranged.materialized_views().unwrap()[0].node_id; - short - .dag - .nodes - .iter_mut() - .find(|n| n.id == view) - .unwrap() - .retention_ms = Some(0); - assert!(matches!( - short.validate(), - Err(InstallablePlanError::Uninstallable( - InstallableDagError::ViewRetentionTooShort { .. } - )) - )); -} - -/// A chain over an offset input keeps its tumbling windows for its window plus the -/// offset: `[60m] offset 10m` reaches 70 minutes back (4,200,000 ms), the -/// unshifted query's chain 60 minutes (3,600,000 ms). Each materialized view -/// carries its own node's requirement; a runtime that keeps the twins' -/// identical tumbling windows once keeps them for the larger one. -#[tokio::test] -async fn an_offset_chain_keeps_its_tumbling_windows_for_its_window_plus_the_offset() { - let plan = installable(&repeating(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.99, latency_ms[60m] offset 10m)", - ])) - .await; - assert_tumbling_window_coverage(&plan, plan.dag.roots[0], 0, 3_600_000); - assert_tumbling_window_coverage(&plan, plan.dag.roots[1], 600_000, 4_200_000); - let below: Vec<_> = plan - .dag - .roots - .iter() - .map(|&root| nodes_below(&plan.dag, root)) - .collect(); - for view in &plan.materialized_views().unwrap() { - let required = [3_600_000, 4_200_000] - .into_iter() - .zip(&below) - .filter(|(_, nodes)| nodes.contains(&view.node_id)) - .map(|(retention, _)| retention) - .max() - .expect("materialized tumbling window is consumed"); - assert_eq!(view.retention_ms(), required); - } - plan.validate().unwrap(); -} - -/// Any selected legal tumbling-window width must cover the requested interval exactly. -fn assert_tumbling_window_coverage( - plan: &InstallablePlan, - root: PhysicalASAPNodeId, - start: i64, - end: i64, -) { - let below = nodes_below(&plan.dag, root); - let mut intervals: Vec<_> = plan - .materialized_views() - .unwrap() - .iter() - .filter(|view| below.contains(&view.node_id)) - .map(|view| { - let [window] = view.inputs.raw_data_windows() else { - panic!("tumbling window reads one raw interval: {:?}", view.inputs); - }; - assert!(window.width_ms() > 0 && 600_000 % window.width_ms() == 0); - ( - window.end_offset_ms(), - window.end_offset_ms() + window.width_ms(), - ) - }) - .collect(); - intervals.sort_unstable(); - assert!(!intervals.is_empty()); - assert_eq!(intervals[0].0, start); - assert_eq!(intervals.last().unwrap().1, end); - for pair in intervals.windows(2) { - assert_eq!(pair[0].1, pair[1].0); - } -} - -/// Every node `root` reads, directly or through other nodes. -fn nodes_below(dag: &PhysicalASAPDAG, root: PhysicalASAPNodeId) -> BTreeSet { - let mut below = BTreeSet::new(); - let mut pending = vec![root]; - while let Some(node) = pending.pop() { - for edge in dag.edges.iter().filter(|edge| edge.consumer == node) { - if below.insert(edge.producer) { - pending.push(edge.producer); - } - } - } - below -} - -/// A standalone scalar query (`scalar(sum(latency_ms))`) has no DAG node -/// to answer it through the public API, so it is refused with its entry. -#[tokio::test] -async fn a_scalar_query_is_refused() { - match installable_at_query_time("scalar(sum(latency_ms))").await { - Err(PlanError::Installable(InstallablePlanError::UnsupportedQueryRoot { - entry_index: 0, - })) => {} - other => panic!("{:?}", other.map(|p| p.query_texts)), - } -} diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs index 04555299e..2ad36086e 100644 --- a/crates/integration-tests/tests/sharing_issue_regressions.rs +++ b/crates/integration-tests/tests/sharing_issue_regressions.rs @@ -2,9 +2,9 @@ use asap_cost::CostInputs; use asap_devtools::planner_fuzz::{read_seed_file, FuzzSettings, FuzzWorkload, Planner}; use asap_devtools::planning_stages::run_search_over; -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; + use asap_planner::{PlanSearchSettings, PlanningModels}; -use asap_types::stages::HowToComputeAggregateMeasure; + use serde_json::json; use std::path::Path; @@ -48,7 +48,6 @@ fn cost(w: &FuzzWorkload) -> f64 { &w.data, models, PlanSearchSettings::default(), - None, ) .unwrap(); let selection = run.run.selection.as_ref().unwrap(); @@ -58,8 +57,8 @@ fn cost(w: &FuzzWorkload) -> f64 { } /// #923: deriving the coarse sum costs only a sum over the 150 fine groups. -/// Stage 2 offers that plan; Stage 3 keeps a `MatAtQuery` build whose -/// `NotMaterialized` sibling would avoid a second scan (#1142). +/// Stage 2 offers that plan; Stage 3 selects a `OneEvaluation` build whose +/// not-materialized sibling would avoid a second scan (#1142). #[test] #[ignore = "Stage 3 misses the cheaper plan Stage 2 offers (#1142)"] fn exact_coarse_sum_reuses_fine_sum() { @@ -111,362 +110,6 @@ fn grouped_filtered_quantile_plans() { ); } -/// #886: SQL count-ranked TopK offers a heap over raw rows, absorbing the count. -#[test] -fn sql_count_topk_has_a_fused_raw_row_heap() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - let roots = Planner::new(FuzzSettings::default()) - .lower(&workload(886)) - .unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let (index, target) = inventory - .targets - .iter() - .enumerate() - .find(|(_, target)| target.fused_computation.is_some()) - .expect("SQL TopK must offer a fused raw-row heap"); - for choice in target.ways_to_compute.len()..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - let composed = asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &selected, - ) - .unwrap(); - let roots: Vec<_> = composed; - let root = roots[0].as_operator().unwrap(); - for node in asap_types::ir::OperatorNode::reachable(root) { - if let Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(_, _), - input, - child, - .. - }) = node.asap() - { - assert_eq!( - input.weight, - asap_types::ir::schema::SummaryInputExpr::Constant(1.0) - ); - assert!(!child.contains_asap(), "heap consumes raw rows"); - } - } - let guarded = Planner::new(FuzzSettings::default()).price(&workload(886), &roots, false); - assert!( - guarded.unwrap_err().contains("NoAccuracyGuarantee"), - "frequency intervals must not be claimed as Top-K membership guarantees" - ); - use asap_executor::values::Value; - let rows = (0..7) - .map(|_| { - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("one-host".into()), - Value::Float64(1.0), - ] - }) - .collect(); - let actual = physical_common::execute_raw_rows_at(root, rows, 1000); - assert_eq!(actual.len(), 1, "{actual:?}"); - assert!( - actual[0] - .iter() - .any(|v| matches!(v, Value::Utf8(s) if s.as_ref() == "one-host")), - "{actual:?}" - ); - assert!( - actual[0] - .iter() - .any(|v| matches!(v, Value::Float64(count) if *count == 7.0)), - "{actual:?}" - ); - let compiled = Planner::new(FuzzSettings { - enforce_accuracy_guarantees: false, - ..Default::default() - }) - .price(&workload(886), &roots, false); - assert!(compiled.is_ok(), "fused heap must compile: {compiled:?}"); - } -} - -/// #923: label projection differences below the same per-series average do not -/// prevent an exact sum from rolling up the finer sum's output. -#[test] -fn exact_rollup_catalog_crosses_identical_per_series_average() { - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(923)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let owner = expanded.query_roots[1].as_operator().unwrap().0 as usize; - assert!( - expanded.candidates[owner] - .iter() - .any(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup), - "coarse root has no rollup candidate" - ); -} - -use crate::physical_common; - -/// #902: selecting one group preserves values and SQL's one NULL row when -/// the group is absent, including empty inputs and groups of NULL samples. -#[test] -fn filtered_group_preserves_sql_empty_and_null_semantics() { - use asap_executor::values::Value; - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::ir::{operator::AggIntent, NonASAPOp}; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&workload(902)) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let (owner, candidate) = expanded.candidates.iter().enumerate().flat_map(|(owner, candidates)| candidates.iter().map(move |c| (owner,c))) - .find(|(_, c)| c.origin == OriginOfCandidateForMemoGroup::Rollup && matches!(c.root_node.non_asap(), Some(NonASAPOp::Aggregate { measures, child, .. }) - if matches!(measures.as_slice(), [AggIntent::Min { .. }]) && matches!(child.non_asap(), Some(NonASAPOp::Filter { .. })))) - .expect("filtered group read candidate"); - // The same readout keeps a KLL's rank bound: it selects a whole group, - // never predicates on an approximate value or combines populations. - use asap_accuracy::propagation::recompose_guarantees; - use asap_logical_asap_optimization::test_support::compose_logical_candidate; - use asap_types::ir::{schema::SketchAlgorithm, QueryRoot}; - use std::rc::Rc; - let filter = candidate.root_node.children()[0]; - let grouped = filter.children()[0]; - let inv = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::clone(grouped)), - ]), - ) - .unwrap(); - let choices: Vec<_> = inv - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap_or(0) - }) - .collect(); - let approximate = compose_logical_candidate(&inv, &choices).unwrap().remove(0); - let approximate = approximate.as_operator().unwrap(); - let filtered = Rc::new( - filter - .with_new_children(|_| Rc::clone(approximate)) - .unwrap(), - ); - let readout = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&filtered)) - .unwrap(), - ); - let recomposed = recompose_guarantees(&[QueryRoot::Operator(readout)]).unwrap(); - assert!(approximate.guarantee.is_some()); - assert_eq!( - recomposed[0].as_operator().unwrap().guarantee, - approximate.guarantee - ); - // A range of keys can select multiple populations; their extrema do not - // inherit one selected quantile's rank bound. - let mut many_groups = (*filtered).clone(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred, .. }) = - &mut many_groups.operator - { - if let asap_types::ir::ScalarExpr::Compare { op, .. } = &mut pred.0 { - *op = asap_types::ir::scalar::CompareOpKind::Gt; - } - } - let many_groups = Rc::new(many_groups); - let invalid = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&many_groups)) - .unwrap(), - ); - assert!( - recompose_guarantees(&[QueryRoot::Operator(invalid)]).unwrap()[0] - .as_operator() - .unwrap() - .guarantee - .is_none() - ); - // A filter that also reads the estimate decides by an approximate value - // which group is kept, so the selected group's bound does not hold. - let mut on_estimate = (*filtered).clone(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { pred, .. }) = - &mut on_estimate.operator - { - pred.0 = asap_types::ir::ScalarExpr::BoolAnd(vec![ - pred.0.clone(), - asap_types::ir::ScalarExpr::IsNotNull(Box::new(asap_types::ir::ScalarExpr::Column(1))), - ]); - } - let on_estimate = Rc::new(on_estimate); - let invalid = Rc::new( - candidate - .root_node - .with_new_children(|_| Rc::clone(&on_estimate)) - .unwrap(), - ); - assert!( - recompose_guarantees(&[QueryRoot::Operator(invalid)]).unwrap()[0] - .as_operator() - .unwrap() - .guarantee - .is_none() - ); - let row = |city: &str, value| { - vec![ - Value::Timestamp(0), - Value::Utf8(city.into()), - Value::Utf8("host".into()), - value, - ] - }; - for rows in [ - vec![], - vec![row("LA", Value::Float64(5.0))], - vec![row("NYC", Value::Null)], - vec![ - row("NYC", Value::Float64(10.0)), - row("NYC", Value::Float64(30.0)), - row("LA", Value::Float64(90.0)), - ], - ] { - let before = physical_common::execute_raw_rows_at( - &expanded.groups[owner].logical_node, - rows.clone(), - 1000, - ); - let after = physical_common::execute_raw_rows_at(&candidate.root_node, rows, 1000); - assert_eq!(format!("{after:?}"), format!("{before:?}")); - assert_eq!(after.len(), 1); - } -} - -/// #923: SQL counts roll up by merging integer states, without float casts. -#[test] -fn sql_count_rollup_executes_with_integer_results() { - use asap_executor::values::Value; - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::ir::ASAPOp; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let mut w = workload(902); - w.queries.truncate(2); - w.queries[0].entry.query.0 = - "SELECT city, host, COUNT(*) AS c FROM metrics GROUP BY city, host".into(); - for query in [ - "SELECT city, COUNT(*) AS c FROM metrics GROUP BY city", - "SELECT COUNT(*) AS c FROM metrics", - ] { - w.queries[1].entry.query.0 = query.into(); - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&w) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups( - memo, - ) - .unwrap(); - let (owner, candidate) = expanded - .candidates - .iter() - .enumerate() - .flat_map(|(owner, candidates)| candidates.iter().map(move |c| (owner, c))) - .find(|(_, c)| { - c.origin == OriginOfCandidateForMemoGroup::Rollup - && asap_types::ir::OperatorNode::reachable(&c.root_node) - .iter() - .any(|node| { - matches!(node.asap(), Some(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(child.asap(), Some(ASAPOp::SummaryMerge { group_by:Some(_), .. }))) - }) - }) - .expect("integer count state rollup candidate"); - let rows = vec![ - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("a".into()), - Value::Null, - ], - vec![ - Value::Timestamp(0), - Value::Utf8("NYC".into()), - Value::Utf8("b".into()), - Value::Float64(10.0), - ], - vec![ - Value::Timestamp(0), - Value::Null, - Value::Utf8("a".into()), - Value::Null, - ], - ]; - let before = - physical_common::execute_raw_rows(&expanded.groups[owner].logical_node, rows.clone()); - let after = physical_common::execute_raw_rows(&candidate.root_node, rows); - let sorted = |rows: Vec>| { - let mut rows: Vec<_> = rows.iter().map(|row| format!("{row:?}")).collect(); - rows.sort(); - rows - }; - assert!(after - .iter() - .all(|row| matches!(row.last(), Some(Value::Int64(_))))); - assert_eq!(sorted(before), sorted(after)); - let before_empty = - physical_common::execute_raw_rows(&expanded.groups[owner].logical_node, vec![]); - let after_empty = physical_common::execute_raw_rows(&candidate.root_node, vec![]); - assert_eq!(sorted(before_empty), sorted(after_empty)); - } -} - /// #922: duplicate exact TopK readers share their result and inner window work. #[test] fn duplicate_exact_topk_shares_work() { @@ -505,68 +148,3 @@ fn sql_sum_shares_nested_windows_and_slower_readers() { "{repeated_cost} vs {single}" ); } - -/// #923: sketches over an inner range function expose the same finer rollup. -#[test] -fn sketch_rollup_crosses_inner_range_function() { - use asap_logical_asap_optimization::test_support::{ - generate_per_computation_candidates, try_populate_pass1, - }; - use asap_types::stages::OriginOfCandidateForMemoGroup; - let mut w = workload(923); - for (query, keys) in w.queries.iter_mut().zip(["job, region", "job"]) { - query.entry.query.0 = format!("quantile by ({keys}) (0.95, sum_over_time(m_gauge[5m]))"); - query.entry.requirements.accuracy = - asap_types::ir::properties::AccuracyTarget::max_error(0.05).failure_probability(0.01); - } - let roots: Vec<_> = Planner::new(FuzzSettings::default()) - .lower(&w) - .unwrap() - .into_iter() - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let memo = try_populate_pass1(initial, &inventory).unwrap(); - let expanded = - asap_logical_asap_optimization::test_support::sharing_rules_and_shared_state_groups(memo) - .unwrap(); - let owner = expanded.query_roots[1].as_operator().unwrap().0 as usize; - assert!(expanded.candidates[owner] - .iter() - .any(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup)); -} - -/// #886: unsupported per-group SQL count sketches are explicitly pruned; -/// the exact counter and separately typed raw-row TopK fusion remain available. -#[test] -fn sql_grouped_count_sketches_are_not_offered() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_types::ir::{operator::AggIntent, NonASAPOp}; - use asap_types::stages::PruneReason; - let w = workload(886); - let roots = Planner::new(FuzzSettings::default()).lower(&w).unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let (index, target) = inventory.targets.iter().enumerate().find(|(_, target)| matches!(inventory.aggregate(target).non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) if matches!(measures.as_slice(), [AggIntent::Count { .. }]))).unwrap(); - assert!(target - .ways_to_compute - .iter() - .all(|candidate| !matches!(candidate, HowToComputeAggregateMeasure::Summary(_)))); - assert_eq!( - inventory - .pruned_of_target(index, false) - .iter() - .filter(|(_, reason)| *reason == PruneReason::NoRelationalCountSketchRealization) - .count(), - 2 - ); - assert!(inventory - .targets - .iter() - .any(|target| target.fused_computation.is_some())); -} diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs index a5044bd34..6546ede02 100644 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ b/crates/integration-tests/tests/sketch_rollup.rs @@ -4,7 +4,6 @@ //! the planner offers a SQL grouped query, the rolled-up answer equals the //! direct build's (deterministic merges) or is within the sketch's bound //! (KLL), and its coverage is the direct build's. -use crate::physical_common; use std::collections::BTreeMap; use std::rc::Rc; @@ -84,7 +83,7 @@ fn as_float64(node: &Rc) -> Rc { /// The answers of the logical DAG `root` over [`rows`], by group key. fn execute(root: &Rc) -> BTreeMap { let root = &as_float64(root); - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -249,7 +248,7 @@ async fn direct_and_rolled_up( ); let rolled = replace(&candidate, &build, &merge); rolled.validate_structure().unwrap(); - out.push((*kind.algorithm(), candidate, rolled)); + out.push((*kind, candidate, rolled)); } out } diff --git a/crates/integration-tests/tests/sketch_rollup_planning.rs b/crates/integration-tests/tests/sketch_rollup_planning.rs deleted file mode 100644 index 2519479b7..000000000 --- a/crates/integration-tests/tests/sketch_rollup_planning.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! Sketch rollup in planning: the memo's rollup rule offers answering a -//! coarse sketch grouping from the finest build of the same sketch over the -//! same rows, and plan selection selects it when one build and a merge cost -//! less than a build per query over the raw rows. Selection prices fixed -//! plans made of the memo's candidates, so no assertion depends on a timed -//! search. -use asap_executor::physical_planner::promql_rows; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{builds_and_rollups, expanded_memo, select_rollup}; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::QueryRoot; -use asap_types::workload::{ - DataWorkload, Predictability, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, - RepeatedDemand, RepetitionInterval, -}; - -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "http", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("user_id", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ]), - ) -} - -/// A predictable query repeating every `interval_ms` at `accuracy`. -fn entry(accuracy: &AccuracyTarget, interval_ms: u32) -> QueryWorkloadEntry { - QueryWorkloadEntry { - query: Default::default(), - requirements: QueryRequirements { - accuracy: accuracy.clone(), - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - interval_ms, - ))), - range_evaluation: None, - } -} - -/// `http` arrives at 2e6 rows/s (W-RollupHeavy's table), 10,000 series, -/// a million distinct users at most per evaluation, and holds a minute of -/// rows (the catalog has no time index, so a query's time predicate is a -/// filter over every row held). -fn data() -> DataWorkload { - serde_json::from_value(serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 2_000_000.0}}, - "sources": {"http": { - "series_count": 10_000, - "sample_interval": 5, - "values_non_negative": true, - "max_distinct_per_evaluation": 1_000_000, - "rows_held": 120_000_000, - }}, - })) - .unwrap() -} - -/// The SQL `queries` lowered at `accuracy`. -async fn sql_roots(queries: &[&str], accuracy: &AccuracyTarget) -> Vec { - let mut roots = Vec::new(); - for query in queries { - let root = lower_sql(query, &catalog(), accuracy.clone()) - .await - .unwrap(); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -/// Distinct users by (region, service) and by service, both at relative -/// ε = 0.05 per value over the same window (W-RollupHeavy's targets): the -/// memo offers the coarse KMV rolled up from the finer one, and plan -/// selection selects one build and a rollup, cheaper than a build per -/// query from the raw rows. -#[tokio::test] -async fn stage3_selects_a_distinct_count_rollup_when_it_is_cheaper() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative) - .accuracy_scope(AccuracyScope::PerValue); - let window = "WHERE ts >= now() - INTERVAL '5 minutes'"; - let roots = sql_roots( - &[ - &format!( - "SELECT region, service, COUNT(DISTINCT user_id) AS value FROM http {window} \ - GROUP BY region, service" - ), - &format!( - "SELECT service, COUNT(DISTINCT user_id) AS value FROM http {window} \ - GROUP BY service" - ), - ], - &accuracy, - ) - .await; - let entries = vec![entry(&accuracy, 60_000); 2]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 2, &entries, &data()); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} - -/// p99 latency by (region, service), by region and by service at rank -/// ε = 0.05 (template 13's shape): both coarse groupings roll up from one -/// KLL, and plan selection selects that plan. -#[tokio::test] -async fn coarse_quantile_groupings_roll_up_from_one_kll() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(AccuracyScope::PerValue); - let window = "WHERE ts >= now() - INTERVAL '5 minutes'"; - let query = |keys: &str| { - format!( - "SELECT {keys}, approx_percentile_cont(latency, 0.99) AS value FROM http {window} \ - GROUP BY {keys}" - ) - }; - let roots = sql_roots( - &[ - &query("region, service"), - &query("region"), - &query("service"), - ], - &accuracy, - ) - .await; - let entries = vec![entry(&accuracy, 60_000); 3]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 3, &entries, &data()); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 2)); -} - -/// #840's `quantile by (host, job)` and `quantile by (job)` over 10,000 -/// series, every 10 s at rank ε = 0.01, δ = 0.001: the memo offers the -/// coarse KLL rolled up from the finer one, and its guarantee meets the -/// target, so plan selection selects it. -#[test] -fn promql_quantile_by_job_rolls_up_from_by_host_job() { - let accuracy = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank); - let roots: Vec = [ - "quantile by (host, job) (0.5, data)", - "quantile by (job) (0.5, data)", - ] - .iter() - .map(|query| { - let root = lower_promql(query, accuracy.clone()).unwrap(); - QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()) - }) - .collect(); - let data: DataWorkload = serde_json::from_value(serde_json::json!({ - "arrival": {"ingesting": {"rows_per_second": 666.67}}, - "sources": {"data": {"series_count": 10_000, "metric_type": "gauge", - "sample_interval": 15_000, - // 15 minutes of raw rows. - "rows_held": 600_000}}, - })) - .unwrap(); - let entry = entry(&accuracy, 10_000); - let entries = [entry.clone(), entry]; - let memo = expanded_memo(roots, &entries); - let (selected, separate, rolled_up) = select_rollup(&memo, 2, &entries, &data); - assert!(rolled_up < separate, "{rolled_up} vs {separate}"); - assert_eq!(builds_and_rollups(&selected), (1, 1)); -} diff --git a/crates/integration-tests/tests/sql_group_bound.rs b/crates/integration-tests/tests/sql_group_bound.rs deleted file mode 100644 index e8497c4fa..000000000 --- a/crates/integration-tests/tests/sql_group_bound.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! A whole-result accuracy target over SQL `GROUP BY` groups (#863): the -//! distinct values the data workload declares for the grouping columns -//! (`SourceFacts::distinct_values_per_label`) bound the groups one -//! evaluation returns, so a per-group sketch can be sized to meet the target -//! over all of them. Without declarations the groups stay unbound and only -//! the exact plan meets it. - -use crate::planner_layering_common; - -use asap_devtools::planning_stages::lower_sql_roots; -use asap_devtools::SqlCatalog; -use asap_types::ir::schema::{DataType, Field, Schema, SketchAlgorithm}; -use asap_types::workload::PlanningWorkload; -use planner_layering_common::*; - -/// The `http` table of the RollupHeavy workload. -fn http_catalog() -> SqlCatalog { - let utf8 = |name| Field::plain(name, DataType::Utf8, false); - let float = |name| Field::plain(name, DataType::Float64, false); - SqlCatalog::new().with_table( - "http", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - utf8("region"), - utf8("service"), - utf8("endpoint"), - utf8("status"), - float("user_id"), - float("latency"), - ], - 0, - vec![], - ), - ) -} - -/// `query` every minute with a whole-result target (`metric` error 0.05, -/// failure probability 0.05) over `http`: 10,000 label combinations, and -/// `declared` as each column's distinct values. -fn workload(query: &str, metric: &str, declared: serde_json::Value) -> PlanningWorkload { - let http = serde_json::json!({ - "series_count": 10000, "sample_interval": 5, "values_non_negative": true, - "max_distinct_per_evaluation": 1000000, "distinct_values_per_label": declared, - "rows_held": 1000000000 - }); - serde_json::from_value(serde_json::json!({ - "query_workload": {"language": {"sql": "datafusion_sql"}, "query_batch": null, - "repeating_queries": [{ - "query": query, "demand": {"fixed_interval": 60000}, - "requirements": {"accuracy": {"accuracy_metric": metric, "max_error": 0.05, - "failure_probability": 0.05, "accuracy_scope": "whole_result"}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 2e6}}, - "sources": {"http": http}} - })) - .expect("workload") -} - -/// Every `http` column's distinct values: 4 regions, 25 services, 25 -/// endpoints, 4 statuses. -fn every_column() -> serde_json::Value { - serde_json::json!({"region": 4, "service": 25, "endpoint": 25, "status": 4}) -} - -/// Evaluate the exact computation and each supported local summary choice. -/// These tests exercise group/error models, not a timed search's visited archive. -fn run_group_choices(workload: &PlanningWorkload) -> Run { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots = lower_sql_roots(workload, &http_catalog()).expect("lowers"); - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - assert_eq!(local.targets.len(), 1, "one grouped statistic per fixture"); - let candidates = (0..local.targets[0].ways_to_compute.len()) - .map(|choice| compose_logical_candidate(&local, &[choice]).unwrap()) - .collect(); - run_explicit_candidates( - workload, - roots, - candidates, - asap_planner::PlanningModels::builtin(), - ) -} - -/// The sketches the selected plan builds (empty for the exact plan). -fn selected_sketches( - query: &str, - metric: &str, - declared: serde_json::Value, -) -> Vec { - let workload = workload(query, metric, declared); - let run = run_group_choices(&workload); - sketch_builds(run.physical(&run.selection.selected).dag()) - .into_iter() - .map(|(_, algorithm, _)| algorithm) - .collect() -} - -/// Template 14: distinct users per (service, endpoint). -const DISTINCT_USERS: &str = "SELECT service, endpoint, COUNT(DISTINCT user_id) AS value \ - FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service, endpoint"; - -/// Template 13: the p99 latency per (region, service). -const P99_LATENCY: &str = "SELECT region, service, approx_percentile_cont(latency, 0.99) AS value \ - FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY region, service"; - -/// With 25 services and 25 endpoints declared, at most 625 groups: a -/// distinct-count sketch per group, each failing with probability at most -/// 0.05 / 625, meets the whole-result target. -#[test] -fn declared_grouping_columns_bound_the_groups_of_a_distinct_count() { - let sketches = selected_sketches(DISTINCT_USERS, "relative", every_column()); - assert!( - matches!( - sketches.as_slice(), - [SketchAlgorithm::Hll | SketchAlgorithm::Kmv] - ), - "{sketches:?}" - ); -} - -/// With 4 regions and 25 services declared, at most 100 groups: a KLL per -/// group meets the whole-result rank target. -#[test] -fn declared_grouping_columns_bound_the_groups_of_a_quantile() { - let sketches = selected_sketches(P99_LATENCY, "rank", every_column()); - assert_eq!(sketches, [SketchAlgorithm::Kll]); -} - -/// Without declared distinct values the groups are unbound: no sketch -/// meets a whole-result target, and the exact plan is selected. -#[test] -fn undeclared_grouping_columns_leave_only_the_exact_plan() { - for (query, metric) in [(DISTINCT_USERS, "relative"), (P99_LATENCY, "rank")] { - let sketches = selected_sketches(query, metric, serde_json::json!({})); - assert!(sketches.is_empty(), "{query}: {sketches:?}"); - } -} - -/// One grouping column declared is not enough: `service` has no declared -/// count, so the groups of `GROUP BY region, service` stay unbound. -#[test] -fn a_partly_declared_grouping_leaves_only_the_exact_plan() { - let sketches = selected_sketches(P99_LATENCY, "rank", serde_json::json!({"region": 4})); - assert!(sketches.is_empty(), "{sketches:?}"); -} - -/// A computed key named like a declared column no longer holds that -/// column's values: `upper(service) AS service` gives no bound. -#[test] -fn a_computed_grouping_column_leaves_only_the_exact_plan() { - let query = "SELECT service, COUNT(DISTINCT user_id) AS value FROM \ - (SELECT concat(service, endpoint) AS service, user_id, ts FROM http) \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; - let sketches = selected_sketches(query, "relative", every_column()); - assert!(sketches.is_empty(), "{sketches:?}"); -} - -/// The groups every priced plan's grouped builds and aggregates are priced -/// at (`... into N states`, `... into N groups`), and the rows its scans read. -fn priced_groups_and_scan_rows( - workload: PlanningWorkload, -) -> ( - std::collections::BTreeSet, - std::collections::BTreeSet, -) { - let run = run_group_choices(&workload); - let (mut groups, mut scanned) = ( - std::collections::BTreeSet::new(), - std::collections::BTreeSet::new(), - ); - for cost in run.selection.costs.values() { - for node in cost.per_node.values() { - let words: Vec<&str> = node.detail.split([' ', ':', ';']).collect(); - for window in words.windows(3) { - if window[0] == "into" && matches!(window[2], "states" | "groups") { - groups.insert(window[1].parse().expect("a group count")); - } - if window[0] == "scan" && window[2] == "samples" { - scanned.insert(window[1].parse().expect("a row count")); - } - } - } - } - (groups, scanned) -} - -/// A SQL grouping with an undeclared column gets no invented count: each -/// of the table's 10,000 declared groups of rows is in one group, so its -/// builds are priced at 10,000; without `series_count`, each row read is -/// in at most one group, so they are priced at the rows the scan reads. -/// Declared, `GROUP BY service, endpoint` is 25 x 25. -#[test] -fn an_undeclared_sql_grouping_is_priced_at_the_groups_of_rows_or_the_rows_read() { - let undeclared = || workload(DISTINCT_USERS, "relative", serde_json::json!({})); - let (groups, _) = priced_groups_and_scan_rows(undeclared()); - assert_eq!(groups, std::collections::BTreeSet::from([10_000])); - let mut without_series = undeclared(); - let http = without_series - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut("http") - .unwrap(); - http.series_count = None; - http.sample_interval = None; - let (groups, scanned) = priced_groups_and_scan_rows(without_series); - assert_eq!(groups, scanned); - let (groups, _) = - priced_groups_and_scan_rows(workload(DISTINCT_USERS, "relative", every_column())); - assert_eq!(groups, std::collections::BTreeSet::from([625])); -} - -/// Each endpoint is in one service: declaring `(service, endpoint)`'s 625 -/// combinations prices `GROUP BY service, endpoint` at 625 groups, where -/// the per-column product (25 x 625) is capped only by the 10,000 groups of -/// rows. -#[test] -fn a_declared_column_set_prices_a_sql_grouping_at_its_combinations() { - let hierarchical = - serde_json::json!({"region": 4, "service": 25, "endpoint": 625, "status": 4}); - let (groups, _) = - priced_groups_and_scan_rows(workload(DISTINCT_USERS, "relative", hierarchical.clone())); - assert_eq!(groups, std::collections::BTreeSet::from([10_000])); - let mut declared = workload(DISTINCT_USERS, "relative", hierarchical); - declared - .data_workload - .as_mut() - .unwrap() - .sources - .get_mut("http") - .unwrap() - .distinct_values_per_label_set = vec![asap_types::workload::LabelSetDistinctValues { - labels: vec!["service".to_string(), "endpoint".to_string()], - distinct_values: 625, - }]; - let (groups, _) = priced_groups_and_scan_rows(declared); - assert_eq!(groups, std::collections::BTreeSet::from([625])); -} diff --git a/crates/integration-tests/tests/sql_to_physical.rs b/crates/integration-tests/tests/sql_to_physical.rs deleted file mode 100644 index 258821b74..000000000 --- a/crates/integration-tests/tests/sql_to_physical.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! SQL frontend, the planner pipeline's selection, physical compilation and -//! fresh-run execution. -use crate::physical_common; -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::selected_dag; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use futures::StreamExt; -use physical_common::compile_physical_asap_dag; -use std::sync::Arc; - -/// SQL filtering and grouped aggregation survive logical/physical lowering; -/// rebinding the compiled DAG runs against new data rather than cached results. -#[tokio::test] -async fn sql_filter_grouped_sum_executes_and_rebinds() { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, true), - ]), - ); - for query in [ - "SELECT service, SUM(value) AS total FROM metrics WHERE value > 1 GROUP BY service", - "SELECT service, SUM(value) AS total FROM metrics GROUP BY service", - ] { - let logical = lower_sql(query, &catalog, AccuracyTarget::exact()) - .await - .unwrap(); - let selected = selected_dag(logical, AccuracyTarget::exact()); - let dag = compile_physical_asap_dag(&selected).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::Scan { .. }) - ) - }) - .expect("raw SQL scan"); - let schema = Arc::new(scan.output_schema.clone()); - assert!(schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - for multiplier in [1., 2.] { - let rows = [ - ("api", Some(2.)), - ("api", Some(3.)), - ("api", None), - ("batch", Some(4.)), - ("batch", Some(1.)), - ] - .into_iter() - .map(|(service, value)| { - schema - .fields - .iter() - .map(|field| match field.name.as_str() { - "service" => Value::Utf8(service.into()), - "value" => { - value.map_or(Value::Null, |value| Value::Float64(value * multiplier)) - } - _ => panic!("unexpected field {field:?}"), - }) - .collect() - }) - .collect(); - let PhysicalASAPOperatorPayload::NonASAP(asap_types::ir::NonASAPOp::Scan { - source, - predicates: _, - schema: _scan_schema, - }) = &scan.payload - else { - unreachable!() - }; - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), rows).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let mut stream = bound - .execute( - plan.roots(), - RunContext::new( - Scope::Query { - evaluation_time_ms: 300_000, - }, - Limits::default(), - ) - .unwrap(), - ) - .unwrap() - .remove(0); - let mut batches = Vec::new(); - while let Some(batch) = stream.next().await { - batches.push(batch.unwrap()); - } - let mut actual: Vec<_> = batches - .iter() - .flat_map(|batch| batch.to_rows()) - .map(|row| { - let Value::Utf8(service) = &row[0] else { - panic!("missing service") - }; - let Value::Float64(value) = row[1] else { - panic!("missing sum") - }; - (service.to_string(), value) - }) - .collect(); - actual.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!( - actual, - vec![ - ("api".into(), 5. * multiplier), - ( - "batch".into(), - 4. * multiplier - + if query.contains("WHERE") && multiplier == 1. { - 0. - } else { - multiplier - } - ) - ] - ); - } - } -} diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 5f894c98e..411f5b612 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -30,8 +30,8 @@ use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, - SketchParams, SketchStatistic, SummaryUpdate, + ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchStatistic, + SummaryUpdate, }; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; use asap_types::workload::SqlDialect; @@ -593,16 +593,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { }; assert_eq!( family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); assert_eq!( input, @@ -619,16 +610,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { ); assert_eq!( summary_input.schema.fields[0].dtype, - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 269, - failure_probability: None - } - ), - GroupingStrategy::default() - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) ); assert!( @@ -690,16 +672,7 @@ async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { }; assert_eq!( family, - &FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 18, - relative_error: Some(0.01) - } - ), - GroupingStrategy::default() - ) + &FieldDataType::Sketch(SketchAlgorithm::Hll, GroupingStrategy::default()) ); assert_eq!( input, diff --git a/crates/integration-tests/tests/sql_trailing_window.rs b/crates/integration-tests/tests/sql_trailing_window.rs deleted file mode 100644 index 81a229329..000000000 --- a/crates/integration-tests/tests/sql_trailing_window.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! A SQL trailing window, `WHERE ts >= now() - INTERVAL 'w'` on the table's -//! time index, plans like a PromQL range selector `m[w]`: Pass 2 builds its -//! summaries in tumbling windows, Stage 2 maintains them at ingestion time, -//! a deployment that keeps no raw data still gets a plan, and Stage 3 prices -//! each scan by its window. - -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_logical_asap_optimization::DeploymentCapabilities; -use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, Operator, QueryRoot}; -use asap_types::workload::{ - DataArrival, DataWorkload, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, - SourceFacts, SqlDialect, -}; -use planner_layering_common::*; - -/// The 5 and 15 minute distinct counts of W-RollupHeavy's template 9. -const QUERIES: [&str; 2] = [ - "SELECT dst_subnet, COUNT(DISTINCT src_addr) AS value FROM flows \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY dst_subnet", - "SELECT dst_subnet, COUNT(DISTINCT src_addr) AS value FROM flows \ - WHERE ts >= now() - INTERVAL '15 minutes' GROUP BY dst_subnet", -]; -const WINDOWS_MS: [u64; 2] = [300_000, 900_000]; - -/// `flows(ts, dst_subnet, src_addr)`, time index `ts`. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "flows", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("dst_subnet", DataType::Utf8, false), - Field::plain("src_addr", DataType::Float64, false), - ], - 0, - vec![], - ), - ) -} - -/// Both queries every 60 s, relative ε = 0.05 per value, over 1,000 rows/s. -fn workload() -> PlanningWorkload { - let entry = |query: &str| RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(60_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget { - accuracy_metric: Some(AccuracyMetric::Relative), - accuracy_scope: AccuracyScope::PerValue, - ..AccuracyTarget::max_error(0.05).failure_probability(0.05) - }, - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }; - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::SQL(SqlDialect::DataFusionSQL), - query_batch: None, - repeating_queries: Some(QUERIES.iter().map(|q| entry(q)).collect()), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: [( - "flows".to_string(), - SourceFacts { - max_distinct_per_evaluation: Some(1_000_000), - ..SourceFacts::default() - }, - )] - .into(), - }), - } -} - -async fn roots(workload: &PlanningWorkload) -> Vec { - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - roots -} - -async fn run(deployment: &DeploymentCapabilities) -> Run { - let workload = workload(); - let roots = roots(&workload).await; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let compose = |choices: &[usize]| compose_logical_candidate(&local, choices).unwrap(); - let exact = compose(&vec![0; local.targets.len()]); - let choices = local - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|choice| { - matches!( - choice, - HowToComputeAggregateMeasure::Summary( - asap_types::ir::schema::SketchAlgorithm::Kmv - ) - ) - }) - .expect("KMV distinct-count sketch with a failure bound") - }) - .collect::>(); - let summary = compose(&choices); - let grids = workload - .query_workload - .entries() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let tumbling_windows = tumbling_window_candidates(&summary, &grids); - assert_eq!(tumbling_windows.len(), WINDOWS_MS.len()); - let mut candidates = vec![exact, summary.clone()]; - candidates.extend( - window_composition_pass2_candidates(&summary, &tumbling_windows) - .unwrap() - .into_iter() - .filter(|candidate| { - candidate.tumbling_window_candidates.len() == tumbling_windows.len() - }) - .map(|candidate| candidate.roots.unwrap()), - ); - let settings = CostInputs::default(); - run_explicit_candidates( - &workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_deployment_capabilities(deployment), - ) -} - -/// Pass 2 offers each query's distinct-count sketch in tumbling windows, and -/// Stage 2 maintains a tumbling-window chain at ingestion time, as for `m[5m]`. -#[tokio::test] -async fn sql_windows_get_tumbling_windows_and_ingestion_time_builds() { - let run = run(&DeploymentCapabilities::default()).await; - for (query, window_ms) in WINDOWS_MS.iter().enumerate() { - let tumbling = run.logical.iter().any(|c| { - sketch_builds(&c.dag).iter().any(|(build, _, _)| { - readers(&c.dag, &c.query_roots, *build).contains(&query) - && matches!(window_form(&c.dag, *build), WindowForm::Tumbling { .. }) - }) - }); - assert!( - tumbling, - "query {query} ({window_ms} ms) has no tumbling-window candidate" - ); - } - let at_ingestion = run.physical.iter().any(|p| { - p.dag().nodes.iter().any(|n| { - matches!( - n.payload, - Operator::ASAP(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) && runs_at_ingestion(p, n.id) - }) - }); - assert!( - at_ingestion, - "no tumbling-window chain maintained at ingestion time" - ); -} - -/// Without raw data at query time, the ingestion-time builds still answer -/// both queries, so the workload has a plan (`run_stages_with` panics when -/// none is selected) and it reads no raw scan at query time. -#[tokio::test] -async fn sql_windows_have_a_plan_without_raw_data() { - let deployment = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let run = run(&deployment).await; - let selected = run.physical(&run.selection.selected); - for n in &selected.dag().nodes { - if matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) { - assert!(runs_at_ingestion(selected, n.id), "{}", selected.id); - } - } -} - -/// Each query-time scan is priced by the window above it: the 15 minute -/// query's scan reads 3x the rows of the 5 minute one. -#[tokio::test] -async fn sql_scans_are_priced_by_their_window() { - let run = run(&DeploymentCapabilities::default()).await; - let mut scanned_rows = std::collections::BTreeSet::new(); - for (id, cost) in &run.selection.costs { - let p = run.physical(id); - for (node, node_cost) in &cost.per_node { - let is_query_time_scan = p.dag().nodes.iter().any(|n| { - n.id == *node - && matches!(n.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) - && !runs_at_ingestion(p, n.id) - }); - if !is_query_time_scan { - continue; - } - let rows: u64 = node_cost - .detail - .strip_prefix("scan ") - .and_then(|d| d.split_whitespace().next()) - .and_then(|n| n.parse().ok()) - .unwrap_or_else(|| panic!("scan detail: {}", node_cost.detail)); - scanned_rows.insert(rows); - } - } - // 1 row per ms over each window. - for window_ms in WINDOWS_MS { - assert!( - scanned_rows.contains(&window_ms), - "no scan of {window_ms} rows: {scanned_rows:?}" - ); - } -} diff --git a/crates/integration-tests/tests/sql_two_percentiles.rs b/crates/integration-tests/tests/sql_two_percentiles.rs deleted file mode 100644 index 6ae9ff645..000000000 --- a/crates/integration-tests/tests/sql_two_percentiles.rs +++ /dev/null @@ -1,555 +0,0 @@ -//! Repeating SQL percentiles over one table, every 10 s at ε = 0.01. Q1, -//! the overall p50 and p90 over the last 5 minutes: every candidate that -//! computes one of its percentiles exactly and the other from a sketch -//! compiles, is priced, and answers the exact one as `percentile_cont` does -//! and the sketched one within the sketch's bound. With Q2, the p90 per -//! city over the last 20 minutes, Q1 and Q2 share the table scan, and Q2's -//! per-city tumbling-window chain serves Q1 through its newest tumbling windows rolled up across -//! cities. - -use crate::physical_common; -use crate::planner_layering_common; - -use asap_cost::CostInputs; -use asap_executor::values::Value; -use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{PlanningWorkload, SqlDialect}; -use planner_layering_common::*; -use std::rc::Rc; - -const Q1: &str = "SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY value), \ - percentile_cont(0.9) WITHIN GROUP (ORDER BY value) FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE"; -const Q2: &str = "SELECT city, percentile_cont(0.9) WITHIN GROUP (ORDER BY value) \ - FROM metrics WHERE ts > now() - INTERVAL '20' MINUTE GROUP BY city"; -/// Every query's accuracy target. -const EPSILON: f64 = 0.01; -const MINUTE_MS: i64 = 60_000; -/// The evaluation time of every run. -const EVALUATION_TIME_MS: i64 = 1_800_000_000_000; - -/// `metrics(ts, city, value)`, time index `ts`. -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "metrics", - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - ) -} - -/// `queries`, each every 10 s at ε = [`EPSILON`], over `metrics` ingesting -/// 666.67 rows per second. -fn workload(queries: &[&str]) -> PlanningWorkload { - let entry = |query: &&str| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"max_error": {EPSILON}, "failure_probability": 0.001}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }; - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": {{"sql": "datafusion_sql"}}, "query_batch": null, - "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 666.67}}}}, - "sources": {{"metrics": {{"series_count": 100}}}}}}}}"#, - queries.iter().map(entry).collect::>().join(", ") - )) - .unwrap() -} - -/// Every stage over `queries`. -async fn run(queries: &[&str]) -> Run { - let workload = workload(queries); - let mut roots = Vec::new(); - for entry in workload.query_workload.entries() { - let root = lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap_or_else(|e| panic!("{}: {e}", entry.query.0)); - roots.push(QueryRoot::Operator(root)); - } - // Every output accepted whatever its guarantee: this test checks the - // answers against the bound. - let settings = CostInputs { - ..CostInputs::default() - }; - let candidates = stage1(queries) - .await - .into_iter() - .map(|(_, roots)| roots) - .collect(); - run_explicit_candidates( - &workload, - roots, - candidates, - asap_planner::PlanningModels::builtin() - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - ) -} - -/// Two rows a second over the 20 minutes before the evaluation: city -/// `c{i % 3}`, or none for every seventh row, and a heavy-tailed value from -/// a fixed linear congruential generator. -fn rows() -> Vec<(i64, Option, f64)> { - let mut state: u64 = 0x2545_f491_4f6c_dd1d; - let mut next = move || { - state = state - .wrapping_mul(6_364_136_223_846_793_005) - .wrapping_add(1_442_695_040_888_963_407); - (state >> 33) as f64 / (1u64 << 31) as f64 - }; - (0..20 * 60 * 2) - .map(|i: i64| { - let ts = EVALUATION_TIME_MS - 20 * MINUTE_MS + (i + 1) * 500; - let city = (i % 7 != 0).then(|| format!("c{}", i % 3)); - (ts, city, 10.0 / (1.0 - next()).max(1e-6).sqrt()) - }) - .collect() -} - -/// The values of [`rows`] in `(T - window, T]` of the city `city` (every -/// city for `None`), sorted. -fn values_in(window_ms: i64, city: Option<&str>) -> Vec { - let mut values: Vec = rows() - .into_iter() - .filter(|(ts, row_city, _)| { - *ts > EVALUATION_TIME_MS - window_ms - && city.is_none_or(|c| row_city.as_deref() == Some(c)) - }) - .map(|(_, _, value)| value) - .collect(); - values.sort_by(f64::total_cmp); - values -} - -/// SQL `percentile_cont` of sorted `values`. -fn percentile_cont(q: f64, values: &[f64]) -> f64 { - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -/// Whether `answer` is the `q` percentile of sorted `values` exactly, or -/// within `algorithm`'s bound: a KLL's rank error, a DDSketch's relative -/// value error. -fn within_bound(algorithm: Option<&SketchAlgorithm>, q: f64, answer: f64, values: &[f64]) -> bool { - let exact = percentile_cont(q, values); - if (answer - exact).abs() <= 1e-9 * exact.abs() { - return true; - } - match algorithm { - Some(SketchAlgorithm::Kll) => { - let n = values.len() as f64; - let below = values.partition_point(|&v| v < answer) as f64 / n; - let at_most = values.partition_point(|&v| v <= answer) as f64 / n; - below - EPSILON - 1.0 / n <= q && q <= at_most + EPSILON + 1.0 / n - } - Some(SketchAlgorithm::DDSketch) => (answer - exact).abs() <= EPSILON * exact.abs(), - _ => false, - } -} - -/// `root`'s answer rows at [`EVALUATION_TIME_MS`] over [`rows`], every node at -/// query time, each tumbling-window chain as its tumbling windows. -fn answers(root: &Rc) -> Vec> { - answers_over(root, rows()) -} - -/// [`answers`] over the table rows `table` (time, city, value). -fn answers_over( - root: &Rc, - table: Vec<(i64, Option, f64)>, -) -> Vec> { - let table = table - .into_iter() - .map(|(ts, city, value)| { - vec![ - Value::Timestamp(ts), - city.map_or(Value::Null, |city| Value::Utf8(city.into())), - Value::Float64(value), - ] - }) - .collect(); - // The executor runs a tumbling-window chain as its tumbling windows. - let root = asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(root) - .unwrap(); - physical_common::execute_raw_rows_at(&root, table, EVALUATION_TIME_MS) -} - -fn as_f64(value: &Value) -> f64 { - match value { - Value::Float64(value) => *value, - other => panic!("not a Float64: {other:?}"), - } -} - -/// The sketch algorithm `root` reads, if any. -fn sketch_of(root: &Rc) -> Option { - OperatorNode::reachable(root) - .into_iter() - .find_map(|node| match node.asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some(*kind.algorithm()), - _ => None, - }) -} - -fn operator(root: &QueryRoot) -> &Rc { - match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root expected"), - } -} - -/// Q1 with one percentile exact and the other from a sketch: every such -/// plan compiles and is priced (an exact aggregate split from a sketched -/// measure names its one output), and answers each percentile exactly or -/// within the sketch's bound. -#[tokio::test] -async fn q1_with_one_exact_and_one_sketched_percentile_compiles_and_matches() { - let run = run(&[Q1]).await; - let mixed: Vec<&Logical> = run - .logical - .iter() - .filter(|c| { - let reached = OperatorNode::reachable(operator(&c.roots[0])); - reached - .iter() - .any(|n| matches!(n.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - && reached - .iter() - .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - .collect(); - assert!(!mixed.is_empty(), "no mixed exact and sketch plan for Q1"); - for candidate in &mixed { - assert!( - run.physical_of(candidate) - .any(|p| run.cost(&p.id).is_some()), - "{}: no physical candidate priced: {:#?}", - candidate.id, - run.selection - .rejected - .iter() - .filter(|r| run.physical_of(candidate).any(|p| p.id == r.id)) - .map(|r| format!("{}: {:?} {}", r.id, r.cause, r.reason)) - .collect::>() - ); - } - let values = values_in(5 * MINUTE_MS, None); - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let candidate = mixed - .iter() - .find(|c| sketch_of(operator(&c.roots[0])) == Some(algorithm)) - .unwrap_or_else(|| panic!("no mixed {algorithm:?} plan")); - let root = operator(&candidate.roots[0]); - // The percentile the exact aggregate computes. - let exact_q = OperatorNode::reachable(root) - .iter() - .find_map(|n| match n.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => match measures.as_slice() { - [AggIntent::Quantile { q, .. }] => Some(*q), - _ => None, - }, - _ => None, - }) - .expect("one exact percentile"); - let rows = answers(root); - assert_eq!(rows.len(), 1, "{}", candidate.id); - for (column, q) in [0.5, 0.9].into_iter().enumerate() { - let answer = as_f64(&rows[0][column]); - let bound = (q != exact_q).then_some(&algorithm); - assert!( - within_bound(bound, q, answer, &values), - "{}: p{} = {answer}, exact {}", - candidate.id, - q * 100.0, - percentile_cont(q, &values) - ); - } - } -} - -/// The id and query roots of every Stage 1 candidate of `queries`. -async fn stage1(queries: &[&str]) -> Vec<(String, Vec)> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = workload(queries); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let mut roots = Vec::new(); - for entry in &entries { - roots.push(QueryRoot::Operator( - lower_sql_dialect( - &entry.query.0, - &catalog(), - SqlDialect::DataFusionSQL, - entry.requirements.accuracy.clone(), - ) - .await - .unwrap(), - )); - } - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - // Explicit fixture configurations: exact; one local substitution at a - // time (mixed percentiles); and all quantiles using each supported family. - // Do not enumerate a Cartesian product of complete workload choices. - let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); - for (target, options) in local.targets.iter().enumerate() { - for option in 1..options.choice_count() { - let mut choice = vec![0; local.targets.len()]; - choice[target] = option; - choices.insert(choice); - } - } - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let together: Option> = local.targets.iter().map(|target| target.ways_to_compute.iter() - .position(|candidate| matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if *family == algorithm))).collect(); - if let Some(choice) = together { - choices.insert(choice); - } - } - let share = |roots: Vec| { - asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - _ => unreachable!(), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect::>() - }; - let grids = entries - .iter() - .map(|entry| entry.evaluation_grid_ms()) - .collect::>(); - let mut candidates = Vec::new(); - for choice in choices { - let roots = share(compose_logical_candidate(&local, &choice).unwrap()); - candidates.push(roots.clone()); - let tumbling_windows = tumbling_window_candidates(&roots, &grids); - if !tumbling_windows.is_empty() { - candidates.extend( - window_composition_pass2_candidates(&roots, &tumbling_windows) - .unwrap() - .into_iter() - .filter(|candidate| { - candidate.tumbling_window_candidates.len() == tumbling_windows.len() - }) - .filter_map(|candidate| candidate.roots.ok()) - .map(share), - ); - } - } - candidates - .into_iter() - .enumerate() - .map(|(index, roots)| (format!("explicit-{}", index + 1), roots)) - .collect() -} - -/// Q1 and Q2 read the same rows of `metrics`: a candidate reads them with -/// one scan. A table scan has no unique key, and is shared all the same: -/// each reader reads every row it outputs. -#[tokio::test] -async fn q1_and_q2_share_the_table_scan() { - let scans = |root: &QueryRoot| -> Vec<*const OperatorNode> { - OperatorNode::reachable(operator(root)) - .iter() - .filter(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .map(Rc::as_ptr) - .collect() - }; - let candidates = stage1(&[Q1, Q2]).await; - assert!( - candidates.iter().any(|(_, roots)| { - let q2_scans = scans(&roots[1]); - scans(&roots[0]).iter().any(|scan| q2_scans.contains(scan)) - }), - "no candidate shares the scan" - ); -} - -/// Whether Q2's per-city tumbling-window chain (120 tumbling windows of 10 s) serves both of -/// Q1's percentiles in `roots`: Q1 merges each city's newest 30 tumbling windows (its -/// 5 minutes), then rolls the cities up into one state, which both of its -/// estimates read. -fn newest_tumbling_windows_rolled_up(roots: &[QueryRoot]) -> bool { - let (q1, q2) = (operator(&roots[0]), operator(&roots[1])); - let q2_chains: Vec<*const OperatorNode> = OperatorNode::reachable(q2) - .iter() - .filter(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count: 120, - .. - }) - ) - }) - .map(Rc::as_ptr) - .collect(); - let rollups: Vec> = OperatorNode::reachable(q1) - .into_iter() - .filter(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - children, - group_by: Some(keys), - newest_tumbling_windows: None, - }) => { - keys.is_empty() - && matches!( - children[0].asap(), - Some(ASAPOp::SummaryMerge { - children: chain, - newest_tumbling_windows: Some(30), - group_by: None, - }) if q2_chains.contains(&Rc::as_ptr(&chain[0])) - ) - } - _ => false, - }) - .collect(); - OperatorNode::reachable(q1) - .iter() - .filter(|n| match n.asap() { - Some(ASAPOp::SummaryEstimate { summary_input, .. }) => { - rollups.iter().any(|r| Rc::ptr_eq(r, summary_input)) - } - _ => false, - }) - .count() - == 2 -} - -/// The values of [`rows`] of city `city` (no city for `None`) in the last -/// 20 minutes, sorted. -fn city_values(city: Option<&str>) -> Vec { - let mut values: Vec = rows() - .into_iter() - .filter(|(ts, row_city, _)| { - *ts > EVALUATION_TIME_MS - 20 * MINUTE_MS && row_city.as_deref() == city - }) - .map(|(_, _, value)| value) - .collect(); - values.sort_by(f64::total_cmp); - values -} - -/// Q2's per-city chain serves both of Q1's percentiles through its newest -/// 30 tumbling windows rolled up across cities. Each query's answers are within the -/// sketch's bound of the exact `percentile_cont`, and over a window with -/// no row Q1 still answers one NULL row, as the global SQL aggregate the -/// rollup stands for. -#[tokio::test] -async fn q2_tumbling_window_chain_serves_q1_through_newest_tumbling_windows_and_a_rollup() { - let candidates = stage1(&[Q1, Q2]).await; - for algorithm in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let (id, roots) = candidates - .iter() - .find(|(_, roots)| { - newest_tumbling_windows_rolled_up(roots) - && sketch_of(operator(&roots[0])) == Some(algorithm) - }) - .unwrap_or_else(|| panic!("no {algorithm:?} newest-tumbling-windows rollup candidate")); - let q1 = answers(operator(&roots[0])); - let values = values_in(5 * MINUTE_MS, None); - assert_eq!(q1.len(), 1, "{id}"); - for (column, q) in [0.5, 0.9].into_iter().enumerate() { - let answer = as_f64(&q1[0][column]); - assert!( - within_bound(Some(&algorithm), q, answer, &values), - "{id}: Q1 p{} = {answer}, exact {}", - q * 100.0, - percentile_cont(q, &values) - ); - } - let older = rows() - .into_iter() - .filter(|(ts, _, _)| *ts <= EVALUATION_TIME_MS - 5 * MINUTE_MS) - .collect(); - let empty = answers_over(operator(&roots[0]), older); - assert!( - matches!(empty.as_slice(), [row] if matches!(row.as_slice(), [Value::Null, Value::Null])), - "{id}: {empty:?}" - ); - let q2 = answers(operator(&roots[1])); - assert_eq!(q2.len(), 4, "{id}: three cities and none"); - for row in q2 { - let city = match &row[0] { - Value::Utf8(city) => Some(city.to_string()), - Value::Null => None, - other => panic!("city: {other:?}"), - }; - let values = city_values(city.as_deref()); - let answer = as_f64(&row[1]); - assert!( - within_bound(Some(&algorithm), 0.9, answer, &values), - "{id}: Q2 p90 of {city:?} = {answer}, exact {}", - percentile_cont(0.9, &values) - ); - } - } -} - -/// A global SQL `COUNT(*)` is 0 over no rows, but a rollup of the per-city -/// counts would sum no count into NULL. A rollup to no keys over a table is -/// offered, yet no candidate answers the global count from the per-city -/// counts: an exact rollup never sums Int64 counts again. -#[tokio::test] -async fn a_global_count_is_not_rolled_up_from_grouped_counts() { - let global = "SELECT COUNT(*) FROM metrics WHERE ts > now() - INTERVAL '5' MINUTE"; - let by_city = "SELECT city, COUNT(*) FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY city"; - let finalized = |root: &QueryRoot| -> Vec<*const OperatorNode> { - OperatorNode::reachable(operator(root)) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .map(Rc::as_ptr) - .collect() - }; - for (id, roots) in stage1(&[global, by_city]).await { - let by_city_values = finalized(&roots[1]); - assert!( - !finalized(&roots[0]) - .iter() - .any(|value| by_city_values.contains(value)), - "{id}: the global count reads the per-city counts" - ); - } -} diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index 25117bb35..2fdaf76ce 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -30,8 +30,6 @@ use asap_types::workload::{ }; use futures::StreamExt; -use crate::physical_common; - fn accuracy() -> AccuracyTarget { AccuracyTarget::max_error(0.01).failure_probability(0.01) } @@ -99,10 +97,11 @@ fn output_columns(root: &QueryRoot) -> Vec<(String, String, bool)> { /// Compose every candidate of `root`; returns how many there were. fn compose_all(label: &str, root: QueryRoot) -> usize { let expected = output_columns(&root); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[root]), - ) - .unwrap(); + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ + root, + ])) + .unwrap(); let radix: Vec<_> = inventory .targets .iter() @@ -178,10 +177,11 @@ fn promql_candidates_compose() { /// The summary estimates of `root`'s candidates that vary one target's /// choice, every other target taking its first. fn composed_estimates(root: QueryRoot) -> Vec> { - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[root]), - ) - .unwrap(); + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ + root, + ])) + .unwrap(); let targets = inventory.targets.len(); let choices: Vec<_> = inventory .targets @@ -220,7 +220,7 @@ fn update_of(estimate: &OperatorNode) -> (SketchAlgorithm, SummaryUpdate) { else { panic!("sketch state expected") }; - (*kind.algorithm(), input.clone()) + (*kind, input.clone()) } /// A heap sketch's bound is per item frequency, not Top-K membership, and @@ -268,10 +268,11 @@ fn topk_over_an_aggregate_ranks_its_labels() { async fn subquery_in_a_measure_filter_is_substituted() { let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ FROM flows GROUP BY src"; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[sql(query).await]), - ) - .unwrap(); + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ + sql(query).await, + ])) + .unwrap(); assert_eq!(inventory.targets.len(), 2); let inner = inventory .targets @@ -325,7 +326,7 @@ async fn subquery_in_a_measure_filter_is_substituted() { /// Run the logical DAG `root` over `rows` of its one scanned table, each row /// given by column name. Rows come back shown and sorted. fn execute(root: &Rc, rows: &[Vec<(&str, Value)>]) -> Vec> { - let dag = physical_common::compile_physical_asap_dag(root).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); let scan = dag .nodes .iter() @@ -402,12 +403,11 @@ async fn avg_as_sum_and_count( query: &str, ) -> (Rc, Rc) { let exact = lower_sql(query, catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ QueryRoot::Operator(exact.clone()), - ]), - ) - .unwrap(); + ])) + .unwrap(); let choice: Vec<_> = inventory .targets .iter() @@ -538,12 +538,11 @@ async fn filtered_sql_avg_is_null_without_matching_rows() { /// PromQL `avg` divides its `sum` and `count` vectors with a `BinaryOp`. #[test] fn promql_avg_divides_sum_and_count_vectors() { - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[promql( - "avg by (job) (m)", - )]), - ) - .unwrap(); + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ + promql("avg by (job) (m)"), + ])) + .unwrap(); assert_eq!( inventory.targets[0].ways_to_compute[0], HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]) @@ -582,12 +581,11 @@ async fn every_candidate( Result, asap_logical_asap_optimization::PerComputationCandidateError>, )> { let root = lower_sql(query, catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ + let inventory = + generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ QueryRoot::Operator(root), - ]), - ) - .unwrap(); + ])) + .unwrap(); assert_eq!(inventory.targets.len(), 1, "{query}"); inventory.targets[0] .ways_to_compute @@ -695,46 +693,6 @@ async fn sql_avg_of_an_int64_column_keeps_the_exact_columns() { assert_eq!(execute(&exact, &rows), want); } -/// `count(*)` counts a non-null column, never a nullable one; executed, it -/// counts the rows whose nullable column is NULL too. -#[tokio::test] -async fn sql_count_star_counts_a_non_null_column() { - let catalog = table(&[("v", true), ("n", false)]); - let query = "SELECT k, count(*) FROM t GROUP BY k"; - for (candidate, composed) in every_candidate(&catalog, query).await { - let composed = composed.unwrap_or_else(|e| panic!("{candidate:?}: {e}")); - composed[0].validate_structure().unwrap(); - let updates = count_updates(&composed[0]); - assert_eq!(updates.len(), 1, "{candidate:?}"); - assert!(reads_only(&updates[0], "n"), "{candidate:?}: {updates:?}"); - } - let (exact, root) = { - let exact = lower_sql(query, &catalog, accuracy()).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(exact.clone()), - ]), - ) - .unwrap(); - let QueryRoot::Operator(root) = compose_logical_candidate(&inventory, &[0]) - .unwrap() - .remove(0) - else { - panic!("operator root expected") - }; - (exact, root) - }; - let row = |k: &str, v: Option| { - vec![ - ("k", Value::Utf8(k.into())), - ("v", v.map_or(Value::Null, Value::Float64)), - ("n", Value::Float64(0.)), - ] - }; - let rows = vec![row("a", None), row("a", Some(1.)), row("b", None)]; - assert_eq!(execute(&root, &rows), execute(&exact, &rows)); -} - /// `count(*)` with no column known to be non-null (one nullable numeric /// column, or several and none named `value`) is unsupported for every /// candidate, not a count of a nullable column or an invalid DAG. diff --git a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs b/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs deleted file mode 100644 index dbe386cac..000000000 --- a/crates/integration-tests/tests/stage2_physical_asap_optimization_sql.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! SQL heavy-hitter queries through Stage 1 (every Pass 1 choice composed) -//! and physical ASAP-aware optimization: exact top-k sorts by the ranked -//! measure, descending, whatever the group keys are called or typed, no -//! valid candidate is pruned, and every candidate answers the ranked value -//! then the group keys naming each ranked item. - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; -use asap_physical_asap_optimization::optimize_physical_asap; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::stages::HowToComputeAggregateMeasure; - -fn approximate() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.001) -} - -fn names(node: &OperatorNode) -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() -} - -/// The `(sort column name, ascending)` of an exact top-k's `Sort`, if `root` -/// is one (a heavy-hitter summary candidate has none). -fn exact_topk_sort(root: &OperatorNode) -> Option<(String, bool)> { - let NonASAPOp::Project { child: limit, .. } = root.non_asap()? else { - return None; - }; - let NonASAPOp::Limit { - n: Some(5), - child: sort, - .. - } = limit.non_asap()? - else { - return None; - }; - let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { - panic!("sort below limit") - }; - let [key] = keys.as_slice() else { - panic!("one sort key") - }; - let ScalarExpr::Column(column) = key.expr else { - panic!("sort by a column") - }; - Some((sort.schema.fields[column].name.clone(), key.ascending)) -} - -/// Every Pass 1 choice of `sql` under `accuracy`, composed and physically -/// optimized: each must succeed, every exact top-k must sort by `ranked` -/// descending, and every root must output `root_names`. Returns how many -/// candidates had an exact top-k. -async fn check_every_choice( - sql: &str, - accuracy: AccuracyTarget, - ranked: &str, - root_names: &[&str], -) -> usize { - let catalog = SqlCatalog::new().with_table( - "metrics", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("host", DataType::Utf8, false), - Field::plain("port", DataType::Int64, false), - ]), - ); - let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); - let NonASAPOp::Aggregate { measures, .. } = root.expect_non_asap() else { - panic!("{sql}: heavy-hitter aggregate root") - }; - assert!( - matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]), - "{sql}" - ); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // The cartesian product of every target's candidates. - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|prefix: Vec| { - (0..target.choice_count()).map(move |i| { - let mut choice = prefix.clone(); - choice.push(i); - choice - }) - }) - .collect(); - } - let mut exact = 0; - for choice in choices { - let composed = match compose_logical_candidate(&inventory, &choice) { - Ok(composed) => composed, - // Stage 1 cannot yet compose a heavy-hitter summary over some - // keys (an integer or several key columns); Stage 2 never sees - // that candidate. Every other choice must compose. - Err(e) => { - let summarized_topk = inventory.targets.iter().zip(&choice).any(|(t, &i)| { - let NonASAPOp::Aggregate { measures, .. } = - inventory.aggregate(t).expect_non_asap() - else { - return false; - }; - matches!(measures[t.measure_index], AggIntent::TopK { .. }) - && t.computation_for(i).map(|(c, _)| c) - != Some(&HowToComputeAggregateMeasure::Exact) - }); - assert!(summarized_topk, "{sql}, choice {choice:?}: {e}"); - continue; - } - }; - let roots: Vec<_> = composed; - let candidate = optimize_physical_asap( - "L1", - &roots, - &[], - &asap_types::workload::DataWorkload::default(), - false, - ) - .unwrap_or_else(|e| panic!("{sql}, choice {choice:?} pruned: {e}")) - .remove(0); - let QueryRoot::Operator(physical) = &candidate.roots[0] else { - panic!("operator root") - }; - assert_eq!(names(physical), root_names, "{sql}, choice {choice:?}"); - if let Some(sort) = exact_topk_sort(physical) { - assert_eq!( - sort, - (ranked.to_string(), false), - "{sql}, choice {choice:?}" - ); - exact += 1; - } - } - exact -} - -/// Each query under an exact and an approximate target; every exact top-k -/// sorts by the ranked measure. -async fn check(sql: &str, ranked: &str, root_names: &[&str]) { - for accuracy in [AccuracyTarget::exact(), approximate()] { - let exact = check_every_choice(sql, accuracy.clone(), ranked, root_names).await; - assert!( - exact > 0, - "{sql} under {accuracy:?}: an exact top-k candidate" - ); - } -} - -/// Grouping by a column named `value` still ranks by the count. -#[tokio::test] -async fn group_key_named_value_ranks_by_count() { - check( - "SELECT value, COUNT(*) AS c FROM metrics GROUP BY value ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "value"], - ) - .await; -} - -/// An integer group key leaves no `value` column; the count is still ranked. -#[tokio::test] -async fn integer_group_key_ranks_by_count() { - check( - "SELECT port, COUNT(*) AS c FROM metrics GROUP BY port ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "port"], - ) - .await; -} - -/// Two group keys, one of them an integer. -#[tokio::test] -async fn multi_key_ranks_by_count() { - check( - "SELECT host, port, COUNT(*) AS c FROM metrics GROUP BY host, port \ - ORDER BY c DESC LIMIT 5", - "count(Int64(1))", - &["topk_5", "host", "port"], - ) - .await; -} - -/// A SUM ranking sorts by the sum. -#[tokio::test] -async fn sum_ranking_ranks_by_sum() { - check( - "SELECT port, SUM(value) AS s FROM metrics GROUP BY port ORDER BY s DESC LIMIT 5", - "sum(metrics.value)", - &["topk_5", "port"], - ) - .await; -} diff --git a/crates/integration-tests/tests/summary_coverage_e2e.rs b/crates/integration-tests/tests/summary_coverage_e2e.rs index f2ce4aad3..dc5446817 100644 --- a/crates/integration-tests/tests/summary_coverage_e2e.rs +++ b/crates/integration-tests/tests/summary_coverage_e2e.rs @@ -31,7 +31,7 @@ fn kll_state(pre: Rc) -> Rc { .into_iter() .filter(|node| { matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::Kll) + if kind == &SketchAlgorithm::Kll) }) .collect(); assert_eq!( diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs deleted file mode 100644 index 7f5c7cff2..000000000 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ /dev/null @@ -1,887 +0,0 @@ -//! Tumbling windows merged by `SummaryMerge` answer the same per-series query -//! as one build over the whole window (#509 Example 3/4, Pattern B): tumbling windows -//! written by hand, and the planner's window-composition rule's -//! `SummaryTumblingWindowChain`, expanded into the tumbling windows the executor runs. -use crate::physical_common; - -use std::collections::BTreeMap; -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::physical_planner::test_support::views_at; -use asap_executor::physical_planner::{compile, promql_rows, CompiledPhysicalDAG}; -use asap_executor::runtime::Scope; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::values::{Batch, Value}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::{explicit_local_candidates, explicit_summary_candidate}; -use asap_logical_asap_optimization::test_support::{ - tumbling_window_candidates, window_composition_pass2_candidates, - window_composition_shared_chain_candidates, -}; -use asap_types::ir::operator::operator_properties::TimeShift; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains; -use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::Schema; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::test_support::compile_physical_asap_dag_with_node_ids; - -const EVALUATION_MS: i64 = 300_000; -const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; -const TUMBLING_WINDOW_COUNT: i64 = 5; - -/// An explicit supported single-build fixture for the tumbling-window merge rules. -fn single_build(query: &str, accuracy: AccuracyTarget) -> Rc { - let root = lower_promql(query, accuracy.clone()).unwrap(); - let root = promql_rows::with_series_identity(&root).unwrap(); - if accuracy.is_exact() { - explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - OperatorNode::reachable(candidate) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - }) - .expect("an exact accumulator fixture") - } else { - // Pin DDS so the actual positive-sample interpolation and tumbling-window merge - // checks do not depend on the optimizer's preferred implementation. - explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch) - } -} - -/// The per-entity `SummaryAgg` of `root` over its raw `TimeRange`. -fn summary_agg(root: &Rc) -> Rc { - OperatorNode::reachable(root) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a summary build") -} - -/// Rewrite `root` so its summary is built in tumbling windows: tumbling window `i` is -/// `TimeRange(1m)` over `TimeShift(i·1m)` over the same `Scan`, and a -/// `SummaryMerge` combines the tumbling windows. Returns the root and the tumbling windows. -fn tumbling_windows(root: &Rc) -> (Rc, Vec>) { - let agg = summary_agg(root); - let Some(NonASAPOp::TimeRange { kind, child, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let tumbling_windows: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: i * TUMBLING_WINDOW_WIDTH_MS, - at: None, - }, - child: child.clone(), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(TUMBLING_WINDOW_WIDTH_MS as u64), - kind: *kind, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(agg.operator.map_children(|_| range.clone())).unwrap() - }) - .collect(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.clone(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let root = replace(root, &agg, &merged); - root.validate_structure().unwrap(); - (root, tumbling_windows) -} - -/// `root` with its summary built in tumbling windows by the planner's -/// window-composition rule, for a query repeating every minute: the -/// planned root (a `SummaryMerge` over one `SummaryTumblingWindowChain`) and its expansion -/// into the tumbling windows the executor runs. -fn planned_tumbling_windows(root: &Rc) -> (Rc, Rc) { - let roots = [QueryRoot::Operator(Rc::clone(root))]; - let rule_candidates = - tumbling_window_candidates(&roots, &[Some(TUMBLING_WINDOW_WIDTH_MS as u64)]); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one build to split: {rule_candidates:?}") - }; - assert_eq!( - ( - rule_candidate.tumbling_window_width_ms, - rule_candidate.tumbling_window_count - ), - (TUMBLING_WINDOW_WIDTH_MS as u64, 5) - ); - let mut pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let pass2_candidate = pass2_candidates.pop().unwrap(); - assert!(pass2_candidates.is_empty()); - let QueryRoot::Operator(planned) = pass2_candidate.roots.unwrap().remove(0) else { - panic!("an operator root") - }; - planned.validate_structure().unwrap(); - assert_eq!( - planned.guarantee, root.guarantee, - "merged tumbling windows keep the whole build's guarantee" - ); - let expanded = expand_tumbling_window_chains(&planned).unwrap(); - expanded.validate_structure().unwrap(); - (planned, expanded) -} - -/// `node` with `from` replaced by `to`. -fn replace( - node: &Rc, - from: &Rc, - to: &Rc, -) -> Rc { - if Rc::ptr_eq(node, from) { - return to.clone(); - } - Rc::new( - node.with_new_children(|child| replace(child, from, to)) - .unwrap() - .with_guarantee(node.guarantee.clone()), - ) -} - -/// One raw data sample: series, timestamp (ms) and value. -type RawDataSample = (&'static str, i64, f64); - -/// Two series, four positive samples per minute in [0, 5m), each -/// `phase_ms` after its 15-s step (0: on the tumbling-window boundaries). -fn samples(phase_ms: i64) -> Vec { - let mut rows = vec![]; - for (s, series) in ["a", "b"].into_iter().enumerate() { - for minute in 0..TUMBLING_WINDOW_COUNT { - for j in 0..4 { - let ts = minute * TUMBLING_WINDOW_WIDTH_MS + j * 15_000 + phase_ms; - rows.push((series, ts, (s * 100) as f64 + (minute * 4 + j + 1) as f64)); - } - } - } - rows -} - -struct Exported { - dag: PhysicalASAPDAG, - /// Each `SummaryAgg` node with its tumbling window. - builds: Vec<(u64, i64)>, -} - -fn export(root: &Rc, assignment: &MaterializationAssignment) -> Exported { - let timed = - apply_materialization_timings(root, assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - let tumbling_window = |node: &OperatorNode| -> i64 { - match node.children()[0].non_asap() { - Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms / TUMBLING_WINDOW_WIDTH_MS, - _ => 0, - } - }; - let mut builds = vec![]; - for node in &compiled.dag.nodes { - let logical = compiled.node_ids.operator_node(node.id).unwrap(); - if let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &logical.operator { - builds.push((node.id as u64, tumbling_window(child))); - } - } - Exported { - dag: compiled.dag, - builds, - } -} - -fn schema_of(dag: &PhysicalASAPDAG, id: u64) -> Arc { - Arc::new( - dag.nodes - .iter() - .find(|node| node.id as u64 == id) - .unwrap() - .output_schema - .clone(), - ) -} - -/// A store holding every sample as raw data at the `Scan` of `m`; each -/// `TimeRange` keeps its own window of them. -fn raw_store(dag: &PhysicalASAPDAG, samples: &[RawDataSample]) -> InMemoryDataStore { - physical_common::raw_data_store(dag, |_, schema| { - samples - .iter() - .map(|&(series, ts, value)| { - let labels = BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ("series".to_string(), series.to_string()), - ]); - promql_rows::series_row(schema, &labels, ts, value).unwrap() - }) - .collect() - }) -} - -fn run(plan: &CompiledPhysicalDAG, store: InMemoryDataStore, scope: Scope) -> Vec> { - let mut rows: Vec> = physical_common::execute(plan, store, scope) - .remove(0) - .iter() - .flat_map(|batch| batch.to_rows().to_vec()) - .collect(); - rows.sort_by_key(|row| format!("{row:?}")); - rows -} - -/// `Value` has no equality; its debug form identifies every scalar exactly. -fn render(rows: &[Vec]) -> Vec { - rows.iter().map(|row| format!("{row:?}")).collect() -} - -fn has_timestamp(row: &[Value], ms: i64) -> bool { - row.iter() - .any(|v| matches!(v, Value::Timestamp(t) if *t == ms)) -} - -fn query_scope() -> Scope { - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - } -} - -/// Compile `root` with everything at query time (Pattern B2: tumbling windows rebuilt -/// from raw rows at query time). -fn compile_at_query_time( - root: &Rc, -) -> Result<(Exported, CompiledPhysicalDAG), String> { - let exported = export(root, &MaterializationAssignment::default()); - let plan = - compile(&exported.dag, &[], &[exported.dag.roots[0] as u64]).map_err(|e| e.to_string())?; - Ok((exported, plan)) -} - -fn execute(root: &Rc, samples: &[RawDataSample]) -> Vec> { - let (exported, plan) = compile_at_query_time(root).unwrap(); - run(&plan, raw_store(&exported.dag, samples), query_scope()) -} - -/// Build the tumbling windows in ingestion runs (stored `MatAtIngestion`, each with its -/// own build timestamp), then merge them in the query run (Pattern B1). -fn execute_retained( - root: &Rc, - tumbling_windows: &[Rc], - samples: &[RawDataSample], -) -> Vec> { - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - execute_retained_as(root, &assignment, samples) -} - -/// The planner's tumbling-window chain maintained at ingestion time, as Stage 2's B1 -/// option assigns it (`MatAtIngestion` on the chain), then expanded: each -/// tumbling window carries the chain's timing and materialization, so the tumbling windows are -/// built in ingestion runs and merged in the query run. -fn execute_maintained_chain( - planned: &Rc, - samples: &[RawDataSample], -) -> Vec> { - let chain = OperatorNode::reachable(planned) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .expect("a tumbling-window chain"); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&chain, Materialization::MatAtIngestion(Medium::Memory)); - let timed = - apply_materialization_timings(planned, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - for tumbling_window in OperatorNode::reachable(&expanded) - .iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - { - assert_eq!( - tumbling_window.materialization, - Materialization::MatAtIngestion(Medium::Memory) - ); - } - execute_retained_as(&expanded, &MaterializationAssignment::default(), samples) -} - -fn execute_retained_as( - root: &Rc, - assignment: &MaterializationAssignment, - samples: &[RawDataSample], -) -> Vec> { - let mut exported = export(root, assignment); - let builds: Vec = exported.builds.iter().map(|&(build, _)| build).collect(); - let views = views_at(&mut exported.dag, &builds).unwrap(); - // Each tumbling window's ingestion run refreshes its view from the raw samples. - let mut store = InMemoryDataStore::default(); - for &(build, tumbling_window) in &exported.builds { - let plan = compile(&exported.dag, &[], &[build]).unwrap(); - let end = EVALUATION_MS - tumbling_window * TUMBLING_WINDOW_WIDTH_MS; - let rows = run( - &plan, - raw_store(&exported.dag, samples), - Scope::Ingestion { - window_start_ms: end - TUMBLING_WINDOW_WIDTH_MS, - window_end_ms: end, - }, - ); - assert!(rows.iter().all(|row| has_timestamp(row, end))); - store = store - .with_materialized_view( - build as usize, - vec![Batch::try_new(schema_of(&exported.dag, build), rows).unwrap()], - ) - .unwrap(); - } - let plan = compile(&exported.dag, &views, &[exported.dag.roots[0] as u64]).unwrap(); - run(&plan, store, query_scope()) -} - -fn values(rows: &[Vec]) -> Vec { - rows.iter() - .flat_map(|row| { - row.iter().filter_map(|v| match v { - Value::Float64(v) => Some(*v), - _ => None, - }) - }) - .collect() -} - -/// Returns the per-series values, which every execution path agrees on. -fn assert_tumbling_window_merge_matches_single_build( - query: &str, - accuracy: AccuracyTarget, - samples: &[RawDataSample], -) -> Vec { - let single = single_build(query, accuracy); - let (tumbling_windows, tumbling_window_builds) = tumbling_windows(&single); - let expected = execute(&single, samples); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert!(expected.iter().all(|row| has_timestamp(row, EVALUATION_MS))); - let result = values(&expected); - let expected = render(&expected); - assert_eq!( - render(&execute(&tumbling_windows, samples)), - expected, - "tumbling windows rebuilt at query time" - ); - assert_eq!( - render(&execute_retained( - &tumbling_windows, - &tumbling_window_builds, - samples - )), - expected, - "retained tumbling windows" - ); - let (planned, expanded) = planned_tumbling_windows(&single); - assert_eq!( - render(&execute(&expanded, samples)), - expected, - "the planner's tumbling-window chain, expanded" - ); - assert_eq!( - render(&execute_maintained_chain(&planned, samples)), - expected, - "the planner's tumbling-window chain, maintained at ingestion time" - ); - let unexpanded = compile_at_query_time(&planned).err().unwrap_or_default(); - assert!( - unexpanded.contains("SummaryTumblingWindowChain"), - "{unexpanded}" - ); - result -} - -/// Exact per-series Sum over 5 merged 1-minute tumbling windows equals one 5-minute -/// build, timestamped at the evaluation time. -#[test] -fn exact_sum_over_tumbling_windows_matches_single_build() { - let sums = assert_tumbling_window_merge_matches_single_build( - "sum_over_time(m[5m])", - AccuracyTarget::exact(), - // 1 ms past each step: the window `(0, 5m]` holds all 20 samples. - &samples(1), - ); - assert_eq!(sums, vec![210., 2210.]); -} - -/// The selected quantile sketch over 5 merged 1-minute tumbling windows equals one -/// 5-minute build, within the target of the exact p99 (20 and 120). -#[test] -fn quantile_over_tumbling_windows_matches_single_build() { - let p99 = assert_tumbling_window_merge_matches_single_build( - "quantile_over_time(0.99, m[5m])", - AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative), - &samples(0), - ); - let mut p99 = p99; - p99.sort_by(f64::total_cmp); - assert_eq!(p99.len(), 2); - for (estimate, exact) in p99.into_iter().zip([20., 120.]) { - assert!( - (estimate - exact).abs() <= 0.05 * exact, - "{estimate} vs {exact}" - ); - } -} - -/// Exact counter Rate over 5 merged 1-minute tumbling windows equals one 5-minute -/// build: the merged increase is divided by the whole 5-minute window, not -/// by one tumbling window's minute. -#[test] -fn exact_rate_over_tumbling_windows_matches_single_build() { - let rates = assert_tumbling_window_merge_matches_single_build( - "rate(m[5m])", - AccuracyTarget::exact(), - &samples(0), - ); - assert_eq!(rates.len(), 2); - // 19 counted over 285 s of samples, extrapolated to the 5-min window. - for rate in rates { - assert!((rate - 20.0 / 300.0).abs() < 1e-9, "{rate}"); - } -} - -/// With samples off the tumbling-window boundaries, Increase extrapolates to the -/// start of the whole 5-minute window, as one build does, not to the start -/// of the newest tumbling window. -#[test] -fn exact_increase_off_the_tumbling_window_boundaries_matches_single_build() { - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &samples(7_000), - ); - assert_eq!(increases.len(), 2); -} - -/// The single build of `query`, evaluated at `evaluation_ms` over every -/// sample, as values. -fn whole_build_values_at(query: &str, evaluation_ms: i64, samples: &[RawDataSample]) -> Vec { - let root = single_build(query, AccuracyTarget::exact()); - let (exported, plan) = compile_at_query_time(&root).unwrap(); - values(&run( - &plan, - raw_store(&exported.dag, samples), - Scope::Query { - evaluation_time_ms: evaluation_ms, - }, - )) -} - -/// A counter over `[5m] offset 1m` evaluated a minute later reads the same -/// window as one without the offset: the counter window ends at the -/// evaluation time minus the offset, not at the evaluation time. -#[test] -fn offset_counter_window_ends_at_the_offset() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let plain = whole_build_values_at(&format!("{function}(m[5m])"), EVALUATION_MS, &samples); - let shifted = whole_build_values_at( - &format!("{function}(m[5m] offset 1m)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(shifted, plain, "{function}"); - } -} - -/// A counter over `[5m] @ 300` reads the window ending at 300 s whatever the -/// evaluation time, and `@ 300 offset 1m` the one ending at 240 s: the same -/// values as the unanchored counter evaluated at 300 s and at 240 s. Today -/// it reads the window ending at the evaluation time (#748). -#[test] -#[ignore = "the counter window drops an @ anchor and its offset: #748"] -fn anchored_counter_window_ends_at_the_anchor() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let at_300 = whole_build_values_at(&format!("{function}(m[5m])"), EVALUATION_MS, &samples); - let at_240 = whole_build_values_at( - &format!("{function}(m[5m])"), - EVALUATION_MS - TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_ne!(at_300, at_240, "{function}: the two windows differ"); - let anchored = whole_build_values_at( - &format!("{function}(m[5m] @ 300)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(anchored, at_300, "{function} @ 300"); - let anchored_offset = whole_build_values_at( - &format!("{function}(m[5m] @ 300 offset 1m)"), - EVALUATION_MS + TUMBLING_WINDOW_WIDTH_MS, - &samples, - ); - assert_eq!(anchored_offset, at_240, "{function} @ 300 offset 1m"); - } -} - -/// Tumbling windows whose ranges read the scan through two stacked `TimeShift`s (the -/// tumbling window's shift over a zero shift) compile and answer as the single build: -/// the per-entity build looks through every shift to its raw source. -#[test] -fn tumbling_windows_over_stacked_time_shifts_match_single_build() { - let single = single_build("increase(m[5m])", AccuracyTarget::exact()); - let (tumbling_windows, _) = tumbling_windows(&single); - let agg = summary_agg(&single); - let Some(NonASAPOp::TimeRange { child: scan, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let zero_shift = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 0, - at: None, - }, - child: scan.clone(), - })) - .unwrap(); - let stacked = replace(&tumbling_windows, scan, &zero_shift); - let expected = render(&execute(&single, &samples(0))); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!(render(&execute(&stacked, &samples(0))), expected); -} - -/// Exact counter Increase over 5 merged 1-minute tumbling windows equals one 5-minute -/// build: the merge joins consecutive tumbling windows across their boundaries. -#[test] -fn exact_increase_over_tumbling_windows_matches_single_build() { - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &samples(0), - ); - assert_eq!(increases.len(), 2); -} - -/// A middle tumbling window with no samples, for one series or for every series: the -/// merge joins the tumbling windows on either side of the empty one, and Increase -/// matches one 5-minute build. -#[test] -fn exact_increase_over_an_empty_middle_tumbling_window_matches_single_build() { - let middle = 2 * TUMBLING_WINDOW_WIDTH_MS..3 * TUMBLING_WINDOW_WIDTH_MS; - for emptied in [&["a"][..], &["a", "b"]] { - let gapped: Vec = samples(0) - .into_iter() - .filter(|(series, ts, _)| !(emptied.contains(series) && middle.contains(ts))) - .collect(); - let increases = assert_tumbling_window_merge_matches_single_build( - "increase(m[5m])", - AccuracyTarget::exact(), - &gapped, - ); - assert_eq!(increases.len(), 2, "{emptied:?}"); - } -} - -/// A counter over `[5m] offset 1m` split into the planner's tumbling windows, read -/// at query time or maintained at ingestion time, matches one build: the -/// tumbling windows' shifts fold the outer offset, and the merged counter window ends -/// at the evaluation time minus the offset. -#[test] -fn a_counter_over_tumbling_windows_with_an_outer_offset_matches_single_build() { - let samples = samples(7_000); - for function in ["rate", "increase"] { - let single = single_build( - &format!("{function}(m[5m] offset 1m)"), - AccuracyTarget::exact(), - ); - let expected = render(&execute(&single, &samples)); - assert_eq!(expected.len(), 2, "{function}: {expected:?}"); - let (planned, expanded) = planned_tumbling_windows(&single); - assert_eq!( - render(&execute(&expanded, &samples)), - expected, - "{function}: expanded tumbling windows" - ); - assert_eq!( - render(&execute_maintained_chain(&planned, &samples)), - expected, - "{function}: tumbling windows maintained at ingestion time" - ); - } -} - -/// `root`'s summary built from tumbling windows `(shift, width)` merged by one -/// `SummaryMerge`, both in minutes, or the IR's refusal of that merge. -fn tumbling_windows_at( - root: &Rc, - tumbling_windows: &[(i64, i64)], -) -> Result, String> { - let agg = summary_agg(root); - let Some(NonASAPOp::TimeRange { kind, child, .. }) = agg.children()[0].non_asap() else { - panic!("expected a raw time range under the summary"); - }; - let children = tumbling_windows - .iter() - .map(|&(shift, width)| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift * TUMBLING_WINDOW_WIDTH_MS, - at: None, - }, - child: child.clone(), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis((width * TUMBLING_WINDOW_WIDTH_MS) as u64), - kind: *kind, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(agg.operator.map_children(|_| range.clone())).unwrap() - }) - .collect(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) - .map_err(|error| format!("{error:?}"))?; - Ok(replace(root, &agg, &merged)) -} - -/// Counter tumbling windows form one window only back to back: with a gap between -/// them, compiling the counter evaluation is an error; overlapping tumbling windows -/// are already refused by the `SummaryMerge`, whose counter inputs must be -/// disjoint in time. -#[test] -fn counter_tumbling_windows_with_a_gap_or_an_overlap_are_rejected() { - let single = single_build("rate(m[5m])", AccuracyTarget::exact()); - let gapped = tumbling_windows_at(&single, &[(0, 1), (2, 1)]).unwrap(); - let error = compile_at_query_time(&gapped).err().unwrap_or_default(); - assert!(error.contains("ambiguous logical windows"), "{error}"); - let overlap = tumbling_windows_at(&single, &[(0, 2), (1, 2)]) - .err() - .unwrap_or_default(); - assert!(overlap.contains("InputsOverlap"), "{overlap}"); -} - -/// A 2-min and a 5-min quantile, both every minute, on 1-min tumbling windows: the -/// window-composition rule offers one 5-tumbling-window chain both read, the 2-min -/// query merging only its newest 2 tumbling windows. Expanded, rebuilt at query time -/// or maintained at ingestion time, each query's answer from the shared -/// chain equals its answer from its own chain. -#[test] -fn newest_tumbling_windows_of_a_shared_chain_match_a_separate_chain() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let roots = [ - QueryRoot::Operator(single_build( - "quantile_over_time(0.5, m[2m])", - accuracy.clone(), - )), - QueryRoot::Operator(single_build("quantile_over_time(0.99, m[5m])", accuracy)), - ]; - let grid = Some(TUMBLING_WINDOW_WIDTH_MS as u64); - let rule_candidates = tumbling_window_candidates(&roots, &[grid, grid]); - assert_eq!(rule_candidates.len(), 2, "{rule_candidates:?}"); - let pass2_candidates = window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let both = |shared: bool| -> Vec> { - pass2_candidates - .iter() - .find(|c| { - c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty() != shared - }) - .expect("both builds in tumbling windows") - .roots - .as_ref() - .unwrap() - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Rc::clone(node), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect() - }; - let (shared, separate) = (both(true), both(false)); - let newest = OperatorNode::reachable(&shared[0]) - .into_iter() - .find_map(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => Some(*newest_tumbling_windows), - _ => None, - }); - assert_eq!(newest, Some(Some(2))); - let data = samples(0); - for (query, (shared, separate)) in shared.iter().zip(&separate).enumerate() { - let expected = execute(&expand_tumbling_window_chains(separate).unwrap(), &data); - assert!(!expected.is_empty()); - assert_eq!( - render(&execute( - &expand_tumbling_window_chains(shared).unwrap(), - &data - )), - render(&expected), - "Q{}: the shared chain rebuilt at query time", - query + 1 - ); - assert_eq!( - render(&execute_maintained_chain(shared, &data)), - render(&expected), - "Q{}: the shared chain maintained at ingestion time", - query + 1 - ); - } -} - -/// Runs `root` at query time at `evaluation_ms` over every sample; each -/// `TimeRange` keeps its own window `(T - s - w, T - s]` for a range of -/// width `w` over a shift of `s` (none for an unshifted range). -fn execute_ranges_at_their_windows( - root: &Rc, - samples: &[RawDataSample], - evaluation_ms: i64, -) -> Vec> { - let (exported, plan) = compile_at_query_time(root).unwrap(); - let scope = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - run(&plan, raw_store(&exported.dag, samples), scope) -} - -/// A 5-min quantile every 30 s (10 × 30-s tumbling windows) and a 5-min quantile -/// every minute (5 × 1-min tumbling windows): the rule offers one chain of 30-s -/// tumbling windows both read in full, each at its own interval. Expanded, each -/// query's answer from the shared chain equals one build over its window, -/// at 5 min, and for the 30-s query also at 4.5 min, off the 1-min grid. -#[test] -fn queries_at_different_intervals_read_one_chain_of_the_finer_tumbling_windows() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let singles = [ - single_build("quantile_over_time(0.5, m[5m])", accuracy.clone()), - single_build("quantile_over_time(0.99, m[5m])", accuracy), - ]; - let roots: Vec = singles - .iter() - .map(|root| QueryRoot::Operator(Rc::clone(root))) - .collect(); - let rule_candidates = tumbling_window_candidates( - &roots, - &[Some(30_000), Some(TUMBLING_WINDOW_WIDTH_MS as u64)], - ); - let widths: Vec = rule_candidates - .iter() - .map(|c| c.tumbling_window_width_ms) - .collect(); - assert_eq!(widths, [30_000, TUMBLING_WINDOW_WIDTH_MS as u64]); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && !c.shared_chains.is_empty()) - .expect("one shared chain"); - // The whole set's shared chains, built alone, are the same candidates. - let alone = window_composition_shared_chain_candidates(&roots, &rule_candidates); - let first = alone.first().expect("one shared chain alone"); - assert_eq!(first.shared_chains, shared.shared_chains); - assert_eq!(first.roots.as_ref().ok(), shared.roots.as_ref().ok()); - assert_eq!( - ( - shared.shared_chains[0].tumbling_window_width_ms, - shared.shared_chains[0].tumbling_window_count - ), - (30_000, 10) - ); - let data = samples(0); - let shared_roots: Vec> = shared - .roots - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand_tumbling_window_chains(&node).unwrap(), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect(); - for (query, evaluation_ms) in [(0, EVALUATION_MS), (1, EVALUATION_MS), (0, 270_000)] { - let expected = execute_ranges_at_their_windows(&singles[query], &data, evaluation_ms); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!( - render(&execute_ranges_at_their_windows( - &shared_roots[query], - &data, - evaluation_ms - )), - render(&expected), - "Q{} at {evaluation_ms} ms", - query + 1 - ); - } -} - -/// A 5-min quantile every minute (5 × 1-min tumbling windows) and a 3-min one every -/// 90 s (2 × 90-s tumbling windows): neither width divides the other, so the rule -/// offers one chain at their gcd, 10 × 30-s tumbling windows, the 3-min query merging -/// the newest 6. Expanded, each query's answer from the shared chain equals -/// one build over its window, at times on each query's own grid. -#[test] -fn queries_at_60_and_90_seconds_read_one_chain_of_30_second_tumbling_windows() { - let accuracy = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let singles = [ - single_build("quantile_over_time(0.5, m[5m])", accuracy.clone()), - single_build("quantile_over_time(0.99, m[3m])", accuracy), - ]; - let roots: Vec = singles - .iter() - .map(|root| QueryRoot::Operator(Rc::clone(root))) - .collect(); - let rule_candidates = tumbling_window_candidates( - &roots, - &[Some(TUMBLING_WINDOW_WIDTH_MS as u64), Some(90_000)], - ); - let widths: Vec<(u64, u32)> = rule_candidates - .iter() - .map(|c| (c.tumbling_window_width_ms, c.tumbling_window_count)) - .collect(); - assert_eq!(widths, [(TUMBLING_WINDOW_WIDTH_MS as u64, 5), (90_000, 2)]); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && !c.shared_chains.is_empty()) - .expect("one shared chain at the gcd"); - assert_eq!( - ( - shared.shared_chains[0].tumbling_window_width_ms, - shared.shared_chains[0].tumbling_window_count - ), - (30_000, 10) - ); - let data = samples(0); - let shared_roots: Vec> = shared - .roots - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand_tumbling_window_chains(&node).unwrap(), - QueryRoot::Scalar(_) => panic!("an operator root"), - }) - .collect(); - for (query, evaluation_ms) in [(0, EVALUATION_MS), (1, 270_000), (1, 180_000)] { - let expected = execute_ranges_at_their_windows(&singles[query], &data, evaluation_ms); - assert_eq!(expected.len(), 2, "{expected:?}"); - assert_eq!( - render(&execute_ranges_at_their_windows( - &shared_roots[query], - &data, - evaluation_ms - )), - render(&expected), - "Q{} at {evaluation_ms} ms", - query + 1 - ); - } -} diff --git a/crates/logical-asap-optimization/Cargo.toml b/crates/logical-asap-optimization/Cargo.toml index 665246936..3154d7233 100644 --- a/crates/logical-asap-optimization/Cargo.toml +++ b/crates/logical-asap-optimization/Cargo.toml @@ -21,7 +21,7 @@ asap-frontend-promql = { path = "../frontend-promql", optional = true } [features] # The test helpers (`test_support`) the later stage crates' tests share. -test-support = ["dep:asap-frontend-promql"] +test-support = ["dep:asap-frontend-promql", "asap-types/test-support"] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/logical-asap-optimization/src/deployment_capabilities.rs b/crates/logical-asap-optimization/src/deployment_capabilities.rs index f449513b6..5caa09a78 100644 --- a/crates/logical-asap-optimization/src/deployment_capabilities.rs +++ b/crates/logical-asap-optimization/src/deployment_capabilities.rs @@ -20,10 +20,10 @@ //! contradiction, refused as bad input ([`DeploymentCapabilities::validate`]); //! * whether query-time results are retained across evaluations. If they //! are, Stage 2 also offers a tumbling-window chain whose readers evaluate -//! together at its tumbling-window width as `MatAtQueryKeptAcrossEvaluations`: each +//! together at its tumbling-window width as query-time `AcrossEvaluations`: each //! evaluation builds one new tumbling window and keeps the others for the next ones, //! covering the readers' longest look-back. A chain two or more queries -//! read keeps its `MatAtQuery` option too (shared within one evaluation), +//! read keeps its `OneEvaluation` option too (shared within one evaluation), //! and Stage 3 picks the cheaper. If they are not, a query-time result is //! reused only within one evaluation. //! @@ -53,7 +53,7 @@ pub struct DeploymentCapabilities { /// Whether raw data samples are kept for query-time scans; default `true`. pub raw_data_retained: bool, /// Whether a query-time result may be kept across evaluations - /// (`MatAtQueryKeptAcrossEvaluations`); default `false`. + /// (query-time `AcrossEvaluations`); default `false`. pub query_time_results_retained: bool, } diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index 3864772dc..b9b7a8d4d 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -4,7 +4,7 @@ //! (`asap_types::stages::LogicalASAPMemoCandidates`) that Stage 2 implements. //! //! Identical-sub-DAG sharing over the logical DAG (Stage 0, the `stage0` -//! module, behind [`generate_dag_of_logical_memo_groups`]) runs before the +//! module, behind [`memo_of_queries`]) runs before the //! stages; Stage 1 adds the ASAP-aware sharing that is invisible at the //! logical level (Pass 2, the `asap_aware_cse` module), e.g. //! `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading one built sketch. @@ -13,8 +13,7 @@ //! (`asap-accuracy`). It depends on no other stage crate, and no other //! stage crate depends on it except in their tests (CI checks the crate //! graph): the stage-boundary types (its input -//! `DagOfLogicalMemoGroups`, Pass 1's `PerComputationLogicalMemoCandidates` -//! and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and +//! `Memo`, Pass 1's `Memo` and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and //! the planner facade calls each stage in turn. It depends on no //! query-language frontend either, except with the `test-support` feature, //! whose test helpers (`test_support`) lower PromQL test queries through @@ -33,12 +32,11 @@ //! re-exports the internals other crates' tests call. //! //! - `stage0` — Stage 0: hash-consing the workload's logical DAG into memo -//! groups (`stage0::cse`), behind [`generate_dag_of_logical_memo_groups`]. +//! groups (`stage0::cse`), behind [`memo_of_queries`]. //! - `pass1::per_computation_candidates` — Pass 1: per-computation candidate //! generation and composition, behind -//! [`generate_per_computation_logical_memo_candidates`]. -//! - `pass2::asap_aware_cse` and `pass2::memo` — Pass 2: ASAP-aware -//! common-subexpression elimination and the memo rules, behind +//! [`pass1_memo`]. +//! - `pass2::memo` — Pass 2: the rules over groups, behind //! [`generate_logical_asap_memo_candidates`]. //! - `pass1::realization` — the summary families that can serve an aggregate //! intent, the intent's accuracy target and `(ε, δ)` budget, and default @@ -78,25 +76,15 @@ pub mod test_support; pub use deployment_capabilities::{ DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, }; -// Stage 0: the memo groups of the workload's query roots -// (`asap_types::stages::DagOfLogicalMemoGroups`). -pub use stage0::generate_dag_of_logical_memo_groups; +// Stage 0: the memo groups of the workload's query roots, without +// candidates (`asap_types::stages::Memo`). +pub use stage0::memo_of_queries; // Pass 1: its entry and its errors. Its output and the types the output // holds are in `asap_types::stages`. -pub use pass1::{ - generate_per_computation_logical_memo_candidates, PerComputationCandidateError, - PerComputationLogicalMemoCandidatesError, -}; +pub use pass1::{pass1_memo, Pass1Error, PerComputationCandidateError}; // Pass 2: its entry and its error. -pub use pass1::memo_population::CandidatePopulationError; pub use pass2::logical_memo_candidates::{ generate_logical_asap_memo_candidates, pass2_after_local_window_composition, pass2_local_window_composition, }; -pub use pass2::memo::state_groups::Pass2RuleError; -// The CandidateLogicalASAPDAGs a memo stands for (developer tools' -// reference pricing). -pub use pass2::memo::enumeration::{ - build_logical_asap_dags_from_chosen_candidates, enumerate, AllChosenCandidatesPerMemoGroup, - ChosenCandidatePerMemoGroup, ErrorBuildingLogicalASAPDagsFromChosenCandidates, -}; +pub use pass2::memo::rules::Pass2RuleError; diff --git a/crates/logical-asap-optimization/src/pass1.rs b/crates/logical-asap-optimization/src/pass1.rs index 2f115053b..6c2f87b43 100644 --- a/crates/logical-asap-optimization/src/pass1.rs +++ b/crates/logical-asap-optimization/src/pass1.rs @@ -1,14 +1,10 @@ //! Stage 1, Pass 1: per-computation candidate generation and composition, //! and the summary realizations it offers. Its one entry is -//! [`generate_per_computation_logical_memo_candidates`], whose output -//! ([`PerComputationLogicalMemoCandidates`](asap_types::stages::PerComputationLogicalMemoCandidates)) -//! Pass 2 extends. +//! [`pass1_memo`], whose output +//! memo Pass 2 extends. -/// Stage 1, Pass 1's entry and output: the per-computation candidates on -/// the memo groups ([`PerComputationLogicalMemoCandidates`](asap_types::stages::PerComputationLogicalMemoCandidates)). +/// Stage 1, Pass 1's entry: the initial candidates of the memo groups. pub(crate) mod logical_memo_candidates; -/// Pass 1's candidates on Stage 0's memo groups (`try_populate_pass1`). -pub(crate) mod memo_population; /// Stage 1, Pass 1's computation targets, their candidates and their /// composition. No execution timing is assigned: that is a Stage 2 /// materialization decision. Its target list @@ -20,7 +16,5 @@ pub(crate) mod per_computation_candidates; mod per_computation_candidates_tests; pub(crate) mod realization; -pub use logical_memo_candidates::{ - generate_per_computation_logical_memo_candidates, PerComputationLogicalMemoCandidatesError, -}; +pub use logical_memo_candidates::{pass1_memo, Pass1Error}; pub use per_computation_candidates::PerComputationCandidateError; diff --git a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs index 1d20e5e79..066f41724 100644 --- a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs @@ -1,310 +1,53 @@ -//! Stage 1, Pass 1's entry and output: the per-computation candidates attached to the -//! memo groups of the workload's hash-consed DAG -//! ([`PerComputationLogicalMemoCandidates`]), which Pass 2 extends. - -use std::rc::Rc; +//! Stage 1, Pass 1's entry and output: each group's initial candidates +//! (exact, exact accumulator, one summary build per listed parameter +//! configuration, fused builds), with every intermediate result its own +//! group (a [`Memo`]), which Pass 2 extends. use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{QueryRoot, SchemaDerivationError}; use asap_types::workload::DataWorkload; -use asap_accuracy::sizing::{ - per_input_failure_probability, with_summaries_sized_for_failure_probability, -}; use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; -use super::memo_population::try_populate_pass1; -use super::memo_population::CandidatePopulationError; use crate::deployment_capabilities::SupportedSummary; use crate::pass1::per_computation_candidates::{ generate_per_computation_candidates, PerComputationCandidateError, }; -use asap_types::stages::{ - DagOfLogicalMemoGroups, Memo, PerComputationLogicalMemoCandidates, PrunedCandidate, -}; +use asap_types::stages::Memo; -/// Why [`generate_per_computation_logical_memo_candidates`] failed. +/// Why [`pass1_memo`] failed. #[derive(Debug, thiserror::Error)] -pub enum PerComputationLogicalMemoCandidatesError { +pub enum Pass1Error { /// A query's DAG cannot be planned: it already holds ASAP or physical /// annotations, or a measure's accuracy target is invalid. #[error(transparent)] PerComputation(#[from] PerComputationCandidateError), - /// The targets do not belong to the DAG of logical memo groups. - #[error(transparent)] - Population(#[from] CandidatePopulationError), - /// A candidate's summaries sized for its readers' failure probability - /// did not rebuild. - #[error(transparent)] - Sizing(#[from] SchemaDerivationError), } -/// Stage 1, Pass 1: the per-computation candidates on the memo groups of -/// `dag_of_logical_memo_groups`. It finds one computation target per +/// Stage 1, Pass 1: the initial candidates of the memo groups of `memo` +/// (Stage 0's, without candidates). It finds one computation target per /// measure of each aggregate group with a realization besides exact /// execution, drops the summaries the deployment does not support /// (`supported_summaries`, the deployment's /// `DeploymentCapabilities::supported_summaries`; `None` supports every /// one) and, unless `accuracy` or `evidence` may accept an output without /// its guarantee, the summaries no query reading them can use (a query -/// `q`'s target is `accuracy_targets[q]`). It then attaches each group's -/// candidates, each with its summaries sized for its readers' failure -/// probabilities (`size_for_failure_probability`): one sized candidate -/// per accuracy-budget split. Every pruned way to compute a measure is in -/// the output's `pruned_candidates`. -pub fn generate_per_computation_logical_memo_candidates( - dag_of_logical_memo_groups: DagOfLogicalMemoGroups, +/// `q`'s target is `accuracy_targets[q]`). It then adds each way to compute +/// a group as candidates, through `Memo::find_or_add_group` and +/// `Memo::add_candidate`. +pub fn pass1_memo( + memo: Memo, accuracy_targets: &[AccuracyTarget], accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider, data: &DataWorkload, supported_summaries: Option<&[SupportedSummary]>, -) -> Result { - let mut per_computation = generate_per_computation_candidates(&dag_of_logical_memo_groups)? +) -> Result { + let mut per_computation = generate_per_computation_candidates(&memo)? .without_unsupported_summaries(supported_summaries); if !accuracy.may_accept_regardless_of_guarantee() && !evidence.may_provide_evidence() { per_computation = per_computation.without_summaries_no_query_can_use(accuracy_targets, data, accuracy); } - let mut memo = try_populate_pass1(dag_of_logical_memo_groups, &per_computation)?; - size_for_failure_probability(&mut memo, &per_computation.roots, accuracy_targets, data)?; - let pruned_candidates = per_computation - .pruned_ways_to_compute - .into_iter() - .map(|pruned| PrunedCandidate { - group: per_computation.targets[pruned.target].aggregate_group, - candidate: pruned.way_to_compute, - reason: pruned.reason, - }) - .collect(); - Ok(PerComputationLogicalMemoCandidates { - memo, - pruned_candidates, - }) -} - -/// Sizes every candidate of `memo` for its readers' failure probabilities, -/// in place: each approximate input of a candidate with `n` -/// accuracy budget parts (`accuracy_budget_parts`) gets the smallest -/// [`per_input_failure_probability`] `δ_q / (n · V_q)` among the queries `q` -/// reaching its group, `V_q` the values `q`'s target covers (`G` of its -/// root, `roots[q]`, for the whole result) -/// ([`with_summaries_sized_for_failure_probability`]). These are the candidates' final -/// sizes: Pass 2 inherits them or takes the strictest shared reader's -/// ([`size_for_all_readers`](asap_accuracy::estimators::size_for_all_readers)), -/// and Stage 3 only checks the composed accuracy. A candidate no reader -/// gives a failure probability (exact, per-value with no `δ`, or a -/// whole-result target over an unbound `G`) keeps its Pass 1 size. -fn size_for_failure_probability( - memo: &mut Memo, - roots: &[QueryRoot], - accuracy_targets: &[AccuracyTarget], - data: &DataWorkload, -) -> Result<(), PerComputationLogicalMemoCandidatesError> { - for (group, candidates) in memo.groups.iter().zip(&mut memo.candidates) { - for candidate in candidates.iter_mut() { - let Some(failure_probability) = group - .reached_by_queries() - .iter() - .filter_map(|&query| { - per_input_failure_probability( - accuracy_targets.get(query)?, - candidate.accuracy_budget_parts, - roots.get(query)?, - data, - ) - }) - .reduce(f64::min) - else { - continue; - }; - let sized = with_summaries_sized_for_failure_probability( - &[QueryRoot::Operator(Rc::clone(&candidate.root_node))], - failure_probability, - data, - )?; - let [QueryRoot::Operator(node)] = &sized[..] else { - unreachable!("one operator root in, one out") - }; - candidate.root_node = Rc::clone(node); - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{ - lower_promql, shared_summary_state_groups, sharing_rules_and_shared_state_groups, - }; - use asap_accuracy::{IgnoreAccuracyGuarantees, NoAccuracyEvidence}; - use asap_types::ir::properties::{AccuracyMetric, AccuracyScope, AccuracyTarget}; - use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; - use asap_types::ir::{ASAPOp, OperatorNode}; - use asap_types::stages::CandidateForMemoGroup; - use asap_types::workload::DurationMs; - - /// Rank error 0.01 at δ = 0.01 over `scope`. - fn rank_target(scope: AccuracyScope) -> AccuracyTarget { - AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(scope) - } - - /// Metric `x` with `series` series (`None`: undeclared). - fn data(series: Option) -> DataWorkload { - let data = DataWorkload::sampled_every(["x".to_string()], DurationMs(1_000)); - match series { - Some(series) => data.with_series_count(series), - None => data, - } - } - - /// Pass 1's memo for `queries`, each lowered and entered with its target. - fn pass1(queries: &[(&str, AccuracyTarget)], data: &DataWorkload) -> Memo { - let roots: Vec = queries - .iter() - .map(|(query, target)| { - QueryRoot::Operator(lower_promql( - query, - target.clone().accuracy_scope(AccuracyScope::PerValue), - )) - }) - .collect(); - let targets: Vec = - queries.iter().map(|(_, target)| target.clone()).collect(); - generate_per_computation_logical_memo_candidates( - generate_dag_of_logical_memo_groups(&roots), - &targets, - &IgnoreAccuracyGuarantees, - &NoAccuracyEvidence, - data, - None, - ) - .unwrap() - .memo - } - - /// The failure probability of every KLL `candidates` build. - fn kll_failure_probabilities<'a>( - candidates: impl IntoIterator, - ) -> Vec> { - candidates - .into_iter() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Kll => match kind.params() { - SketchParams::Kll { - failure_probability, - .. - } => Some(*failure_probability), - _ => None, - }, - _ => None, - }) - .collect() - } - - /// The failure probability of every KLL build of `memo`'s candidates - /// for one accuracy budget part (the whole budget). - fn whole_share_kll(memo: &Memo) -> Vec> { - kll_failure_probabilities( - memo.candidates - .iter() - .flatten() - .filter(|candidate| candidate.accuracy_budget_parts == 1), - ) - } - - /// Pass 1 sizes each sketch for `δ / V`, `V` the values its query's - /// target covers: one for a per-value target, `⌊1 / share⌋` = 20 for - /// the groups holding at least 5% of the records (whether or not `G` is - /// declared), `G` = 10 000 series for the whole result. A whole-result - /// target over an undeclared `G` leaves the per-value size (KLL's - /// default failure probability, `None`), which Stage 3 then rejects. - #[test] - fn pass1_sizes_each_sketch_for_the_values_its_target_covers() { - let query = "quantile_over_time(0.99, x[5m])"; - let five_percent = AccuracyScope::GroupsWithRecordShareAtLeast(0.05); - for (scope, series, failure_probability) in [ - (AccuracyScope::PerValue, Some(10_000), None), - (five_percent, None, Some(0.01 / 20.0)), - (five_percent, Some(10_000), Some(0.01 / 20.0)), - ( - AccuracyScope::WholeResult, - Some(10_000), - Some(0.01 / 10_000.0), - ), - (AccuracyScope::WholeResult, None, None), - ] { - let memo = pass1(&[(query, rank_target(scope))], &data(series)); - let found = whole_share_kll(&memo); - assert!(!found.is_empty(), "{scope:?}"); - assert!( - found.iter().all(|f| *f == failure_probability), - "{scope:?} {series:?}: {found:?}" - ); - } - } - - /// A group several queries reach is sized for the strictest of them: - /// the per-series quantiles are both a whole-result query's value (over - /// `G` = 10 000 series) and the input of a global `max` (one value), so - /// they take `δ / 10 000`. - #[test] - fn a_group_several_queries_reach_takes_the_strictest_failure_probability() { - let whole = rank_target(AccuracyScope::WholeResult); - let memo = pass1( - &[ - ("quantile_over_time(0.99, x[5m])", whole.clone()), - ("max(quantile_over_time(0.99, x[5m]))", whole), - ], - &data(Some(10_000)), - ); - let found = whole_share_kll(&memo); - assert!(!found.is_empty()); - assert!( - found.iter().all(|f| *f == Some(0.01 / 10_000.0)), - "{found:?}" - ); - } - - /// Pass 2 shares one build among readers sized differently in Pass 1 - /// at the strictest reader's size: a per-value 0.95 quantile (δ) and a - /// whole-result 0.99 quantile over `G` = 10 000 series (δ / 10 000) - /// share a KLL sized for δ / 10 000. Stage 3 does not resize it. - #[test] - fn a_shared_build_takes_its_strictest_readers_size() { - let memo = pass1( - &[ - ( - "quantile_over_time(0.95, x[5m])", - rank_target(AccuracyScope::PerValue), - ), - ( - "quantile_over_time(0.99, x[5m])", - rank_target(AccuracyScope::WholeResult), - ), - ], - &data(Some(10_000)), - ); - let memo = sharing_rules_and_shared_state_groups(memo).unwrap(); - let found = kll_failure_probabilities( - shared_summary_state_groups(&memo) - .iter() - .filter(|group| memo.groups[group.0 as usize].reached_by_queries() == [0, 1]) - .flat_map(|group| &memo.candidates[group.0 as usize]), - ); - assert!(!found.is_empty()); - assert!( - found.iter().all(|f| *f == Some(0.01 / 10_000.0)), - "{found:?}" - ); - } + let _ = (memo, per_computation); + todo!("#1207 PR 3: add each way to compute as candidates, one per listed θ") } diff --git a/crates/logical-asap-optimization/src/pass1/memo_population.rs b/crates/logical-asap-optimization/src/pass1/memo_population.rs deleted file mode 100644 index b3b411f1a..000000000 --- a/crates/logical-asap-optimization/src/pass1/memo_population.rs +++ /dev/null @@ -1,1231 +0,0 @@ -//! The hash-consed group DAG and Pass 1's implementation candidates per -//! group (design steps 1 and 2, Pass 1 part). -//! -//! Groups are the workload's logical DAG interned by -//! [`crate::stage0::cse::share_common_sub_dags`], the identical-expression rule's legality -//! included: structurally equal nodes are one group when sharing them is -//! legal: an ASAP node, or a non-ASAP node that returns the same rows for -//! the same input rows (`NonASAPOp::is_shareable_when_equal`). -//! A unique key is not needed; an unordered `LIMIT` stays one group per -//! reader. -//! -//! A Pass 1 target's aggregate group gets one candidate per combination of -//! its measures' Pass 1 candidates: the aggregate alone composed by -//! [`super::per_computation_candidates::compose_logical_candidate`] with that choice, every target beneath it -//! exact. Its local nodes are the composed nodes above the first node -//! equal to one of the group's descendant groups, which are its children. -//! The local nodes read each child group through its node (the same -//! `Rc`), and the candidate outputs the group's schema, so a parent reads -//! it in the group's place (a query's top-k root, which no parent reads, -//! may output the ranked value beside the key). -//! -//! A fused computation is one more candidate of the outer group, whatever -//! else reads the groups it absorbs: those keep their own candidates for -//! their other readers. A candidate equal to one the group already has (the -//! same structural id, input groups, fused groups and accuracy budget parts) is not -//! added again. -//! -//! A query whose value combines `n` approximate inputs splits its budget -//! into `n` equal shares, so a summary's size depends on its readers: each -//! sketch candidate is offered once per share `1..=n`, `n` the most -//! approximate inputs a query reaching the group can have -//! ([`CandidateForMemoGroup::accuracy_budget_parts`]). This records the sizing -//! configuration used during generation; the concrete sketch parameters carry -//! the resulting error contract. The connection planner may select a conservative -//! larger configuration, and validates the recomposed query-root guarantee. - -use asap_types::stages::HowToComputeAggregateMeasure; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use super::per_computation_candidates::{ - compose_logical_candidate_with_accuracy_budget_parts, PerComputationCandidates, -}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; - -use asap_types::stages::{ - CandidateForMemoGroup, DagOfLogicalMemoGroups, Memo, MemoGroup, MemoGroupId, - OriginOfCandidateForMemoGroup, -}; - -/// A Pass 1 inventory does not belong to the initial logical memo. -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum CandidatePopulationError { - #[error("Pass 1 root dependencies differ from the initial logical memo")] - RootDependenciesChanged, - #[error("Pass 1 target {target} is not a valid measure of an initial aggregate group")] - InvalidTarget { target: usize }, - #[error("Pass 1 target {target} references missing absorbed target {absorbed}")] - InvalidAbsorption { target: usize, absorbed: usize }, -} - -/// Populate implementation candidates without changing initial group identities. -/// The inventory must have walked `initial`'s groups -/// (`PerComputationCandidates::walked`); its targets name them by id. -pub fn try_populate_pass1( - initial: DagOfLogicalMemoGroups, - per_computation: &PerComputationCandidates, -) -> Result { - if !per_computation.walked(&initial) { - return Err(CandidatePopulationError::RootDependenciesChanged); - } - for (index, target) in per_computation.targets.iter().enumerate() { - if !matches!( - per_computation.aggregate(target).non_asap(), - Some(asap_types::ir::NonASAPOp::Aggregate { measures, .. }) if target.measure_index < measures.len() - ) { - return Err(CandidatePopulationError::InvalidTarget { target: index }); - } - if let Some(fused) = &target.fused_computation { - if let Some(&absorbed) = fused - .absorbed_targets - .iter() - .find(|&&absorbed| absorbed >= per_computation.targets.len()) - { - return Err(CandidatePopulationError::InvalidAbsorption { - target: index, - absorbed, - }); - } - } - } - let candidates = pass1_candidates(per_computation, &initial.groups); - Ok(Memo { - groups: initial.groups, - query_roots: initial.query_roots, - candidates, - }) -} - -/// `g` and every group below it. -fn reach(groups: &[MemoGroup], g: MemoGroupId, reached: &mut HashSet) { - if reached.insert(g) { - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, reached); - } - } -} - -/// Every group's Pass 1 candidates, by group index. -fn pass1_candidates( - per_computation: &PerComputationCandidates, - groups: &[MemoGroup], -) -> Vec> { - // Each target aggregate group's targets, one per measure: Pass 1 walks - // the groups, so an aggregate several queries read has one target per - // measure. - let mut targets_of: HashMap> = HashMap::new(); - for (t, target) in per_computation.targets.iter().enumerate() { - targets_of - .entry(target.aggregate_group) - .or_default() - .push(t); - } - // The groups' own nodes, which the composed candidates read. - let logical_nodes: Rc<[Rc]> = groups - .iter() - .map(|group| Rc::clone(&group.logical_node)) - .collect(); - let own = |g: usize| CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&groups[g].logical_node), - input_groups: groups[g].child_groups.clone(), - origin: OriginOfCandidateForMemoGroup::Pass1, - accuracy_budget_parts: 1, - }; - // Each query's most approximate inputs: its targets that offer a - // summary, each one sketch state at most. - let offers_a_summary = |t: usize| { - let target = &per_computation.targets[t]; - let summary = |c: &HowToComputeAggregateMeasure| { - matches!(c, HowToComputeAggregateMeasure::Summary(_)) - }; - target.ways_to_compute.iter().any(summary) - || target - .fused_computation - .as_ref() - .is_some_and(|fused| fused.ways_to_compute.iter().any(summary)) - }; - let mut most_approximate_inputs = vec![0usize; per_computation.roots.len()]; - for (g, targets) in &targets_of { - let summaries = targets.iter().filter(|&&t| offers_a_summary(t)).count(); - for &q in &groups[g.0 as usize].queries { - most_approximate_inputs[q] += summaries; - } - } - (0..groups.len()) - .map(|g| match targets_of.get(&MemoGroupId(g as u32)) { - None => vec![own(g)], - Some(targets) => target_candidates( - per_computation, - groups, - &logical_nodes, - MemoGroupId(g as u32), - targets, - groups[g] - .queries - .iter() - .map(|&q| most_approximate_inputs[q]) - .max() - .unwrap_or(1) - .max(1), - ) - .unwrap_or_else(|| vec![own(g)]), - }) - .collect() -} - -/// The candidates of target aggregate group `g`, whose measures' targets -/// are `targets`: one per combination of their Pass 1 candidates (the last -/// target varying fastest), each the aggregate composed alone with that -/// choice. `None` when none composes. -fn target_candidates( - per_computation: &PerComputationCandidates, - groups: &[MemoGroup], - logical_nodes: &Rc<[Rc]>, - g: MemoGroupId, - targets: &[usize], - max_accuracy_budget_parts: usize, -) -> Option> { - let aggregate = Rc::clone(&groups[g.0 as usize].logical_node); - // The groups a candidate's private nodes may bottom out at, by key. - let mut below = HashSet::new(); - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, &mut below); - } - let mut structural_ids = NodeToStructuralIdCache::default(); - let mut descendant: HashMap = HashMap::new(); - let mut ordered: Vec = below.into_iter().collect(); - ordered.sort_unstable(); - for d in ordered { - descendant - .entry(structural_ids.structural_id(&groups[d.0 as usize].logical_node)) - .or_insert(d); - } - let own_key = structural_ids.structural_id(&groups[g.0 as usize].logical_node); - let mut candidates = Vec::new(); - // Each candidate's identity in the memo: its node's structural id, its - // input groups, the groups it fuses and its accuracy budget parts. A candidate - // equal to one already listed is not added again. - let mut known = HashSet::new(); - let mut choice = vec![0; per_computation.targets.len()]; - loop { - let mut implements = vec![g]; - for &t in targets { - if let Some((_, Some(absorbs))) = per_computation.targets[t].computation_for(choice[t]) - { - for &a in absorbs { - let absorbed = per_computation.targets[a].aggregate_group; - if !implements.contains(&absorbed) { - implements.push(absorbed); - } - } - } - } - for accuracy_budget_parts in 1..=max_accuracy_budget_parts { - let Ok(composed) = compose_alone( - per_computation, - logical_nodes, - &aggregate, - targets, - &choice, - accuracy_budget_parts, - ) else { - continue; - }; - let Some(QueryRoot::Operator(root)) = composed.into_iter().next() else { - continue; - }; - if accuracy_budget_parts > 1 && !builds_a_sketch(&root) { - break; - } - let (node, children) = if structural_ids.structural_id(&root) == own_key { - ( - Rc::clone(&groups[g.0 as usize].logical_node), - groups[g.0 as usize].child_groups.clone(), - ) - } else { - let mut children = Vec::new(); - let node = over_groups( - &root, - groups, - &mut structural_ids, - &descendant, - &mut children, - &mut HashMap::new(), - ); - // A parent reads the candidate in the group's place, so it - // outputs the group's schema when it has the same columns, - // with the group's metadata (composition may derive a closed - // label set where the frontend's node leaves it open). A - // query's top-k root may output other columns (the ranked - // value beside the key): it keeps its own. - let group_node = &groups[g.0 as usize].logical_node; - let same_columns = node.schema.fields == group_node.schema.fields - && node.schema.time_index == group_node.schema.time_index; - let node = if !same_columns || node.schema == group_node.schema { - node - } else { - Rc::new(OperatorNode { - schema: group_node.schema.clone(), - ..(*node).clone() - }) - }; - (node, children) - }; - if !known.insert(( - structural_ids.structural_id(&node), - children.clone(), - implements[1..].to_vec(), - accuracy_budget_parts, - )) { - continue; - } - candidates.push(CandidateForMemoGroup { - fused_groups: implements[1..].to_vec(), - root_node: node, - input_groups: children, - origin: OriginOfCandidateForMemoGroup::Pass1, - accuracy_budget_parts, - }); - } - // Mixed-radix over this group's targets only. - let mut advanced = false; - for &t in targets.iter().rev() { - choice[t] += 1; - if choice[t] < per_computation.targets[t].choice_count() { - advanced = true; - break; - } - choice[t] = 0; - } - if !advanced { - break; - } - } - (!candidates.is_empty()).then_some(candidates) -} - -/// `aggregate` composed alone with `choice[t]` for each of its measures' -/// `targets`, every other target left exact: the inventory holds only -/// `targets`, and the targets a chosen fused computation absorbs (not -/// computed then). -fn compose_alone( - per_computation: &PerComputationCandidates, - logical_nodes: &Rc<[Rc]>, - aggregate: &Rc, - targets: &[usize], - choice: &[usize], - accuracy_budget_parts: usize, -) -> Result, super::per_computation_candidates::PerComputationCandidateError> { - let mut kept: Vec = targets.to_vec(); - for &t in targets { - if let Some((_, Some(absorbs))) = per_computation.targets[t].computation_for(choice[t]) { - for &a in absorbs { - if !kept.contains(&a) { - kept.push(a); - } - } - } - } - let position = |t: usize| kept.iter().position(|&k| k == t); - let alone = PerComputationCandidates { - roots: vec![QueryRoot::Operator(Rc::clone(aggregate))], - logical_nodes: Rc::clone(logical_nodes), - targets: kept - .iter() - .map(|&t| { - let mut target = per_computation.targets[t].clone(); - if let Some(fused) = &mut target.fused_computation { - // Absorbed targets not kept are never absorbed here: the - // fusion is kept only when chosen, with all of them. - match fused - .absorbed_targets - .iter() - .map(|&a| position(a)) - .collect() - { - Some(absorbs) => fused.absorbed_targets = absorbs, - None => target.fused_computation = None, - } - } - target - }) - .collect(), - pruned_ways_to_compute: Vec::new(), - }; - let alone_choice: Vec = kept - .iter() - .map(|&t| if targets.contains(&t) { choice[t] } else { 0 }) - .collect(); - compose_logical_candidate_with_accuracy_budget_parts( - &alone, - &alone_choice, - accuracy_budget_parts, - ) -} - -/// `node` with each first node on a path down from its children that -/// equals a group in `descendant` replaced by that group's node (the same -/// `Rc`), and those groups pushed to `children` in the order met: the -/// candidate's private nodes over its child groups. -fn over_groups( - node: &Rc, - groups: &[MemoGroup], - structural_ids: &mut NodeToStructuralIdCache, - descendant: &HashMap, - children: &mut Vec, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for child in node.children() { - let key = Rc::as_ptr(child); - if replaced.contains_key(&key) { - continue; - } - let new = match descendant.get(&structural_ids.structural_id(child)) { - Some(&group) => { - if !children.contains(&group) { - children.push(group); - } - Rc::clone(&groups[group.0 as usize].logical_node) - } - None => match rebuilt.get(&key) { - Some(done) => Rc::clone(done), - None => { - let done = - over_groups(child, groups, structural_ids, descendant, children, rebuilt); - rebuilt.insert(key, Rc::clone(&done)); - done - } - }, - }; - replaced.insert(key, new); - } - if node - .children() - .into_iter() - .all(|child| Rc::ptr_eq(child, &replaced[&Rc::as_ptr(child)])) - { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| Rc::clone(&replaced[&Rc::as_ptr(child)])), - ..(**node).clone() - }) -} - -/// Whether `node`'s private nodes build a sketch, which its accuracy budget parts size. -fn builds_a_sketch(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|n| { - matches!( - n.asap(), - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), - .. - } - ) - ) - }) -} - -/// Interned structural identities: the operator over its children's identities, -/// without the schema, guarantee, timing and materialization composition -/// derives. Memoized per node (the cache keeps each node alive, so its -/// address is not reused). -#[derive(Default)] -pub(crate) struct NodeToStructuralIdCache { - id_of_node: HashMap<*const OperatorNode, (Rc, u64)>, - id_of_encoded_operator: HashMap, u64>, -} - -impl NodeToStructuralIdCache { - pub(crate) fn structural_id(&mut self, node: &Rc) -> u64 { - if let Some((_, key)) = self.id_of_node.get(&Rc::as_ptr(node)) { - return *key; - } - let children: HashMap<*const OperatorNode, u64> = node - .children() - .into_iter() - .map(|child| (Rc::as_ptr(child), self.structural_id(child))) - .collect(); - let operator = node.operator.map_children(|c| children[&Rc::as_ptr(c)]); - // MessagePack preserves non-finite floats and signed zero. HashMap - // compares the complete byte key, so a hash collision cannot merge - // different computations. Cloned/recomposed NaN literals retain identity. - let encoded = - rmp_serde::to_vec(&operator).expect("an IR operator over child IDs must serialize"); - let next = self.id_of_encoded_operator.len() as u64; - let key = *self.id_of_encoded_operator.entry(encoded).or_insert(next); - self.id_of_node - .insert(Rc::as_ptr(node), (Rc::clone(node), key)); - key - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::stage0::generate_dag_of_logical_memo_groups; - - #[test] - fn structural_keys_preserve_nonfinite_and_signed_zero_literals() { - let input = crate::test_support::lower_promql( - "m", - asap_types::ir::properties::AccuracyTarget::exact(), - ); - let literal = |value| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - asap_types::ir::NonASAPOp::Project { - cols: vec![asap_types::ir::ProjectItem { - alias: Some("value".into()), - expr: asap_types::ir::ScalarExpr::literal_f64(value), - }], - qualifier: None, - child: Rc::clone(&input), - }, - )) - .unwrap() - }; - let mut structural_ids = NodeToStructuralIdCache::default(); - for (left, right) in [ - (f64::NAN, f64::INFINITY), - (f64::INFINITY, f64::NEG_INFINITY), - (0.0, -0.0), - ] { - assert_ne!( - structural_ids.structural_id(&literal(left)), - structural_ids.structural_id(&literal(right)) - ); - } - assert_eq!( - structural_ids.structural_id(&literal(1.0)), - structural_ids.structural_id(&literal(1.0)) - ); - let node = literal(f64::NAN); - assert_eq!( - structural_ids.structural_id(&node), - structural_ids.structural_id(&literal(f64::NAN)) - ); - } - - fn populate_pass1( - initial: DagOfLogicalMemoGroups, - inventory: &PerComputationCandidates, - ) -> Memo { - try_populate_pass1(initial, inventory).unwrap() - } - - fn memo_of(inventory: &PerComputationCandidates) -> Memo { - let roots: Vec<_> = inventory.roots.clone(); - populate_pass1(generate_dag_of_logical_memo_groups(&roots), inventory) - } - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - - use asap_types::ir::ASAPOp; - - /// The cross-stage API rejects mismatched root identities without panicking. - #[test] - fn population_rejects_an_inventory_from_other_roots() { - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.95, m[5m])", - AccuracyTarget::max_error(0.01), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - // An inventory is tied to its roots by structure: the same query - // text lowered again is the same DAG, so the other roots must differ. - let other_roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.5, n[1m])", - AccuracyTarget::max_error(0.01), - ))]; - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&other_roots)) - .unwrap(); - assert!(matches!( - try_populate_pass1(initial, &inventory), - Err(CandidatePopulationError::RootDependenciesChanged) - )); - } - - /// Invalid target and fusion indices fail at the stage boundary, not in composition. - #[test] - fn population_checks_target_and_absorption_indices() { - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.95, m[5m])", - AccuracyTarget::max_error(0.01), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut invalid_measure = inventory.clone(); - invalid_measure.targets[0].measure_index = usize::MAX; - assert!(matches!( - try_populate_pass1(initial.clone(), &invalid_measure), - Err(CandidatePopulationError::InvalidTarget { target: 0 }) - )); - let mut invalid_absorption = inventory; - let missing = invalid_absorption.targets.len(); - invalid_absorption.targets[0].fused_computation = - Some(asap_types::stages::FusedComputation { - absorbed_targets: vec![missing], - ways_to_compute: Vec::new(), - }); - assert!(matches!(try_populate_pass1(initial, &invalid_absorption), - Err(CandidatePopulationError::InvalidAbsorption { target: 0, absorbed }) if absorbed == missing)); - } - - /// Capability pruning must change implementations, not logical identities. - #[test] - fn initial_groups_survive_pass1_expansion_and_pruning() { - let roots: Vec<_> = [ - "quantile_over_time(0.95, m[5m])", - "quantile_over_time(0.99, m[5m])", - ] - .iter() - .map(|text| QueryRoot::Operator(lower_promql(text, AccuracyTarget::max_error(0.01)))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let original_groups = initial.groups.clone(); - let original_roots = initial.query_roots.clone(); - let inventory = generate_per_computation_candidates(&initial).unwrap(); - let expanded = populate_pass1(initial.clone(), &inventory); - let restricted_inventory = inventory.without_unsupported_summaries(Some(&[])); - let restricted = populate_pass1(initial, &restricted_inventory); - for memo in [&expanded, &restricted] { - assert_eq!(memo.groups.len(), original_groups.len()); - for (before, after) in original_groups.iter().zip(&memo.groups) { - assert!(Rc::ptr_eq(&before.logical_node, &after.logical_node)); - assert_eq!(before.child_groups, after.child_groups); - assert_eq!(before.parents, after.parents); - assert_eq!(before.queries, after.queries); - } - assert_eq!(memo.query_roots, original_roots); - assert_ne!( - group_of_kind(memo, 0, "Aggregate"), - group_of_kind(memo, 1, "Aggregate") - ); - assert!(memo.candidates.iter().all(|c| !c.is_empty())); - } - assert!( - expanded.candidates.iter().map(Vec::len).sum::() - > restricted.candidates.iter().map(Vec::len).sum::() - ); - } - - /// The memo of PromQL `queries`, each with accuracy target `target`. - fn memo(queries: &[&str], target: AccuracyTarget) -> Memo { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, target.clone()))) - .collect(); - memo_of( - &generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(), - ) - } - - /// The group of the first node of `kind` reached from query `q`'s root, - /// walking down the groups' first children. - fn group_of_kind(memo: &Memo, q: usize, kind: &str) -> usize { - let mut g = memo.query_roots[q].as_operator().unwrap().0 as usize; - while memo.groups[g].logical_node.operator.kind_name() != kind { - g = memo.groups[g].child_groups[0].0 as usize; - } - g - } - - /// The summary algorithms a candidate's private nodes build. - fn algorithms(candidate: &CandidateForMemoGroup) -> Vec { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(format!("{family:?}")), - _ => None, - }) - .collect() - } - - /// The per-job rate sum both queries compute is one group, reached by - /// both, the root of the first and read by the second's arithmetic. - #[test] - fn identical_sub_expressions_of_two_queries_are_one_group() { - let memo = memo( - &[ - "sum by (job) (rate(m[5m]))", - "sum by (job) (rate(m[5m])) * 2", - ], - AccuracyTarget::exact(), - ); - let sum = group_of_kind(&memo, 0, "Aggregate"); - assert_eq!(group_of_kind(&memo, 1, "Aggregate"), sum); - let group = &memo.groups[sum]; - assert_eq!(group.queries, vec![0, 1]); - assert_eq!( - asap_types::stages::queries_rooted_at(&memo.query_roots, MemoGroupId(sum as u32)), - vec![0] - ); - let product = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert_eq!(group.parents, vec![MemoGroupId(product as u32)]); - assert_eq!( - memo.groups[product].child_groups, - vec![MemoGroupId(sum as u32)] - ); - // Children come before parents. - for (g, group) in memo.groups.iter().enumerate() { - assert!(group.child_groups.iter().all(|c| (c.0 as usize) < g)); - } - } - - /// A quantile's aggregate group has an exact candidate (its own node over - /// its input's group) and a KLL and a DDSketch candidate over the same - /// input group. - #[test] - fn a_quantile_target_group_has_exact_kll_and_ddsketch_candidates() { - let memo = memo( - &["quantile_over_time(0.9, m[5m])"], - AccuracyTarget::max_error(0.01), - ); - let quantile = group_of_kind(&memo, 0, "Aggregate"); - let input = memo.groups[quantile].child_groups.clone(); - let candidates = &memo.candidates[quantile]; - assert_eq!(candidates.len(), 3); - assert!(Rc::ptr_eq( - &candidates[0].root_node, - &memo.groups[quantile].logical_node - )); - assert!(algorithms(&candidates[0]).is_empty()); - assert!(algorithms(&candidates[1])[0].contains("Kll")); - assert!(algorithms(&candidates[2])[0].contains("DDSketch")); - for candidate in candidates { - assert!(candidate.fused_groups.is_empty()); - assert_eq!(candidate.input_groups, input); - assert_eq!(candidate.origin, OriginOfCandidateForMemoGroup::Pass1); - } - } - - /// The whole-expression top-k heap over the per-job sums implements both - /// aggregates' groups, and reads the sums' input group; the exact top-k - /// reads the sums' group. - #[test] - fn a_fused_candidate_lists_both_groups_it_implements() { - let memo = memo( - &["topk by (job) (10, sum_over_time(m[1m]))"], - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let topk = group_of_kind(&memo, 0, "Aggregate"); - let sums = memo.groups[topk].child_groups[0]; - let sums_input = memo.groups[sums.0 as usize].child_groups.clone(); - let candidates = &memo.candidates[topk]; - assert_eq!(candidates[0].input_groups, vec![sums]); - let fused: Vec<_> = candidates - .iter() - .filter(|c| !c.fused_groups.is_empty()) - .collect(); - assert!(!fused.is_empty()); - for candidate in fused { - assert_eq!(candidate.fused_groups, vec![sums]); - assert_eq!(candidate.input_groups, sums_input); - assert!(algorithms(candidate)[0].contains("Heap")); - } - } - - /// Every candidate reads its child groups through their nodes (the same - /// `Rc`), and outputs its group's schema, so a parent can read it in - /// the group's place. - #[test] - fn candidates_read_child_group_nodes_and_output_the_group_schema() { - for (queries, target) in [ - ( - vec!["quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])"], - AccuracyTarget::max_error(0.01), - ), - ( - vec!["topk by (job) (10, sum_over_time(m[1m]))"], - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ), - (vec!["sum by (job) (rate(m[5m]))"], AccuracyTarget::exact()), - ] { - let memo = memo(&queries, target); - for (g, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - let reached: HashSet<*const OperatorNode> = - OperatorNode::reachable(&candidate.root_node) - .iter() - .map(Rc::as_ptr) - .collect(); - for child in &candidate.input_groups { - let node = &memo.groups[child.0 as usize].logical_node; - assert!(reached.contains(&Rc::as_ptr(node)), "{queries:?} g{g}"); - } - assert_eq!( - candidate.root_node.schema, - memo.groups[g].logical_node.schema - ); - } - } - } - } - - /// The sketch summaries of the first query of `queries` composed with - /// every target at its `algorithm` summary, by [`super::per_computation_candidates::compose_logical_candidate`]: - /// as the exhaustive search sizes them. - fn composed_sketches( - queries: &[&str], - target: &AccuracyTarget, - algorithm: &str, - ) -> Vec { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, target.clone()))) - .collect(); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - matches!(c, HowToComputeAggregateMeasure::Summary(a) if format!("{a:?}") == algorithm) - }) - .unwrap() - }) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - unreachable!("an operator root") - }; - OperatorNode::reachable(root) - .iter() - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => Some(format!("{family:?}")), - _ => None, - }) - .collect() - } - - /// The sketch candidates of `memo`'s group `g`: each one's summary and - /// accuracy budget parts. - fn sketch_shares(memo: &Memo, g: usize) -> Vec<(String, usize)> { - memo.candidates[g] - .iter() - .filter(|c| !algorithms(c).is_empty()) - .map(|c| (algorithms(c)[0].clone(), c.accuracy_budget_parts)) - .collect() - } - - /// `q0.9 / q0.5` beside a plain `q0.9`: each of the ratio's quantile - /// groups offers every sketch for the whole budget and for half of it, - /// since its query has two inputs that may be approximate. The plain - /// query's quantile is the numerator's group: equal per-series quantiles - /// return the same rows, so they are one group with both readers. A ratio of quantiles carries no - /// guarantee, so composition does not split its budget: the - /// whole-budget candidate is the exhaustive search's sizing, and the - /// half-budget one, larger, is never the cheaper valid one. - #[test] - fn a_ratio_operand_has_a_candidate_per_accuracy_budget_parts() { - let queries = [ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - ]; - let target = AccuracyTarget::max_error(0.01); - let memo = memo(&queries, target.clone()); - let ratio = memo.query_roots[0].as_operator().unwrap().0 as usize; - let operand = memo.groups[ratio].child_groups[0].0 as usize; - for operand in &memo.groups[ratio].child_groups { - let shares: Vec = sketch_shares(&memo, operand.0 as usize) - .into_iter() - .map(|(_, share)| share) - .collect(); - assert_eq!(shares, vec![1, 2, 1, 2]); - } - let plain = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert_eq!(plain, operand); - let whole_kll = &sketch_shares(&memo, operand)[0]; - assert_eq!(whole_kll.1, 1); - assert_eq!( - composed_sketches(&queries[..1], &target, "Kll"), - vec![whole_kll.0.clone(); 2] - ); - } - - /// A difference of two Count-Min counts splits its budget: each - /// operand's half-budget candidate is the Count-Min composition sizes - /// for the difference with both counts sketched. - #[test] - fn a_split_budget_matches_its_half_budget_candidate() { - let queries = ["count by (job) (a) - count by (job) (b)"]; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let memo = memo(&queries, target.clone()); - let difference = memo.query_roots[0].as_operator().unwrap().0 as usize; - let operand = memo.groups[difference].child_groups[0].0 as usize; - let half = sketch_shares(&memo, operand) - .into_iter() - .find(|(family, share)| family.contains("Cms") && *share == 2) - .unwrap(); - assert_eq!(composed_sketches(&queries, &target, "Cms"), vec![half.0; 2]); - } - - /// A group appended after its readers (as a transformation appends a - /// ranking group) is ordered before every group whose candidate reads - /// it. - #[test] - fn topological_order_puts_appended_groups_before_their_readers() { - let mut memo = memo( - &["quantile_over_time(0.9, m[5m])"], - AccuracyTarget::max_error(0.01), - ); - assert_eq!( - memo.topological_order().unwrap(), - (0..memo.groups.len() as u32) - .map(MemoGroupId) - .collect::>() - ); - let quantile = group_of_kind(&memo, 0, "Aggregate"); - let appended = MemoGroupId(memo.groups.len() as u32); - memo.groups.push(memo.groups[1].clone()); - memo.candidates.push(memo.candidates[1].clone()); - let mut reader = memo.candidates[quantile][0].clone(); - reader.input_groups = vec![appended]; - memo.candidates[quantile].push(reader); - let order = memo.topological_order().unwrap(); - let position = |g: MemoGroupId| order.iter().position(|&x| x == g).unwrap(); - assert!(position(appended) < position(MemoGroupId(quantile as u32))); - // A cycle has no order. - memo.candidates[1][0] - .input_groups - .push(MemoGroupId(quantile as u32)); - assert_eq!(memo.topological_order(), None); - } - - /// The workload generator of `joint_search_equivalence`: a seeded - /// xorshift generator. - struct Rng(u64); - - impl Rng { - fn below(&mut self, n: usize) -> usize { - self.0 ^= self.0 << 13; - self.0 ^= self.0 >> 7; - self.0 ^= self.0 << 17; - (self.0 % n as u64) as usize - } - - fn pick<'a, T>(&mut self, items: &'a [T]) -> &'a T { - &items[self.below(items.len())] - } - } - - /// One random PromQL query of `joint_search_equivalence`. - fn random_query(rng: &mut Rng, gauges: &[&str]) -> String { - let m = rng.pick(gauges); - let group = rng.pick(&["job", "job, instance", "instance"]); - let window = rng.pick(&["1m", "5m", "10m"]); - let phi = rng.pick(&["0.5", "0.9", "0.99"]); - let k = rng.pick(&["3", "5"]); - match rng.below(10) { - 0 => format!("sum by ({group}) (sum_over_time({m}[{window}]))"), - 1 => format!("avg by ({group}) (quantile_over_time({phi}, {m}[{window}]))"), - 2 => format!("quantile_over_time({phi}, {m}[{window}])"), - 3 => format!("topk by ({group}) ({k}, sum_over_time({m}[{window}]))"), - 4 => format!("sum by ({group}) (rate(c_total[{window}]))"), - 5 => format!("max by ({group}) (max_over_time({m}[{window}]))"), - 6 => format!("count_over_time({m}[{window}])"), - 7 => format!("sum by ({group}) (increase(c_total[{window}]))"), - 8 => format!("stddev_over_time({m}[{window}])"), - _ => format!("topk({k}, sum by ({group}) (rate(c_total[{window}])))"), - } - } - - /// The roots of `joint_search_equivalence`'s random workload `seed` - /// over `gauges`, lowered as the planner lowers PromQL (with series - /// identity). - fn random_roots(seed: u64, gauges: &[&str]) -> Vec { - use asap_types::ir::properties::AccuracyMetric; - use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, - PrometheusMetricType, Query, QueryLanguage, QueryRequirements, QueryWorkload, - RepeatedDemand, RepeatingEntry, RepetitionInterval, SourceFacts, - }; - let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); - let queries = 2 + rng.below(3); - let entries = (0..queries) - .map(|_| RepeatingEntry { - query: Query(random_query(&mut rng, gauges)), - demand: RepeatedDemand::FixedInterval(RepetitionInterval( - *rng.pick(&[60_000u32, 300_000]), - )), - requirements: QueryRequirements { - accuracy: rng - .pick(&[ - AccuracyTarget::exact(), - AccuracyTarget::max_error(0.05), - AccuracyTarget::max_error(0.01).failure_probability(0.01), - AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Rank), - ]) - .clone(), - max_latency_ms: None, - }, - predictability: Predictability::Predictable { known_at: None }, - range_evaluation: None, - }) - .collect(); - let source = |metric_type| SourceFacts { - series_count: Some(1_000), - metric_type: Some(metric_type), - sample_interval: Some(DurationMs(15_000)), - ..SourceFacts::default() - }; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(entries), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 2_000.0 / 15.0, - }, - sources: gauges - .iter() - .map(|g| (g.to_string(), source(PrometheusMetricType::Gauge))) - .chain([("c_total".to_string(), source(PrometheusMetricType::Counter))]) - .collect(), - }), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator( - asap_types::ir::schema::schema_support::with_promql_series_identity(&node) - .unwrap(), - ), - scalar => scalar, - }) - .collect() - } - - /// Whether a composed sub-DAG is some combination of the memo's - /// per-group candidates: a candidate's private nodes equal the composed - /// nodes operator by operator (schemas and guarantees aside, which - /// composition derives), and each child group's node is matched by a - /// candidate of that group in turn. - struct Combination<'a> { - memo: &'a Memo, - matched: HashMap<(usize, *const OperatorNode), bool>, - own: HashMap<*const OperatorNode, (Rc, serde_json::Value)>, - } - - impl Combination<'_> { - fn own_fields(&mut self, node: &Rc) -> serde_json::Value { - self.own - .entry(Rc::as_ptr(node)) - .or_insert_with(|| { - let operator = node.operator.map_children(|_| ()); - (Rc::clone(node), serde_json::to_value(&operator).unwrap()) - }) - .1 - .clone() - } - - fn group(&mut self, g: usize, composed: &Rc) -> bool { - let key = (g, Rc::as_ptr(composed)); - if let Some(&done) = self.matched.get(&key) { - return done; - } - self.matched.insert(key, false); - let memo = self.memo; - let found = memo.candidates[g] - .iter() - .any(|c| self.private(&c.root_node, composed, &c.input_groups)); - self.matched.insert(key, found); - found - } - - fn private( - &mut self, - node: &Rc, - composed: &Rc, - children: &[MemoGroupId], - ) -> bool { - let memo = self.memo; - if let Some(child) = children - .iter() - .find(|c| Rc::ptr_eq(&memo.groups[c.0 as usize].logical_node, node)) - { - return self.group(child.0 as usize, composed); - } - let (ours, theirs) = (node.children(), composed.children()); - ours.len() == theirs.len() - && self.own_fields(node) == self.own_fields(composed) - && ours - .iter() - .zip(&theirs) - .all(|(a, b)| self.private(a, b, children)) - } - } - - /// For the random workloads of `joint_search_equivalence` (over one - /// gauge and over two), every whole-workload Pass 1 choice composes, - /// for each query, a plan that is a combination of the memo's per-group - /// candidates: the memo misses no Pass 1 candidate. Every target's - /// group has the target's own and fused candidates, also a target a - /// fused computation absorbs; every candidate of a group a parent reads - /// outputs the group's schema. - #[test] - fn every_composed_pass1_candidate_is_a_combination_of_group_candidates() { - let mut checked = 0; - let mut missing = Vec::new(); - let mut other_schema = Vec::new(); - for gauges in [&["m"][..], &["m", "n"]] { - for seed in 1..=60u64 { - let roots = random_roots(seed, gauges); - let inventory = generate_per_computation_candidates( - &generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - if inventory.combinations() > 400 { - continue; - } - let memo = memo_of(&inventory); - // Each target's own and fused candidates, absorbed or not. - let group_of = |t: usize| { - memo.groups - .iter() - .position(|group| { - *group.logical_node == **inventory.aggregate(&inventory.targets[t]) - }) - .unwrap() - }; - for (t, target) in inventory.targets.iter().enumerate() { - let g = group_of(t); - if inventory.targets.iter().enumerate().any(|(u, other)| { - u != t && group_of(u) == g && other.measure_index != target.measure_index - }) { - continue; - } - let whole_budget = |fused: bool| { - memo.candidates[g] - .iter() - .filter(|c| { - c.accuracy_budget_parts == 1 && !c.fused_groups.is_empty() == fused - }) - .count() - }; - let fused = target - .fused_computation - .as_ref() - .map_or(0, |f| f.ways_to_compute.len()); - if whole_budget(false) != target.ways_to_compute.len() - || whole_budget(true) != fused - { - missing.push(format!("{gauges:?} seed {seed} target {t} candidates")); - } - } - for (g, group) in memo.groups.iter().enumerate() { - for candidate in &memo.candidates[g] { - if !group.parents.is_empty() - && candidate.root_node.schema != group.logical_node.schema - { - other_schema.push(format!("{gauges:?} seed {seed} g{g}")); - } - } - } - let root_group = |q: usize| memo.query_roots[q].as_operator().unwrap().0 as usize; - let mut combination = Combination { - memo: &memo, - matched: HashMap::new(), - own: HashMap::new(), - }; - let mut choice = vec![0; inventory.targets.len()]; - loop { - if !inventory.absorbed_choice_varies(&choice) { - if let Ok(composed) = compose_logical_candidate(&inventory, &choice) { - for (q, root) in composed.iter().enumerate() { - let QueryRoot::Operator(node) = root else { - continue; - }; - checked += 1; - if !combination.group(root_group(q), node) { - missing.push(format!("{gauges:?} seed {seed} q{q} {choice:?}")); - } - } - } - } - let mut advanced = false; - for t in (0..choice.len()).rev() { - choice[t] += 1; - if choice[t] < inventory.targets[t].choice_count() { - advanced = true; - break; - } - choice[t] = 0; - } - if !advanced { - break; - } - } - } - } - assert!(checked > 100, "{checked} plans checked"); - assert!( - missing.is_empty(), - "{} of {checked} missing: {missing:#?}", - missing.len() - ); - } - - /// Two SQL queries over one table, each lowered with its own scan, read - /// one scan group: equal scans are shared, a table's keyless rows - /// included. Their aggregates, grouped differently, stay two groups. - #[test] - fn equal_sql_table_scans_are_one_group() { - use crate::test_support::{agg, scan_from}; - use asap_types::ir::operator::{AggIntent, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let table = || { - scan_from( - Source::Table { - table_ref: "t".into(), - }, - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("a", DataType::Utf8, true), - Field::plain("b", DataType::Utf8, true), - ], - 0, - vec![], - ), - ) - }; - let sum = || AggIntent::Sum { col: Some(1) }; - let roots = vec![ - QueryRoot::Operator(agg(vec![2], sum(), table())), - QueryRoot::Operator(agg(vec![3], sum(), table())), - ]; - let memo = memo_of( - &generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(), - ); - assert_eq!(memo.groups.len(), 3); - assert_eq!(memo.groups[0].logical_node.operator.kind_name(), "Scan"); - assert_eq!(memo.groups[0].queries, vec![0, 1]); - assert_eq!(memo.groups[0].parents, vec![MemoGroupId(1), MemoGroupId(2)]); - } -} diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index ea2b891ca..0c0f53a40 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -3,7 +3,7 @@ //! `docs/design_docs/proposals/planner-layering.md`. //! //! [`generate_per_computation_candidates`] walks the memo groups of the -//! workload's hash-consed logical DAG ([`DagOfLogicalMemoGroups`]) and +//! workload's memo groups ([`Memo`]) and //! returns, for every eligible computation, its unranked candidates. A //! computation ([`ComputationTarget`]) is one measure of an `Aggregate` //! group whose intent has a realization besides plain exact execution, once @@ -21,17 +21,13 @@ //! has its own candidates: the whole-expression top-k over a per-item //! `sum` or `count` (`Summary` of a heap family) and the nested sum, a //! grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). -//! - Summary choices here are unsized families. Attaching them to memo -//! groups creates sized candidates, one per split of the accuracy budget -//! (`CandidateForMemoGroup::accuracy_budget_parts`), each sized for ε, its -//! accuracy budget parts and δ over the values its readers' targets cover; Pass 2 -//! inherits sizes or takes the strictest shared reader's, and Stage 3 only -//! checks. +//! - Summary choices here are unsized families. Adding them to memo groups +//! creates one build candidate per listed parameter configuration +//! (`parameter_list`); Stage 3 checks each query's composed accuracy at its +//! root. //! - Candidates are pruned only when provably unable to meet the accuracy //! requirement or to run, or when the exact candidate provably dominates -//! them (a PromQL top-k's heap over its instant vector, one row per item: -//! [`PruneReason::TopKHeapOverOneRowPerItem`]); each pruned candidate -//! keeps its [`PruneReason`]. +//! them (a PromQL top-k's heap over its instant vector, one row per item). //! - A measure filter or `HAVING` prunes nothing. The target records the //! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] //! per consumer `Filter` directly above the aggregate: a key-only predicate @@ -40,8 +36,8 @@ //! outputs then get guarantee `None`). So every candidate of a group //! computes the group's own rows. //! -//! The input must be a logical DAG: ASAP operators, execution timing or a -//! materialization are rejected. No cost model is used. The accuracy model +//! The input must be a logical DAG: ASAP operators are rejected. A logical +//! DAG has no physical properties (#1200). No cost model is used. The accuracy model //! is used only to prune summaries no query reading them can use //! ([`PerComputationCandidates::without_summaries_no_query_can_use`]); //! ranking and discarding valid candidates belong to Stage 3. @@ -89,7 +85,7 @@ //! per output value; attached to the memo groups, each candidate's //! approximate inputs are sized for the values its readers' targets cover //! (`δ / (n · V)`, -//! [`generate_per_computation_logical_memo_candidates`](crate::pass1::logical_memo_candidates::generate_per_computation_logical_memo_candidates)), +//! [`pass1_memo`](crate::pass1::logical_memo_candidates::pass1_memo)), //! their final sizes. Pass 2 inherits them or takes the strictest shared //! reader's, and Stage 3 only checks the composed accuracy. //! @@ -114,22 +110,20 @@ use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, Scala use asap_types::ir::schema::{DataType, Schema}; use asap_types::ir::schema::{ EntityIdentity, ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, - SketchAlgorithm, SketchKind, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, + SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, }; use asap_types::ir::{ ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, }; -use asap_types::physical::Materialization; use thiserror::Error; use asap_accuracy::composition::parse_raw_data_samples_sign_statistic; use asap_accuracy::estimators::hll; use asap_accuracy::propagation::{ - approximate_inputs, bind_guarantees, raw_data_samples_of_metric, root_guarantee, - with_guarantees, + bind_guarantees, raw_data_samples_of_metric, root_guarantee, with_guarantees, }; -use asap_accuracy::sizing::{accuracy_budget, sized, sizing_target, sketch_statistic}; +use asap_accuracy::sizing::{accuracy_budget, sizing_target, sketch_statistic}; use asap_accuracy::AccuracyModel; use asap_types::ir::properties::AccuracyGuarantee; use asap_types::workload::bounds::{declared_non_negative, series_read, values_per_result}; @@ -138,28 +132,23 @@ use asap_types::workload::DataWorkload; use crate::deployment_capabilities::{summary_supported, SupportedSummary}; use crate::pass1::realization::summary_candidates; use asap_types::stages::{ - ComputationTarget, DagOfLogicalMemoGroups, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, MemoGroupId, PruneReason, + ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, + HowToComputeAggregateMeasure, Memo, MemoGroupId, }; /// Unranked candidates of one measure and the candidates pruned for it. #[derive(Debug, Clone, PartialEq)] pub(super) struct ComputationChoices { pub candidates: Vec, - pub pruned: Vec<(HowToComputeAggregateMeasure, PruneReason)>, + pub pruned: Vec, } impl ComputationChoices { /// Prune matching candidates; a pruned exact accumulator form is replaced by /// plain `Exact`, so the target keeps one exact option. - fn prune( - &mut self, - reason: PruneReason, - applies: impl Fn(&HowToComputeAggregateMeasure) -> bool, - ) { + fn prune(&mut self, applies: impl Fn(&HowToComputeAggregateMeasure) -> bool) { let pruned = prune_own(&mut self.candidates, applies); - self.pruned - .extend(pruned.into_iter().map(|choice| (choice, reason))); + self.pruned.extend(pruned); } } @@ -183,22 +172,10 @@ fn prune_own( pruned } -/// A way to compute a target's measure that Pass 1 pruned, and why. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct PrunedWayToCompute { - /// The target, by index in [`PerComputationCandidates::targets`]. - pub target: usize, - /// Whether it was a way of the target's fused computation - /// ([`ComputationTarget::fused_computation`]) rather than its own. - pub of_fused_computation: bool, - pub way_to_compute: HowToComputeAggregateMeasure, - pub reason: PruneReason, -} - /// Compact Pass 1 inventory over the memo groups it walked: one target per /// measure of each aggregate group, however many queries read the group. /// Pass 1's working form, not its output: -/// [`generate_per_computation_logical_memo_candidates`](crate::pass1::generate_per_computation_logical_memo_candidates) +/// [`pass1_memo`](crate::pass1::pass1_memo) /// builds, prunes and attaches it to the memo groups. Public only to tests /// (feature `test-support`). #[derive(Debug, Clone)] @@ -212,56 +189,9 @@ pub struct PerComputationCandidates { /// The targets; a whole-workload choice has one digit per target, /// ranging over [`ComputationTarget::choice_count`]. pub targets: Vec, - /// The ways to compute the targets' measures that Pass 1 pruned, in the - /// order it pruned them. - pub(crate) pruned_ways_to_compute: Vec, } impl PerComputationCandidates { - /// Whether this inventory walked `dag`, or a DAG built the same way from - /// the same roots: the same groups (by structural id, in the same order) - /// and each query's root over the same groups. Its targets' group ids - /// then name `dag`'s groups. - pub(crate) fn walked(&self, dag: &DagOfLogicalMemoGroups) -> bool { - use super::memo_population::NodeToStructuralIdCache; - let mut structural_ids = NodeToStructuralIdCache::default(); - let group_of: HashMap<*const OperatorNode, MemoGroupId> = self - .logical_nodes - .iter() - .enumerate() - .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) - .collect(); - dag.groups.len() == self.logical_nodes.len() - && dag.query_roots.len() == self.roots.len() - && dag - .groups - .iter() - .zip(self.logical_nodes.iter()) - .all(|(group, node)| { - Rc::ptr_eq(&group.logical_node, node) - || structural_ids.structural_id(&group.logical_node) - == structural_ids.structural_id(node) - }) - && dag - .query_roots - .iter() - .zip(&self.roots) - .all(|(groups, root)| { - let nodes = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - matches!(groups, QueryRoot::Operator(_)) - == matches!(root, QueryRoot::Operator(_)) - && groups.operator_refs().len() == nodes.len() - && groups - .operator_refs() - .into_iter() - .zip(nodes) - .all(|(g, node)| group_of.get(&Rc::as_ptr(node)) == Some(g)) - }) - } - /// The `Aggregate` node of `target`'s memo group. pub fn aggregate(&self, target: &ComputationTarget) -> &Rc { &self.logical_nodes[target.aggregate_group.0 as usize] @@ -323,37 +253,18 @@ impl PerComputationCandidates { }) } - /// The ways to compute target `target`'s measure that Pass 1 pruned, - /// with their reasons, in pruning order: its own, or its fused - /// computation's when `of_fused_computation`. - #[cfg(any(test, feature = "test-support"))] - pub fn pruned_of_target( - &self, - target: usize, - of_fused_computation: bool, - ) -> Vec<(HowToComputeAggregateMeasure, PruneReason)> { - self.pruned_ways_to_compute - .iter() - .filter(|pruned| { - pruned.target == target && pruned.of_fused_computation == of_fused_computation - }) - .map(|pruned| (pruned.way_to_compute.clone(), pruned.reason)) - .collect() - } - /// This inventory without the summary candidates the executor does not /// support (`supported_summaries`, the deployment's /// `DeploymentCapabilities::supported_summaries`; `None` supports - /// every one): a target's own are moved to its pruned candidates with - /// [`PruneReason::UnsupportedByDeployment`], and so are a fused - /// computation's (with all of them pruned, the fusion is not offered). + /// every one): a target's own are removed, and so are a fused + /// computation's (with all of them removed, the fusion is not offered). /// The readout is the statistic the measure's estimate reads (the outer measure's for a /// fused computation). With every summary supported it is unchanged. pub fn without_unsupported_summaries( mut self, supported_summaries: Option<&[SupportedSummary]>, ) -> Self { - for (index, target) in self.targets.iter_mut().enumerate() { + for target in self.targets.iter_mut() { let aggregate = &self.logical_nodes[target.aggregate_group.0 as usize]; let Some(NonASAPOp::Aggregate { measures, .. }) = aggregate.non_asap() else { continue; @@ -366,24 +277,9 @@ impl PerComputationCandidates { matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if !summary_supported(supported_summaries, family, readout)) }; - let pruned = |of_fused_computation| { - move |way_to_compute| PrunedWayToCompute { - target: index, - of_fused_computation, - way_to_compute, - reason: PruneReason::UnsupportedByDeployment, - } - }; - let own = prune_own(&mut target.ways_to_compute, unsupported); - self.pruned_ways_to_compute - .extend(own.into_iter().map(pruned(false))); + prune_own(&mut target.ways_to_compute, unsupported); if let Some(fused) = &mut target.fused_computation { - let (removed, kept): (Vec<_>, _) = std::mem::take(&mut fused.ways_to_compute) - .into_iter() - .partition(unsupported); - fused.ways_to_compute = kept; - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(true))); + fused.ways_to_compute.retain(|way| !unsupported(way)); } } self @@ -391,8 +287,7 @@ impl PerComputationCandidates { /// This inventory without the summary candidates no query can use /// (#1047, accuracy A1 of #1095): a summary of a target (its own, or - /// its fused computation's) is moved to its pruned candidates with - /// [`PruneReason::NoErrorBoundForQueryTarget`] when, for every query + /// its fused computation's) is removed when, for every query /// whose root reaches the target's aggregate, /// `output_can_meet_target` says no: the query's output, composed /// with that summary and every other target exact, has no error bound @@ -413,7 +308,7 @@ impl PerComputationCandidates { /// pruned, and with every fused summary pruned the fusion is not /// offered. A summary whose composition fails is kept. /// - /// Pass 1's entry (`generate_per_computation_logical_memo_candidates`) + /// Pass 1's entry (`pass1_memo`) /// calls it only when the deployment's accuracy model and evidence /// cannot accept an output without its guarantee. pub fn without_summaries_no_query_can_use( @@ -471,9 +366,7 @@ impl PerComputationCandidates { }; let mut choices = vec![0; self.targets.len()]; choices[index] = choice; - let Ok(composed) = - compose_with_accuracy_budget_parts(&self, &roots, &choices, Some(1)) - else { + let Ok(composed) = compose(&self, &roots, &choices) else { continue; }; if !queries @@ -550,24 +443,11 @@ impl PerComputationCandidates { fused.ways_to_compute.get(choice - own).cloned() }) .collect(); - let pruned = |of_fused_computation| { - move |way_to_compute| PrunedWayToCompute { - target: index, - of_fused_computation, - way_to_compute, - reason: PruneReason::NoErrorBoundForQueryTarget, - } - }; - let removed = prune_own(&mut target.ways_to_compute, |way| own_pruned.contains(way)); - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(false))); + prune_own(&mut target.ways_to_compute, |way| own_pruned.contains(way)); if let Some(fused) = &mut target.fused_computation { - let (removed, kept): (Vec<_>, _) = std::mem::take(&mut fused.ways_to_compute) - .into_iter() - .partition(|way| fused_pruned.contains(way)); - fused.ways_to_compute = kept; - self.pruned_ways_to_compute - .extend(removed.into_iter().map(pruned(true))); + fused + .ways_to_compute + .retain(|way| !fused_pruned.contains(way)); } } self @@ -667,8 +547,6 @@ fn absorbed_targets(targets: &[ComputationTarget]) -> impl Iterator HavingPlacement { /// the group applies the same predicate, so every candidate of a shared /// aggregate computes the group's own rows (#1089). pub fn generate_per_computation_candidates( - dag: &DagOfLogicalMemoGroups, + memo: &Memo, ) -> Result { - let logical_nodes: Rc<[Rc]> = dag + let logical_nodes: Rc<[Rc]> = memo .groups .iter() .map(|group| Rc::clone(&group.logical_node)) @@ -813,7 +683,17 @@ pub fn generate_per_computation_candidates( .enumerate() .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) .collect(); - let roots = dag.roots_over_groups(); + // Each query's root over the groups' logical nodes, in workload order: + // a group several queries read is one node. + let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); + let roots: Vec = memo + .query_roots + .iter() + .map(|root| match root { + QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), + QueryRoot::Scalar(expr) => QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))), + }) + .collect(); let mut seen = HashSet::new(); // Each aggregate group with the first query that reaches it. let mut aggregates = Vec::new(); @@ -835,10 +715,6 @@ pub fn generate_per_computation_candidates( if !seen.insert(Rc::as_ptr(&node)) { continue; } - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized - { - return Err(PerComputationCandidateError::AssignedPhysicalAnnotation); - } for child in node.children() { consumers .entry(Rc::as_ptr(child)) @@ -852,7 +728,6 @@ pub fn generate_per_computation_candidates( } } let mut targets = Vec::new(); - let mut pruned_ways_to_compute = Vec::new(); for (query, node) in aggregates { let Some(NonASAPOp::Aggregate { measures, @@ -914,25 +789,15 @@ pub fn generate_per_computation_candidates( continue; } if matches!(intent, AggIntent::Avg { .. }) && nullable_avg_input(&node, intent) { - choices.prune(PruneReason::NullableAvgInput, |choice| { + choices.prune(|choice| { matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_)) }); } if matches!(intent, AggIntent::Count { .. }) && node.children()[0].result_kind == OperatorResultKind::Relation { - choices.prune(PruneReason::NoRelationalCountSketchRealization, |choice| { - matches!(choice, HowToComputeAggregateMeasure::Summary(_)) - }); + choices.prune(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_))); } - pruned_ways_to_compute.extend(choices.pruned.into_iter().map( - |(way_to_compute, reason)| PrunedWayToCompute { - target: targets.len(), - of_fused_computation: false, - way_to_compute, - reason, - }, - )); targets.push(ComputationTarget { aggregate_group, measure_index: measure, @@ -944,18 +809,13 @@ pub fn generate_per_computation_candidates( } } add_whole_expression_targets(&mut targets, &logical_nodes); - prune_topk_heaps_over_one_row_per_item( - &mut targets, - &logical_nodes, - &mut pruned_ways_to_compute, - ); + prune_topk_heaps_over_one_row_per_item(&mut targets, &logical_nodes); add_nested_sum_targets(&mut targets, &logical_nodes); close_absorptions(&mut targets); Ok(PerComputationCandidates { roots, logical_nodes, targets, - pruned_ways_to_compute, }) } @@ -1199,15 +1059,14 @@ fn add_whole_expression_targets( } } -/// Prunes the own heap candidates of each PromQL top-k -/// ([`PruneReason::TopKHeapOverOneRowPerItem`]); a fused whole-expression +/// Prunes the own heap candidates of each PromQL top-k: the heap reads +/// one row per item, so exact is no costlier; a fused whole-expression /// heap ([`add_whole_expression_targets`]) has already moved out of them. fn prune_topk_heaps_over_one_row_per_item( targets: &mut [ComputationTarget], logical_nodes: &[Rc], - pruned_ways_to_compute: &mut Vec, ) { - for (index, target) in targets.iter_mut().enumerate() { + for target in targets.iter_mut() { let Some(NonASAPOp::Aggregate { measures, child, .. }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() @@ -1225,18 +1084,10 @@ fn prune_topk_heaps_over_one_row_per_item( if !reads_only_time_series(child) { continue; } - let pruned = prune_own(&mut target.ways_to_compute, |candidate| { + prune_own(&mut target.ways_to_compute, |candidate| { matches!(candidate, HowToComputeAggregateMeasure::Summary(family) if family.capability().answers(StatisticKind::TopK)) }); - pruned_ways_to_compute.extend(pruned.into_iter().map(|way_to_compute| { - PrunedWayToCompute { - target: index, - of_fused_computation: false, - way_to_compute, - reason: PruneReason::TopKHeapOverOneRowPerItem, - } - })); } } @@ -1279,14 +1130,7 @@ fn whole_expression_input( // update serves both heap families; plan selection decides whether // Count-Min is admissible, as for every other Count-Min candidate. let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CountSketchWithHeap, - asap_types::ir::schema::SketchParams::CountSketchWithHeap { - width: 1, - depth: 1, - heap_size: 1, - }, - ), + SketchAlgorithm::CountSketchWithHeap, GroupingStrategy::default(), ); let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; @@ -1342,9 +1186,8 @@ fn with_raw_data_samples_requirement( /// accumulators divided, a fused computation by one state over the absorbed /// target's input, and any other measure by its own single-measure /// aggregate; several measures are joined back on the group keys. A summary -/// is sized here for its measure's accuracy requirement; Pass 1 leaves it -/// unsized; several approximate inputs of one query share its budget -/// equally (`accuracy_budget_parts_per_measure`). A measure filter becomes `SummaryAgg.filter`; +/// is sized here for its measure's accuracy requirement. A measure filter +/// becomes `SummaryAgg.filter`; /// `HAVING` is applied as its target records. A sub-DAG shared between roots /// stays shared. Every node carries the guarantee `asap_accuracy::propagation::composed_guarantee` /// gives it. @@ -1353,38 +1196,15 @@ pub fn compose_logical_candidate( inventory: &PerComputationCandidates, choice: &[usize], ) -> Result, PerComputationCandidateError> { - compose_with_accuracy_budget_parts(inventory, &inventory.roots, choice, None) -} - -/// [`compose_logical_candidate`] with every sketch summary sized for one of -/// `accuracy_budget_parts` equal parts of its measure's budget, whatever the -/// roots compose: the size a summary gets when it is one of -/// `accuracy_budget_parts` approximate inputs of a query -/// ([`accuracy_budget_parts_per_measure`]). `accuracy_budget_parts` 1 sizes -/// for the whole budget. The sub-expression memo composes one aggregate -/// alone this way, once per number of parts its readers may give it. -pub(crate) fn compose_logical_candidate_with_accuracy_budget_parts( - inventory: &PerComputationCandidates, - choice: &[usize], - accuracy_budget_parts: usize, -) -> Result, PerComputationCandidateError> { - compose_with_accuracy_budget_parts( - inventory, - &inventory.roots, - choice, - Some(accuracy_budget_parts), - ) + compose(inventory, &inventory.roots, choice) } /// [`compose_logical_candidate`] of `roots` (some of `inventory.roots`), -/// each summary sized for one of `accuracy_budget_parts` equal parts of its -/// budget, or as -/// [`accuracy_budget_parts_per_measure`] splits it when `None`. -fn compose_with_accuracy_budget_parts( +/// each summary sized for its measure's whole budget. +fn compose( inventory: &PerComputationCandidates, roots: &[QueryRoot], choice: &[usize], - accuracy_budget_parts: Option, ) -> Result, PerComputationCandidateError> { if choice.len() != inventory.targets.len() { return Err(PerComputationCandidateError::InvalidChoice); @@ -1412,29 +1232,7 @@ fn compose_with_accuracy_budget_parts( .filter(|entry| entry.placement == HavingPlacement::BelowAggregate) .filter_map(|entry| entry.filter.as_ref().map(Rc::as_ptr)) .collect(); - // Each summary is first sized for its measure's whole budget. A query - // whose value composes several approximate inputs is then composed - // again with each sized for an equal share ([`accuracy_budget_parts_per_measure`]). - let whole = Sizing::default(); - let composed = compose_roots(roots, &chosen, &pushed_below, &whole)?; - let shares = match accuracy_budget_parts { - None => accuracy_budget_parts_per_measure(&composed, &whole.built.into_inner()), - Some(share) if share > 1 => whole - .built - .into_inner() - .into_iter() - .map(|(_, key)| (key, share)) - .collect(), - Some(_) => HashMap::new(), - }; - if shares.is_empty() { - return Ok(composed); - } - let shared = Sizing { - shares, - ..Sizing::default() - }; - compose_roots(roots, &chosen, &pushed_below, &shared) + compose_roots(roots, &chosen, &pushed_below, &Sizing::default()) } /// One measure of an aggregate: the aggregate and the measure's index. @@ -1443,44 +1241,10 @@ type MeasureKey = (*const OperatorNode, usize); /// How [`compose_logical_candidate`] sizes summaries in one composition. #[derive(Default)] struct Sizing { - /// How many approximate inputs each measure's summary shares its - /// query's budget with; a measure not listed has the whole budget. - shares: HashMap, /// Every sketch summary state built, with the measure it sizes. built: std::cell::RefCell, MeasureKey)>>, } -/// The equal budget split among the approximate inputs of each query -/// (#669b): a query whose value carries a guarantee that is not exact -/// composes every sketch summary state below its root -/// ([`approximate_inputs`]), so with `n ≥ 2` of them each is sized for one -/// of `n` equal shares of the query's target. A state several queries read -/// takes the largest `n`. Only states `built` lists (by the measure they -/// size) are split. Empty when no query composes two approximate inputs. -fn accuracy_budget_parts_per_measure( - roots: &[QueryRoot], - built: &[(Rc, MeasureKey)], -) -> HashMap { - let measure: HashMap<_, _> = built - .iter() - .map(|(state, key)| (Rc::as_ptr(state), *key)) - .collect(); - let mut shares = HashMap::new(); - for root in roots { - let inputs = approximate_inputs(root); - if inputs.len() < 2 { - continue; - } - for state in &inputs { - if let Some(key) = measure.get(&Rc::as_ptr(state)) { - let share = shares.entry(*key).or_insert(1); - *share = inputs.len().max(*share); - } - } - } - shares -} - /// Every root of `roots` with the chosen candidates substituted, each /// summary sized as `sizing` says, and every node with its guarantee. fn compose_roots( @@ -1762,10 +1526,7 @@ fn whole_expression( let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: cache[&Rc::as_ptr(&input)].clone(), input: with_raw_data_samples_requirement(update, raw_data_samples_of), - family: FieldDataType::Sketch( - sized(algorithm, intent, sizing.share(key)), - GroupingStrategy::default(), - ), + family: FieldDataType::Sketch(*algorithm, GroupingStrategy::default()), reduction: partitions, grouping: GroupingStrategy::default(), filter: None, @@ -1929,10 +1690,7 @@ fn substitute_measure( finalized(&kinds[0], intent, read)? } HowToComputeAggregateMeasure::Summary(algorithm) => { - let family = FieldDataType::Sketch( - sized(algorithm, intent, sizing.share(key)), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(*algorithm, GroupingStrategy::default()); let summary_input = state( family, intent, @@ -2212,14 +1970,6 @@ fn key_items(keys: usize) -> Vec { .collect() } -impl Sizing { - /// How many approximate inputs `key`'s summary shares its query's - /// budget with. - fn share(&self, key: MeasureKey) -> usize { - self.shares.get(&key).copied().unwrap_or(1) - } -} - /// What a measure's state reads from each row of `input`, resolvable there. /// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count /// included, reads its input column (the sample value when it names none). @@ -2313,7 +2063,7 @@ fn summary_update( weight: SummaryInputExpr, ) -> Result { let algorithm = match family { - FieldDataType::Sketch(kind, _) => Some(kind.algorithm()), + FieldDataType::Sketch(kind, _) => Some(kind), _ => None, }; Ok(match (intent, algorithm) { @@ -2432,7 +2182,7 @@ mod tests { AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr, }; use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::{SketchParams, SketchStatistic}; + use asap_types::ir::schema::SketchStatistic; /// Approximate count keeps one exact option, its accumulator, beside the /// universal summary; plain exact execution is not offered. #[test] @@ -3143,7 +2893,7 @@ mod tests { // as a column; a query over aggregated rows is planned over labels. let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) .unwrap_or(root); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups(&[ + generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ QueryRoot::Operator(root), ])) .unwrap() @@ -3175,7 +2925,7 @@ mod tests { .filter(|node| { matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) if kind.algorithm().capability().answers(StatisticKind::TopK)) + }) if kind.capability().answers(StatisticKind::TopK)) }) .collect() } @@ -3259,9 +3009,9 @@ mod tests { "topk by (job) (2, sum by (service, job) (rate(m[1m])))", AccuracyTarget::max_error(0.1), ); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), - ) + let inventory = generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ + QueryRoot::Operator(root), + ])) .unwrap(); let topk = topk_target(&inventory); let mut heaps = 0; @@ -3895,89 +3645,6 @@ mod tests { assert_eq!(sign_statistic(&stats), None); } - /// The sketch parameters of every summary state below `root`. - fn sketch_params(root: &Rc) -> Vec { - OperatorNode::reachable(root) - .into_iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .collect() - } - - /// `query` composed with target `t` at the candidate `pick(t, its - /// candidates)` returns the position of. - fn composed_picking( - query: &str, - pick: impl Fn(usize, &[HowToComputeAggregateMeasure]) -> usize, - ) -> Rc { - let inventory = inventory(query); - let choice: Vec = inventory - .targets - .iter() - .enumerate() - .map(|(t, target)| pick(t, &target.ways_to_compute)) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - root.clone() - } - - fn position_of( - candidates: &[HowToComputeAggregateMeasure], - candidate: HowToComputeAggregateMeasure, - ) -> usize { - candidates - .iter() - .position(|c| *c == candidate) - .expect("the candidate is offered") - } - - fn cms(eps: f64, delta: f64) -> SketchParams { - SketchParams::Cms { - width: asap_accuracy::test_support::cms_width(eps), - depth: asap_accuracy::test_support::cms_depth(delta), - } - } - - /// A combination of approximate values splits the query's budget - /// equally among them (#669b): the difference of two Count-Min counts - /// sizes each for ε/2 and δ/2 of the query's ε = 0.01, δ = 0.001. - #[test] - fn approximate_inputs_of_a_combination_share_the_budget_equally() { - let both = composed_picking("count by (job) (a) - count by (job) (b)", |_, c| { - position_of( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ) - }); - assert!(both.guarantee.as_ref().is_some_and(|g| !g.is_exact())); - assert_eq!(sketch_params(&both), vec![cms(0.005, 0.0005); 2]); - } - - /// Only the inputs that are approximate in the combination share the - /// budget: with one count exact, the other gets all of it. - #[test] - fn an_exact_input_takes_no_share_of_the_budget() { - let one = composed_picking("count by (job) (a) - count by (job) (b)", |t, c| { - if t == 0 { - position_of( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ) - } else { - 0 - } - }); - assert_eq!(sketch_params(&one), vec![cms(0.01, 0.001)]); - } - /// The first composed candidate of `query`. fn inventory_root(query: &str) -> Rc { let inventory = inventory(query); @@ -3993,13 +3660,7 @@ mod tests { /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. #[test] fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - asap_types::ir::schema::SketchParams::Cms { width: 8, depth: 2 }, - ), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Cms, GroupingStrategy::default()); let query = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, @@ -4033,16 +3694,7 @@ mod tests { /// once bound to 1. #[test] fn count_min_estimate_over_raw_data_samples_is_gated_on_the_metric() { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Cms, - asap_types::ir::schema::SketchParams::Cms { - width: 272, - depth: 5, - }, - ), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Cms, GroupingStrategy::default()); let query = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, @@ -4076,19 +3728,8 @@ mod tests { /// every group, whose distinct items the source's bound does not bound. #[test] fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { - use asap_types::ir::schema::{HydraKind, HydraParams}; - let hll = |grouping| { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 15, - relative_error: Some(0.05), - }, - ), - grouping, - ) - }; + use asap_types::ir::schema::HydraKind; + let hll = |grouping| FieldDataType::Sketch(SketchAlgorithm::Hll, grouping); let input = |query| crate::test_support::lower_promql(query, AccuracyTarget::exact()); let guarantee = |family: &FieldDataType, query| { source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) @@ -4106,260 +3747,10 @@ mod tests { } let hydra = hll(GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 8, - depth: 2, - shared_rows: 2, - shared_columns: 8, - }, }); assert!(guarantee(&hydra, "a").is_none()); } - /// A deployment whose executor supports only `supported`: each family - /// with its readouts. - fn supporting(supported: &[(SketchAlgorithm, &[StatisticKind])]) -> Vec { - supported - .iter() - .map(|(family, readouts)| SupportedSummary { - family: *family, - readouts: readouts.to_vec(), - }) - .collect() - } - - /// A family the deployment cannot build is pruned from Pass 1 with - /// `UnsupportedByDeployment`; the supported family and the exact option - /// stay. With every summary supported the inventory is unchanged. - #[test] - fn a_family_the_deployment_cannot_build_is_pruned() { - let query = "quantile_over_time(0.99, m[5m])"; - let unrestricted = inventory(query); - let restricted = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::Kll, - &[StatisticKind::Quantile], - )]))); - let target = &restricted.targets[0]; - assert_eq!( - target.ways_to_compute, - [ - HowToComputeAggregateMeasure::Exact, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ] - ); - assert_eq!( - restricted.pruned_of_target(0, false), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch), - PruneReason::UnsupportedByDeployment - )] - ); - let default = inventory(query).without_unsupported_summaries(None); - assert_eq!( - default.targets[0].ways_to_compute, - unrestricted.targets[0].ways_to_compute - ); - } - - /// A family the deployment builds but cannot answer the measure's - /// readout from is pruned: UnivMon, supported for distinct counts and - /// L2 only, is no count candidate; Count-Min, supported for the total, - /// stays. - #[test] - fn a_readout_the_deployment_cannot_answer_prunes_its_family() { - let restricted = inventory("count_over_time(m[1m])").without_unsupported_summaries(Some( - &supporting(&[ - (SketchAlgorithm::Cms, &[StatisticKind::Total]), - ( - SketchAlgorithm::UnivMon, - &[StatisticKind::Distinct, StatisticKind::FrequencyL2], - ), - ]), - )); - let target = &restricted.targets[0]; - assert_eq!( - target.ways_to_compute, - [ - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Cms), - ] - ); - assert_eq!( - restricted.pruned_of_target(0, false), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::UnivMon), - PruneReason::UnsupportedByDeployment - )] - ); - } - - /// A fused whole-expression top-k loses the heap families the - /// deployment cannot answer top-k from, each recorded with its reason, - /// and is not offered once it has none left: the target and the inner - /// sum keep their exact options. - #[test] - fn a_fused_computation_loses_its_unsupported_heaps() { - let query = "topk by (job) (10, sum_over_time(m[1m]))"; - let one_heap = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::CountSketchWithHeap, - &[StatisticKind::TopK], - )]))); - let topk = topk_target(&one_heap); - assert_eq!( - one_heap.targets[topk] - .fused_computation - .as_ref() - .unwrap() - .ways_to_compute, - [HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CountSketchWithHeap - )] - ); - assert_eq!( - one_heap.pruned_of_target(topk, true), - [( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::UnsupportedByDeployment - )] - ); - assert_eq!(one_heap.combinations(), 2); - // Count-Min with a heap answers item frequencies, not top-k. - let none = inventory(query).without_unsupported_summaries(Some(&supporting(&[( - SketchAlgorithm::CmsWithHeap, - &[StatisticKind::ItemFrequency], - )]))); - let topk = topk_target(&none); - let fused = none.targets[topk].fused_computation.as_ref().unwrap(); - assert!(fused.ways_to_compute.is_empty()); - assert_eq!( - none.pruned_of_target(topk, true), - [ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::UnsupportedByDeployment - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - PruneReason::UnsupportedByDeployment - ), - ] - ); - assert_eq!( - none.targets[topk].ways_to_compute, - [HowToComputeAggregateMeasure::Exact] - ); - assert_eq!(none.combinations(), 1); - } - - /// Mergeable intents give exactly their accumulator (`avg`: sum, count); other exact-only intents give `Exact`. - #[test] - fn intent_candidate_matrix() { - use asap_types::ir::operator::agg_intent::{MathFunc, TimeFunc}; - use AggIntent as A; - use ExactKind as E; - let accumulator = |kinds: Vec| HowToComputeAggregateMeasure::ExactAccumulator(kinds); - let exact_count = A::Count { - accuracy: AccuracyTarget::exact(), - }; - let mut matrix: Vec<(A, ComputationChoices)> = [ - (A::Min { col: None }, vec![E::Min]), - (A::Max { col: None }, vec![E::Max]), - (A::IRate, vec![E::IRate]), - (A::Increase, vec![E::Increase]), - (A::Sum { col: None }, vec![E::Sum]), - (A::Rate, vec![E::Rate]), - (A::Avg { col: None }, vec![E::Sum, E::Count]), - ] - .into_iter() - .map(|(intent, kinds)| { - ( - intent, - ComputationChoices { - candidates: vec![accumulator(kinds)], - pruned: vec![], - }, - ) - }) - .collect(); - // An exact count prunes its summaries for requiring an exact answer. - matrix.push(( - exact_count.clone(), - ComputationChoices { - candidates: vec![accumulator(vec![E::Count])], - pruned: summary_candidates(&exact_count) - .iter() - .map(|a| { - ( - HowToComputeAggregateMeasure::Summary(*a), - PruneReason::RequiresExact, - ) - }) - .collect(), - }, - )); - let exact_only = [ - A::PearsonCorr { left: 0, right: 1 }, - A::Extension { - ext_kind: "frequency".into(), - payload: serde_json::json!({ "item": "checkout" }), - }, - A::HistogramQuantile { q: 0.99, le: 0 }, - A::Changes, - A::Delta, - A::IDelta, - A::Deriv, - A::Resets, - A::PredictLinear { seconds: 60.0 }, - A::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.5, - }, - A::HistogramCount, - A::HistogramSum, - A::HistogramAvg, - A::HistogramStdDev, - A::HistogramStdVar, - A::HistogramFraction { - lower: 0.0, - upper: 1.0, - }, - A::Math(MathFunc::Abs), - A::TimeFn(TimeFunc::Hour), - A::Absent { - labels: Default::default(), - }, - A::AbsentOverTime, - A::PresentOverTime, - A::Group, - A::CountValues { label: "v".into() }, - A::LastOverTime, - A::FirstOverTime, - A::MadOverTime, - A::TsOfMinOverTime, - A::TsOfMaxOverTime, - A::TsOfFirstOverTime, - A::TsOfLastOverTime, - ]; - matrix.extend(exact_only.into_iter().map(|intent| { - ( - intent, - ComputationChoices { - candidates: vec![HowToComputeAggregateMeasure::Exact], - pruned: vec![], - }, - ) - })); - for (intent, expected) in &matrix { - assert_eq!( - &computation_candidates_for_intent(intent).unwrap(), - expected, - "{intent:?}" - ); - } - // Correlation never gets a single-input summary. - assert!(summary_candidates(&A::PearsonCorr { left: 0, right: 1 }).is_empty()); - } - /// `avg_over_time` divides with a finite-division guard; a grouped `avg` does not. #[test] fn temporal_average_divides_with_a_finite_division_guard() { @@ -4390,7 +3781,7 @@ mod tests { /// (closed) rows a grouped top-k gets no whole-expression heap. fn inventory_at(query: &str, accuracy: AccuracyTarget) -> PerComputationCandidates { let root = crate::test_support::lower_promql(query, accuracy); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups(&[ + generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ QueryRoot::Operator(root), ])) .unwrap() @@ -4581,342 +3972,4 @@ mod tests { Some(SummaryInputExpr::Tuple(_)) )); } - - /// A KLL quantile over an exact sum accumulator composes per node, the sum adding no error. - #[test] - fn nested_quantile_over_an_exact_sum_composes_per_node() { - let sum = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - let kll = HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll); - let inventory = inventory_at( - "quantile(0.9, sum by (job) (m))", - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ); - let choice: Vec = inventory - .targets - .iter() - .map(|target| { - let candidates = &target.ways_to_compute; - if candidates.contains(&kll) { - position_of(candidates, kll.clone()) - } else { - position_of(candidates, sum.clone()) - } - }) - .collect(); - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - let estimate = OperatorNode::reachable(root) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - .expect("a quantile estimate"); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { child, family, .. }) = summary_input.asap() else { - panic!("a KLL state: {summary_input:#?}") - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("a sketch: {family:?}") - }; - assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); - assert_eq!( - kind.params(), - &SketchParams::Kll { - k: 269, - failure_probability: None - } - ); - let Some(ASAPOp::FinalizeExactAccumulator { child: sum_state }) = child.asap() else { - panic!("the finalized sum: {child:#?}") - }; - let Some(ASAPOp::SummaryAgg { - family: sum_family, - child: leaf, - .. - }) = sum_state.asap() - else { - panic!("the sum's accumulator: {sum_state:#?}") - }; - assert_eq!( - sum_family, - &FieldDataType::ExactAggregate( - ExactKind::Sum, - asap_types::ir::schema::ExactParams::Sum - ) - ); - assert!(!leaf.contains_asap()); - assert!(child - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact)); - let guarantee = root.guarantee.as_ref().expect("a rank bound"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Rank); - assert_eq!( - guarantee.spec.error.evaluate(), - Some(asap_accuracy::test_support::kll_rank_error_99(269)) - ); - } - - /// Pass 1's inventory of `queries`, each lowered with its target, less - /// the summaries no query can use over `data`. - fn usable_inventory( - queries: &[(&str, AccuracyTarget)], - data: &DataWorkload, - ) -> PerComputationCandidates { - let roots: Vec = queries - .iter() - .map(|(query, target)| { - let root = crate::test_support::lower_promql(query, target.clone()); - QueryRoot::Operator(root) - }) - .collect(); - let targets: Vec = - queries.iter().map(|(_, target)| target.clone()).collect(); - generate_per_computation_candidates(&crate::stage0::generate_dag_of_logical_memo_groups( - &roots, - )) - .unwrap() - .without_summaries_no_query_can_use( - &targets, - data, - &asap_accuracy::BuiltinAccuracyModel, - ) - } - - /// A per-value target at ε 0.05, δ 0.01 in `metric`. - fn target_in(metric: AccuracyMetric) -> AccuracyTarget { - AccuracyTarget::max_error(0.05) - .failure_probability(0.01) - .accuracy_metric(metric) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - } - - /// Metric `x`, its samples declared non-negative or not. - fn x_data(non_negative: bool) -> DataWorkload { - let mut data = - DataWorkload::sampled_every(["x".to_string()], asap_types::workload::DurationMs(1_000)); - data.sources.get_mut("x").unwrap().values_non_negative = non_negative; - data - } - - /// The summary families pruned for `reason`, and those still offered. - fn pruned_and_offered( - inventory: &PerComputationCandidates, - reason: PruneReason, - ) -> (Vec, Vec) { - let summary = |way: &HowToComputeAggregateMeasure| match way { - HowToComputeAggregateMeasure::Summary(algorithm) => Some(*algorithm), - _ => None, - }; - let pruned = inventory - .pruned_ways_to_compute - .iter() - .filter(|pruned| pruned.reason == reason) - .filter_map(|pruned| summary(&pruned.way_to_compute)) - .collect(); - let offered = inventory - .targets - .iter() - .flat_map(|target| &target.ways_to_compute) - .filter_map(summary) - .collect(); - (pruned, offered) - } - - /// An average of per-series p99s at a relative target: a relative error - /// passes through `avg` only over samples declared non-negative, and a - /// rank error never meets a relative target. Over an undeclared gauge - /// every quantile sketch is pruned with - /// [`PruneReason::NoErrorBoundForQueryTarget`]; declared non-negative, - /// DDSketch is kept and KLL still pruned. At a rank target, a rank - /// error has no rule through `avg`, so KLL is pruned too. - #[test] - fn a_relative_bound_through_avg_is_pruned_without_a_sign_declaration() { - let query = "avg by (g) (quantile_over_time(0.99, x[100s]))"; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let relative = target_in(AccuracyMetric::Relative); - let undeclared = usable_inventory(&[(query, relative.clone())], &x_data(false)); - let (pruned, offered) = pruned_and_offered(&undeclared, reason); - assert!(pruned.contains(&SketchAlgorithm::DDSketch), "{pruned:?}"); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - assert!(offered.is_empty(), "{offered:?}"); - let declared = usable_inventory(&[(query, relative)], &x_data(true)); - let (pruned, offered) = pruned_and_offered(&declared, reason); - assert_eq!(offered, vec![SketchAlgorithm::DDSketch], "{pruned:?}"); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let rank = usable_inventory(&[(query, target_in(AccuracyMetric::Rank))], &x_data(true)); - let (pruned, offered) = pruned_and_offered(&rank, reason); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - assert!(offered.is_empty(), "{offered:?}"); - } - - /// A summary one of the queries reading its group can use is kept: the - /// per-series p99s are both a query's result, which a KLL rank bound - /// meets, and the input of an average, which no rank bound reaches. - #[test] - fn a_summary_one_of_two_queries_can_use_is_kept() { - let rank = target_in(AccuracyMetric::Rank); - let alone = usable_inventory( - &[("avg(quantile_over_time(0.99, x[5m]))", rank.clone())], - &x_data(false), - ); - let (pruned, _) = pruned_and_offered(&alone, PruneReason::NoErrorBoundForQueryTarget); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let shared = usable_inventory( - &[ - ("quantile_over_time(0.99, x[5m])", rank.clone()), - ("avg(quantile_over_time(0.99, x[5m]))", rank), - ], - &x_data(false), - ); - let (pruned, offered) = - pruned_and_offered(&shared, PruneReason::NoErrorBoundForQueryTarget); - assert!(offered.contains(&SketchAlgorithm::Kll), "{offered:?}"); - assert!(!pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// Exact candidates are never pruned: every target keeps its exact - /// option first, and only summaries are pruned for the target. - #[test] - fn exact_candidates_are_never_pruned_for_the_target() { - let rank = target_in(AccuracyMetric::Rank); - for query in [ - "avg by (g) (quantile_over_time(0.99, x[100s]))", - "avg(quantile_over_time(0.99, x[5m])) > 3", - "sum(count_over_time(x[5m])) / avg(quantile_over_time(0.5, x[5m]))", - ] { - let inventory = usable_inventory(&[(query, rank.clone())], &x_data(false)); - assert!(!inventory.targets.is_empty(), "{query}"); - for target in &inventory.targets { - assert!( - matches!( - target.ways_to_compute.first(), - Some( - HowToComputeAggregateMeasure::Exact - | HowToComputeAggregateMeasure::ExactAccumulator(_) - ) - ), - "{query}: {:?}", - target.ways_to_compute - ); - } - assert!( - inventory - .pruned_ways_to_compute - .iter() - .filter(|pruned| pruned.reason == PruneReason::NoErrorBoundForQueryTarget) - .all(|pruned| matches!( - pruned.way_to_compute, - HowToComputeAggregateMeasure::Summary(_) - )), - "{query}" - ); - } - } - - /// A summary another aggregate over the same source also offers is - /// kept, though no query reading its own group can use it: Pass 2 may - /// build the other group's state from it (a rollup or window rollup), - /// which Pass 1 does not see. - #[test] - fn a_summary_pass2_may_reuse_for_another_aggregate_is_kept() { - let rank = target_in(AccuracyMetric::Rank); - let reason = PruneReason::NoErrorBoundForQueryTarget; - let with_other = usable_inventory( - &[ - ("avg(quantile_over_time(0.99, x[5m]))", rank.clone()), - ("quantile_over_time(0.99, x[10m])", rank), - ], - &x_data(false), - ); - let (pruned, _) = pruned_and_offered(&with_other, reason); - assert!(!pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// A whole-result target with a failure probability over a number of - /// values the data does not bound cannot be met by a per-value - /// guarantee that may fail, whatever its size: KLL is pruned. With the - /// series count declared, it is kept. DDSketch's relative bound never - /// fails, so it is not pruned for the scope (only for the metric under - /// a rank target). - #[test] - fn a_whole_result_target_over_unbound_values_prunes_a_failing_guarantee() { - let whole = target_in(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::WholeResult); - let query = "quantile_over_time(0.99, x[5m])"; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let unbound = usable_inventory(&[(query, whole.clone())], &x_data(false)); - let (pruned, _) = pruned_and_offered(&unbound, reason); - assert!(pruned.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - let bound = usable_inventory(&[(query, whole)], &x_data(false).with_series_count(10_000)); - let (pruned, offered) = pruned_and_offered(&bound, reason); - assert!(offered.contains(&SketchAlgorithm::Kll), "{pruned:?}"); - } - - /// The reuse escape keeps a summary only for another target whose own - /// queries can use the family: two averages of p99s over different - /// windows of one metric at a rank target both offer KLL, and neither - /// can use it (a rank error has no rule through `avg`), so it is - /// pruned for both. Before, each kept it for the other. - #[test] - fn a_summary_no_reusing_target_can_use_is_pruned() { - let rank = target_in(AccuracyMetric::Rank); - let inventory = usable_inventory( - &[ - ("avg(quantile_over_time(0.99, x[5m]))", rank.clone()), - ("avg(quantile_over_time(0.99, x[10m]))", rank), - ], - &x_data(false), - ); - let (pruned, offered) = - pruned_and_offered(&inventory, PruneReason::NoErrorBoundForQueryTarget); - assert_eq!( - pruned - .iter() - .filter(|&&family| family == SketchAlgorithm::Kll) - .count(), - 2, - "{pruned:?}" - ); - assert!(!offered.contains(&SketchAlgorithm::Kll), "{offered:?}"); - } - - /// Classic HLL is pruned when its source's declared distinct bound is - /// outside the contract at every precision (above 2^17 distinct items): - /// its failure probability is 1 at any size. Under the bound it is kept, - /// and with no declared bound too (plan selection may bind one). Two - /// queries over the same metric do not keep it for each other. - #[test] - fn hll_over_a_distinct_bound_outside_its_contract_is_pruned() { - let relative = target_in(AccuracyMetric::Relative); - let queries = [ - ("distinct_over_time(x[5m])", relative.clone()), - ("distinct_over_time(x[10m])", relative), - ]; - let with_bound = |bound: Option| { - let mut data = x_data(true); - data.sources - .get_mut("x") - .unwrap() - .max_distinct_per_evaluation = bound; - data - }; - let reason = PruneReason::NoErrorBoundForQueryTarget; - let (_, offered) = - pruned_and_offered(&usable_inventory(&queries, &with_bound(None)), reason); - assert!(offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - let (_, offered) = pruned_and_offered( - &usable_inventory(&queries, &with_bound(Some(1_000))), - reason, - ); - assert!(offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - let (pruned, offered) = pruned_and_offered( - &usable_inventory(&queries, &with_bound(Some(1_000_000))), - reason, - ); - assert!(pruned.contains(&SketchAlgorithm::Hll), "{pruned:?}"); - assert!(!offered.contains(&SketchAlgorithm::Hll), "{offered:?}"); - } } diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs index 87189af51..757c781fc 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs @@ -3,19 +3,19 @@ use crate::pass1::per_computation_candidates::{ computation_candidates_for_intent, generate_per_computation_candidates, PerComputationCandidateError, }; -use crate::stage0::generate_dag_of_logical_memo_groups; +use crate::stage0::memo_of_queries; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{ DataType, ExactKind, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, - SketchKind, SketchParams, SummaryUpdate, + SummaryUpdate, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, }; -use asap_types::stages::PruneReason; + use asap_types::stages::{HavingEntry, HavingPlacement, HowToComputeAggregateMeasure}; use std::rc::Rc; @@ -117,41 +117,6 @@ fn frequency_statistics_keep_universal_choices() { } } -/// An exact request cannot acquire an approximate sketch merely because one is -/// available; each pruned family records why. -#[test] -fn exact_quantile_stays_exact_and_approximate_keeps_both_families() { - let choices = candidates_for(AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: approximate(), - }); - assert_eq!( - algorithms(&choices), - vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - ); - let exact = computation_candidates_for_intent(&AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: AccuracyTarget::exact(), - }) - .unwrap(); - assert_eq!(exact.candidates, vec![HowToComputeAggregateMeasure::Exact]); - assert_eq!( - exact.pruned, - vec![ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll), - PruneReason::RequiresExact - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch), - PruneReason::RequiresExact - ), - ] - ); -} - /// Scalar roots expose their producer targets; repeated references retain one target identity. #[test] fn scalar_root_producers_are_discovered_once() { @@ -163,15 +128,14 @@ fn scalar_root_producers_are_discovered_once() { QueryRoot::Scalar(ScalarExpr::ScalarSubquery(producer.clone())), QueryRoot::Operator(producer.clone()), ]; - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); + let candidates = generate_per_computation_candidates(&memo_of_queries(&roots)).unwrap(); assert_eq!(candidates.roots.len(), 2); for root in &candidates.roots { root.validate_structure().unwrap(); } assert_eq!(candidates.targets.len(), 1); assert_eq!(**candidates.aggregate(&candidates.targets[0]), *producer); - assert!(producer.timing.is_none()); + assert!(producer.guarantee.is_none()); producer.validate_structure().unwrap(); } @@ -211,8 +175,7 @@ fn exact_promql(query: &str) -> Vec { #[test] fn example_1_exact_q1_keeps_exact_accumulators_only() { let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); + let candidates = generate_per_computation_candidates(&memo_of_queries(&roots)).unwrap(); let found: Vec<_> = candidates .targets .iter() @@ -229,61 +192,15 @@ fn example_1_exact_q1_keeps_exact_accumulators_only() { ])], ] ); - assert!(candidates - .targets - .iter() - .all(|target| candidates.aggregate(target).timing.is_none())); } -/// Physical annotations, ASAP operators and invalid probability requirements -/// fail at the stage boundary. +/// ASAP operators and invalid probability requirements fail at the stage +/// boundary. #[test] -fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { - let mut producer = (*aggregate(AggIntent::Count { - accuracy: approximate(), - })) - .clone(); - producer.timing = Some(asap_types::ir::properties::ExecutionTiming::QueryTime); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::new(producer.clone())) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - let mut stored = (*aggregate(AggIntent::Count { - accuracy: approximate(), - })) - .clone(); - stored.materialization = - asap_types::physical::Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(Rc::new(stored.clone())) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - // An annotated node below an unannotated root is rejected too. - for annotated in [producer, stored] { - let root = having_filter(Rc::new(annotated), always()); - assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root) - ])), - Err(PerComputationCandidateError::AssignedPhysicalAnnotation) - )); - } +fn asap_input_and_invalid_accuracy_are_rejected() { let summary = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(false), - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), + family: FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), grouping: GroupingStrategy::default(), @@ -291,9 +208,7 @@ fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { })) .unwrap(); assert!(matches!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(summary) - ])), + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(summary)])), Err(PerComputationCandidateError::ContainsAsap) )); let epsilon_delta = @@ -333,10 +248,9 @@ fn assigned_physical_annotation_asap_input_and_invalid_accuracy_are_rejected() { vec![], None, ); - let error = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(invalid), - ])) - .unwrap_err(); + let error = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(invalid)])) + .unwrap_err(); assert!( matches!( &error, @@ -394,10 +308,9 @@ fn multi_measure_aggregate_has_one_target_per_measure() { vec![], None, ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(node), - ])) - .unwrap(); + let inventory = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(node)])) + .unwrap(); let candidates = &inventory.targets; assert_eq!( candidates @@ -423,12 +336,10 @@ fn aggregate_without_realization_is_not_a_target() { population: true, }); assert!( - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(node) - ])) - .unwrap() - .targets - .is_empty() + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(node)])) + .unwrap() + .targets + .is_empty() ); } @@ -445,10 +356,9 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { vec![Some(always()), None], None, ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(filtered), - ])) - .unwrap(); + let inventory = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(filtered)])) + .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 2); for (index, target) in targets.iter().enumerate() { @@ -459,36 +369,21 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { assert_eq!(targets[1].measure_filter, None); } -/// Target `target`'s own ways to compute and the ones Pass 1 pruned. +/// Target `target`'s own ways to compute. fn choices_of( inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, target: usize, -) -> crate::pass1::per_computation_candidates::ComputationChoices { - crate::pass1::per_computation_candidates::ComputationChoices { - candidates: inventory.targets[target].ways_to_compute.clone(), - pruned: inventory.pruned_of_target(target, false), - } +) -> Vec { + inventory.targets[target].ways_to_compute.clone() } -fn assert_sql_count_choices( - choices: &crate::pass1::per_computation_candidates::ComputationChoices, -) { +fn assert_sql_count_choices(choices: &[HowToComputeAggregateMeasure]) { assert_eq!( - choices.candidates, + choices, vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ ExactKind::Count ])] ); - assert_eq!( - choices.pruned, - [SketchAlgorithm::Cms, SketchAlgorithm::UnivMon] - .into_iter() - .map(|algorithm| ( - HowToComputeAggregateMeasure::Summary(algorithm), - PruneReason::NoRelationalCountSketchRealization, - )) - .collect::>() - ); } fn quantile() -> AggIntent { @@ -545,10 +440,9 @@ fn equals(column: usize, value: ScalarValue) -> Predicate { fn key_only_having_is_pushed_below_and_keeps_candidates() { let node = grouped_count(None); let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root.clone()), - ])) - .unwrap(); + let inventory = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root.clone())])) + .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 1); assert_eq!(**inventory.aggregate(&targets[0]), *node); @@ -568,7 +462,7 @@ fn key_only_having_is_pushed_below_and_keeps_candidates() { #[test] fn value_having_stays_above_output_and_keeps_candidates() { let value = || equals(1, ScalarValue::Int64(10)); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ + let inventory = generate_per_computation_candidates(&memo_of_queries(&[ QueryRoot::Operator(having_filter(grouped_count(None), value())), QueryRoot::Operator(grouped_count(Some(value()))), ])) @@ -586,37 +480,6 @@ fn value_having_stays_above_output_and_keeps_candidates() { } } -/// `avg`'s one exact option is sum and count states when its input is never -/// null, and plain `Exact` when that rewrite is pruned. -#[test] -fn avg_rewrites_to_sum_and_count_states() { - let avg = || vec![AggIntent::Avg { col: None }]; - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(aggregate_over(scan(false), avg(), vec![], None)), - QueryRoot::Operator(aggregate_over(scan(true), avg(), vec![], None)), - ])) - .unwrap(); - let targets = &inventory.targets; - assert_eq!( - targets[0].ways_to_compute, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count - ])] - ); - assert_eq!( - targets[1].ways_to_compute, - vec![HowToComputeAggregateMeasure::Exact] - ); - assert_eq!( - inventory.pruned_of_target(1, false), - vec![( - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]), - PruneReason::NullableAvgInput - )] - ); -} - /// A key-only `HAVING` above a shared aggregate stays above the output when /// another consumer reads the aggregate without it. #[test] @@ -624,7 +487,7 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { let node = grouped_count(None); let key = equals(0, ScalarValue::Utf8("a".into())); let filtered = having_filter(node.clone(), key.clone()); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ + let targets = generate_per_computation_candidates(&memo_of_queries(&[ QueryRoot::Operator(filtered.clone()), QueryRoot::Operator(node.clone()), ])) @@ -643,7 +506,7 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { // Two consumers applying the same key-only predicate push it below. let first = having_filter(node.clone(), key.clone()); let second = having_filter(node, key); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ + let targets = generate_per_computation_candidates(&memo_of_queries(&[ QueryRoot::Operator(first), QueryRoot::Operator(second), ])) @@ -672,7 +535,7 @@ fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { negated: false, }), ); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ + let inventory = generate_per_computation_candidates(&memo_of_queries(&[ QueryRoot::Operator(filtered.clone()), QueryRoot::Operator(exists), ])) @@ -700,11 +563,10 @@ fn only_the_adjacent_of_stacked_filters_is_having() { let inner_key = equals(0, ScalarValue::Utf8("a".into())); let inner = having_filter(node.clone(), inner_key.clone()); let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(outer), - ])) - .unwrap() - .targets; + let targets = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(outer)])) + .unwrap() + .targets; assert_eq!(targets.len(), 1); assert_eq!( targets[0].having_filters, @@ -726,7 +588,7 @@ fn global_aggregate_having_stays_above() { }] }; let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ + let targets = generate_per_computation_candidates(&memo_of_queries(&[ QueryRoot::Operator(having_filter( aggregate_over(scan(false), count(), vec![], None), never.clone(), @@ -746,45 +608,6 @@ fn global_aggregate_having_stays_above() { } } -/// #509 Example 1 Q2 under the default exact target: the top-k's one option is plain -/// `Exact`, its heap summaries pruned `RequiresExact`; `sum_over_time` gets -/// its exact Sum accumulator. -#[test] -fn example_1_exact_q2_prunes_topk_heaps() { - let roots = exact_promql("topk by (job) (10, sum_over_time(http_requests_total[1m]))"); - let candidates = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); - let found: Vec<_> = candidates - .targets - .iter() - .enumerate() - .map(|(index, _)| choices_of(&candidates, index)) - .collect(); - let topk = found - .iter() - .find(|choices| choices.candidates == [HowToComputeAggregateMeasure::Exact]) - .expect("top-k target"); - assert_eq!( - topk.pruned, - vec![ - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - PruneReason::RequiresExact - ), - ( - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - PruneReason::RequiresExact - ), - ] - ); - assert!(found.iter().any(|choices| choices.candidates - == [HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])] - && choices.pruned.is_empty())); - assert_eq!(found.len(), 2, "{found:#?}"); -} - /// The `HAVING` entries of the target whose measure satisfies `is`. fn having_of( inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, @@ -822,10 +645,9 @@ fn scalar_reference_filter_is_not_having() { }; let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(scan(false), above_scalar(1, agg.clone())); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) - .unwrap(); + let inventory = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root)])) + .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 1); assert_eq!(**inventory.aggregate(&targets[0]), *agg); @@ -837,11 +659,10 @@ fn scalar_reference_filter_is_not_having() { let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root.clone()), - ])) - .unwrap() - .targets; + let targets = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root.clone())])) + .unwrap() + .targets; assert_eq!(targets.len(), 1); assert_eq!( targets[0].having_filters, @@ -857,7 +678,7 @@ fn scalar_reference_filter_is_not_having() { #[test] fn promql_scalar_operand_is_not_having() { let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) + let targets = generate_per_computation_candidates(&memo_of_queries(&roots)) .unwrap() .targets; assert!(!targets.is_empty()); @@ -870,8 +691,7 @@ fn promql_scalar_operand_is_not_having() { } let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)).unwrap(); + let inventory = generate_per_computation_candidates(&memo_of_queries(&roots)).unwrap(); assert!(having_of(&inventory, |intent| matches!( intent, AggIntent::Count { .. } @@ -897,7 +717,7 @@ fn having_shared_with_a_project_consumer_stays_above() { child: node, })) .unwrap(); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ + let targets = generate_per_computation_candidates(&memo_of_queries(&[ QueryRoot::Operator(filtered), QueryRoot::Operator(project), ])) @@ -934,11 +754,10 @@ fn without_and_per_entity_having_stays_above() { })) .unwrap(); let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); - let targets = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ])) - .unwrap() - .targets; + let targets = + generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root)])) + .unwrap() + .targets; assert_eq!(targets.len(), 1, "{reduction:?}"); let placements: Vec<_> = targets[0] .having_filters @@ -963,9 +782,9 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { cols: vec![0], accuracy: approximate(), }); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(producer.clone()), - ])) + let inventory = generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator( + producer.clone(), + )])) .unwrap(); let exact = compose_logical_candidate(&inventory, &[0]).unwrap(); let QueryRoot::Operator(kept) = &exact[0] else { @@ -1000,7 +819,7 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { /// The one root of the workload candidate composed from `roots` and `choice`. fn composed(roots: Vec>, choice: &[usize]) -> Vec> { use crate::pass1::per_computation_candidates::compose_logical_candidate; - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups( + let inventory = generate_per_computation_candidates(&memo_of_queries( &roots .into_iter() .map(QueryRoot::Operator) @@ -1049,27 +868,6 @@ fn summary_agg(node: &OperatorNode) -> &ASAPOp { } } -/// A summary is sized when the workload candidate is composed, for its -/// measure's accuracy requirement. -#[test] -fn composed_summary_is_sized_for_its_measure() { - use asap_accuracy::test_support::default_size_params; - let intent = quantile(); - let [root] = <[_; 1]>::try_from(composed(vec![aggregate(intent.clone())], &[1])).unwrap(); - let ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } = summary_agg(&root) - else { - panic!("sketch state expected") - }; - assert_eq!(*kind.algorithm(), SketchAlgorithm::Kll); - assert_eq!( - kind.params(), - &default_size_params(SketchAlgorithm::Kll, &intent, 0.05, 0.01) - ); -} - /// A measure filter becomes the state's filter; a key-only `HAVING` applied /// by every consumer moves below the state; a value `HAVING` stays a `Filter` /// above the estimate. @@ -1268,9 +1066,9 @@ fn own_having_subquery_is_substituted() { use crate::pass1::per_computation_candidates::compose_logical_candidate; let inner = aggregate(quantile()); let outer = grouped_quantile(Some(above_scalar(1, inner.clone()))); - let inventory = generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(outer.clone()), - ])) + let inventory = generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator( + outer.clone(), + )])) .unwrap(); assert_eq!(inventory.targets.len(), 2); let kll = 1; @@ -1379,7 +1177,7 @@ fn memo_cached_rules_preserve_occurrence_specific_having() { QueryRoot::Operator(shared), QueryRoot::Operator(having_filter(other, key)), ]; - let memo = generate_dag_of_logical_memo_groups(&roots); + let memo = memo_of_queries(&roots); let cached = generate_per_computation_candidates(&memo).unwrap(); assert_eq!(cached.targets.len(), 1); // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/logical-asap-optimization/src/pass1/realization.rs index d5dc1f09c..285a814bb 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/logical-asap-optimization/src/pass1/realization.rs @@ -146,7 +146,7 @@ pub(crate) fn realize_keyed_additive_summary_input( let FieldDataType::Sketch(kind, _) = family else { return None; }; - let heap_algorithm = kind.algorithm(); + let heap_algorithm = kind; if !matches!( heap_algorithm, SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap @@ -353,7 +353,7 @@ mod tests { use crate::test_support::metric_scan; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::schema::{GroupingStrategy, SketchKind, SketchParams}; + use asap_types::ir::schema::GroupingStrategy; use asap_types::ir::{ExprSemantics, Predicate}; /// A heap sketch consumes every raw row, so the keyed input rule refuses @@ -377,17 +377,8 @@ mod tests { k: 5, accuracy: AccuracyTarget::max_error(0.01), }; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 4, - heap_size: 5, - }, - ), - GroupingStrategy::default(), - ); + let family = + FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, GroupingStrategy::default()); let output = Reduction::Reduce(asap_types::ir::operator::operator_properties::GroupKeys::none()); assert!(realize_keyed_additive_summary_input( diff --git a/crates/logical-asap-optimization/src/pass2.rs b/crates/logical-asap-optimization/src/pass2.rs index 2607e1edf..dbdc6f273 100644 --- a/crates/logical-asap-optimization/src/pass2.rs +++ b/crates/logical-asap-optimization/src/pass2.rs @@ -1,11 +1,6 @@ -//! Stage 1, Pass 2: ASAP-aware sharing over the Pass 1 candidates, through the -//! logical memo and its rules. +//! Stage 1, Pass 2: the rules over groups and the logical edges. -/// Stage 1, Pass 2: ASAP-aware common-subexpression elimination (the -/// identical-expression rule). -pub(crate) mod asap_aware_cse; -pub(crate) mod finer_aggregate; /// Stage 1, Pass 2's entry, producing [`asap_types::stages::LogicalASAPMemoCandidates`]. pub(crate) mod logical_memo_candidates; -/// The logical memo: candidates, rules and enumeration over Stage 0's groups. +/// The rules over groups. pub(crate) mod memo; diff --git a/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs b/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs deleted file mode 100644 index 991e96f40..000000000 --- a/crates/logical-asap-optimization/src/pass2/asap_aware_cse.rs +++ /dev/null @@ -1,3557 +0,0 @@ -//! Stage 1 (logical ASAP-aware optimization), Pass 2: ASAP-aware -//! common-subexpression elimination, as defined in -//! `docs/design_docs/proposals/planner-layering.md` (Algorithm 2). -//! -//! The memo rules (`pass2::memo::rules`) are built on these helpers: the -//! window-composition rule for tumbling windows, and two reuse rules (#680), -//! rollup and exact top-k reuse. Identical expressions are shared by memo -//! group identity, and summary builds of one family at different sizes by -//! the memo's shared-producer rule (`memo::rules::shared_producer`); both -//! rules' whole-workload forms are gone. -//! -//! Scans of one metric that project different labels read the same rows. -//! [`with_union_scan_labels`] widens them to the union of their labels, so -//! interning finds them identical (`memo::rules::scan_labels`). -//! -//! # Window-composition rule -//! -//! A query repeating every `E` over a window `W` can build its summary as -//! back-to-back tumbling windows of width `w = gcd(W, E)` (Algorithm 2's -//! `Tumbling(L)` for one query), merged before each read. A *tumbling window -//! candidate* ([`tumbling_window_candidates`]) is a `SummaryAgg` over a range -//! `TimeRange(W)` whose -//! family merges in the family-capability table ([`FamilyCapability`]): -//! exact accumulators included, and also families whose merged readouts -//! are only approximate (heap top-k), whose readers then carry no -//! guarantee and are left to plan selection's evidence. A family without a -//! merge (IRate) has no tumbling-window candidate. -//! [`window_composition_pass2_candidates`] returns one Pass 2 candidate per -//! non-empty subset of the tumbling-window candidates: the build of each in the subset -//! becomes a `SummaryMerge` over one [`ASAPOp::SummaryTumblingWindowChain`] of `W / w` tumbling windows, -//! a compact node whatever the tumbling-window count, with the build's family and -//! size (a mergeable family's merged state has one build's guarantee), and -//! every guarantee above is composed again. Nothing is materialized here: -//! the tumbling windows are rebuilt at each evaluation until Stage 2 keeps them. -//! -//! Nested windows share one chain: when a subset holds builds of one family -//! (at any size) over the same input below their ranges, read the same way, -//! with the same output column names, whose tumbling windows differ in width or count, -//! the subset also comes as a Pass 2 candidate in which each such group -//! reads one chain ([`SharedTumblingWindowChain`]) over the longest window, sized for -//! every member ([`size_for_all_readers`]). Two grids are offered: the -//! finest member's own tumbling windows, for the members whose widths it divides -//! ([`nested_window_groups`]), and, when that differs, the gcd of all the -//! members' widths, even finer than every member's own tumbling windows -//! ([`gcd_window_groups`]); Stage 3's pricing weighs the extra states and -//! merges of finer tumbling windows. Queries repeating at different intervals read the -//! same tumbling windows, each at its own cadence. Every window ends at the evaluation -//! time, so each shorter one is the chain's newest tumbling windows: its -//! `SummaryMerge` merges only those (`newest_tumbling_windows`), and has the guarantee -//! of one build over its own window. A build grouped by fewer keys also -//! reads the chain of a finer grouping over a window at least its own -//! ([`rollup_window_groups`]): it merges its newest tumbling windows, then rolls the -//! groups up into its own, as the rollup rule does. Stage 2 never keeps a -//! chain read at different intervals across evaluations from query time -//! (each reader would need one new tumbling window per evaluation). Long-window segments, sliding -//! windows and Exponential Histograms are not implemented. -//! -//! # Rollup rule -//! -//! A coarse grouped computation can be answered from a finer one the -//! candidate already computes over the same rows: `sum by (job)` from -//! `sum by (job, instance)`. The rule never introduces a finer grouping -//! nobody asked for. A *rollup candidate* ([`rollup_candidates`]) is a -//! coarse build and a finer build of the same family over the same rows (a -//! PromQL scan may project other labels), whose group keys include the -//! coarse ones: -//! - An exact accumulator whose family merges across disjoint groups with -//! an exact merged value ([`rollup_kind`]: sums, counts, minima and -//! maxima; counter states merge only in time). The coarse build becomes -//! an exact accumulator over its coarsest finer build's finalized values, -//! grouped by the coarse keys (a count sums the finer counts), so the -//! value stays exact. -//! - A per-group sketch (KMV, HLL, KLL, DDSketch, Count-Min, Count Sketch, -//! UnivMon, at any size) read only by estimates whose merged readouts -//! keep a guarantee (not a heap's top-k or UnivMon's entropy). The coarse build becomes a rollup (a regrouping -//! `SummaryMerge`) of its finest finer build's states by the coarse -//! keys. The groups of one build partition its rows, so the rolled-up -//! state keeps the family's merged guarantee: the direct coarse build's -//! for KMV, HLL, DDSketch and the linear sketches, the same rank bound -//! for KLL. -//! -//! A sketch build over fewer rows than a finer one, whose extra conjuncts -//! read only the finer keys (`… WHERE city IN ('NYC', 'SF')` from a build -//! by `city`), is answered by first keeping the finer states that pass -//! those conjuncts, a compensating `Filter` over the states -//! ([`filtered_finer`], [`match_finer`](crate::pass2::finer_aggregate)), then -//! merging them. A SQL aggregate's finer *result* is read the same way by -//! the memo rule `memo::rules::finer_aggregate`. -//! -//! [`rollup_pass2_candidates`] returns one Pass 2 candidate per non-empty -//! subset of the rollup *choices* ([`rollup_choices`]): each exact rollup -//! alone, and every sketch rollup of one finest build together, since they -//! share its size. A chosen sketch choice sizes the finest build for every -//! reader, its own and each coarse one's (the strictest consumer, as the -//! shared-producer rule sizes a shared build), and composes every -//! guarantee above it again. So 15 groupings of one window are one choice, -//! not 2^14 subsets. -//! -//! # Exact top-k reuse rule -//! -//! Exact top-k computations of one input and partition with different `k` -//! read one ranking: the rows of the largest top-k, `Limit(k_max)` over -//! `Sort`, which hold every smaller top-k ([`topk_reuse_candidates`], -//! `memo::rules::topk`). Only exact computations over exact input are -//! reused; a sketch's heap is not, as its membership has no guarantee yet. - -use std::collections::HashMap; -use std::collections::HashSet; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::resolve_column_ref; -use asap_types::ir::schema::{ - ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergeInputRule, MergedError, - Schema, SketchKind, SummaryInputExpr, SummaryUpdate, WeightDomain, PROMQL_SERIES_IDENTITY, -}; -use asap_types::ir::ScalarExpr; -use asap_types::ir::SortKey; -use asap_types::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, QueryRoot, - TimeRangeKind, -}; - -use asap_accuracy::estimators::size_for_all_readers; -use asap_accuracy::propagation::recompose_guarantees; - -use asap_types::ir::SchemaDerivationError; - -/// Why a sharing rewrite builds no candidate. The rules report it as text -/// ([`Pass2RuleError`](crate::pass2::memo::state_groups::Pass2RuleError)) or -/// drop the candidate. -#[derive(Debug, thiserror::Error)] -pub enum CseError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] - Unsupported(&'static str), - /// A rule found more than `max` candidates of one kind, whose 2^n subsets - /// it would enumerate: the workload is too large to plan this way, not - /// bad input. - #[error("found {found} {counted}, more than {max}")] - TooManyCandidates { - /// What was counted, e.g. "rollup choices". - counted: &'static str, - /// How many were found, more than `max`. - found: usize, - /// The most the rule enumerates. - max: usize, - }, -} - -/// `roots` with the time-series scans of one metric made identical when -/// they project different labels, so that interning finds them identical: the PromQL frontend projects only the labels each query uses -/// (`ts, value, job, …` for `quantile by (job) (0.5, data)`), and -/// structural CSE shares only nodes with equal schemas. Every scan of a -/// metric without predicates projects the union of the labels its scans -/// project, in the order the walk meets them (before the series identity, -/// with the unique keys moved along). -/// The nodes that pass the labels on (`passes_labels_on`) keep the wider -/// rows above a widened scan; any other consumer reads them through a -/// `Project` that keeps the columns it read, by name. `None` when every metric's scans already -/// agree, when two of them give one label different types or one is closed -/// and the other not, when a node would change arity, or when a consumer -/// would have to narrow a summary state or a grouped or filtered tumbling-window chain -/// would read a `Project`. -pub fn with_union_scan_labels(roots: &[QueryRoot]) -> Result>, CseError> { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - // Each metric's union schema, `None` once two of its scans conflict. - let mut unions: Vec<(&Source, Option)> = Vec::new(); - for node in &nodes { - let Some((source, schema)) = unfiltered_series_scan(node) else { - continue; - }; - let Some(i) = unions.iter().position(|(s, _)| *s == source) else { - unions.push((source, Some(schema.clone()))); - continue; - }; - let Some(union) = unions[i].1.as_mut() else { - continue; - }; - // A scan known to hold every label and one that is not cannot share. - if union.closed != schema.closed { - unions[i].1 = None; - continue; - } - for field in &schema.fields { - match union.fields.iter().find(|f| f.name == field.name) { - Some(known) if known == field => {} - Some(_) => { - unions[i].1 = None; - break; - } - None => { - let at = union - .fields - .iter() - .position(|f| f.name == PROMQL_SERIES_IDENTITY) - .unwrap_or(union.fields.len()); - union.fields.insert(at, field.clone()); - // Unique keys (the series identity and ts) name columns - // by position. - for column in union.unique_keys.iter_mut().flatten() { - if *column >= at { - *column += 1; - } - } - } - } - } - } - let union_of = |node: &OperatorNode| { - let (source, schema) = unfiltered_series_scan(node)?; - let (_, union) = unions.iter().find(|(s, _)| *s == source)?; - union.as_ref().filter(|union| *union != schema) - }; - if !nodes.iter().any(|node| union_of(node).is_some()) { - return Ok(None); - } - let mut widened: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for node in &nodes { - let out = match (union_of(node), node.non_asap()) { - (Some(union), Some(NonASAPOp::Scan { source, .. })) => { - let mut schema = union.clone(); - schema.time_index = schema.fields.iter().position(|f| f.name == "ts"); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - predicates: Vec::new(), - schema, - }))? - } - _ if node - .children() - .iter() - .all(|c| Rc::ptr_eq(c, &widened[&Rc::as_ptr(c)])) => - { - Rc::clone(node) - } - // A node whose renamed outputs would change arity is not widened. - _ if passes_labels_on(node) => { - match node.with_new_children(|c| Rc::clone(&widened[&Rc::as_ptr(c)])) { - Ok(rebuilt) => Rc::new(rebuilt), - Err(_) => return Ok(None), - } - } - // A grouped or filtered tumbling-window chain over a `Project` would hide - // its window from plan selection. - _ if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) => { - return Ok(None); - } - _ => { - let mut children = HashMap::new(); - for child in node.children() { - let wide = &widened[&Rc::as_ptr(child)]; - let Some(narrow) = narrowed(wide, &child.schema)? else { - return Ok(None); - }; - children.insert(Rc::as_ptr(child), narrow); - } - Rc::new(node.with_new_children(|c| Rc::clone(&children[&Rc::as_ptr(c)]))?) - } - }; - widened.insert(Rc::as_ptr(node), out); - } - let mut out = Vec::with_capacity(roots.len()); - for query_root in roots { - let mut mapped = HashMap::new(); - for node in query_root.operator_refs() { - let Some(narrow) = narrowed(&widened[&Rc::as_ptr(node)], &node.schema)? else { - return Ok(None); - }; - mapped.insert(Rc::as_ptr(node), narrow); - } - out.push(match query_root { - QueryRoot::Operator(node) => QueryRoot::Operator(Rc::clone(&mapped[&Rc::as_ptr(node)])), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| Rc::clone(&mapped[&Rc::as_ptr(node)])), - ), - }); - } - Ok(Some(recompose_guarantees(&out)?)) -} - -/// The source and schema of a time-series `Scan` without predicates. -fn unfiltered_series_scan(node: &OperatorNode) -> Option<(&Source, &Schema)> { - match node.non_asap() { - Some(NonASAPOp::Scan { - source: source @ Source::TimeSeries { .. }, - predicates, - schema, - }) if predicates.is_empty() => Some((source, schema)), - _ => None, - } -} - -/// Whether `node`, over a child with more labels, keeps them in its own -/// output: a time range or shift of the rows, and a per-series build or -/// tumbling-window chain without a filter (keyed by the series identity, which -/// determines every label) with its merges, estimates and finalizations. -/// The executor reads a per-series build's raw rows from its `TimeRange` -/// directly, and plan selection reads a tumbling-window chain's window from the rows -/// below it, so nothing may sit between them. -fn passes_labels_on(node: &OperatorNode) -> bool { - matches!( - node.non_asap(), - Some(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) - ) || matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryTumblingWindowChain { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryMerge { .. } - | ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - ) - ) -} - -/// `wide`, which holds every column of `schema` and perhaps more, with -/// only `schema`'s columns, by name: `wide` itself when it has exactly -/// them; `None` when `wide` is a summary state, which has no columns to -/// project. -fn narrowed( - wide: &Rc, - schema: &Schema, -) -> Result>, CseError> { - use asap_types::ir::ProjectItem; - if wide.schema.fields == schema.fields { - return Ok(Some(Rc::clone(wide))); - } - if wide.asap().is_some_and(|op| op.produced_state().is_some()) { - return Ok(None); - } - let cols = schema - .fields - .iter() - .map(|field| { - let column = wide - .schema - .fields - .iter() - .position(|f| f.name == field.name && f.table == field.table) - .expect("a widened node keeps every column it had"); - ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - } - }) - .collect(); - Ok(Some(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: Rc::clone(wide), - }, - ))?)) -} - -fn collect_children_first( - node: &Rc, - seen: &mut std::collections::HashSet<*const OperatorNode>, - out: &mut Vec>, -) { - if !seen.insert(Rc::as_ptr(node)) { - return; - } - for child in node.children() { - collect_children_first(child, seen, out); - } - out.push(Rc::clone(node)); -} - -/// Whether builds of families `a` and `b` can be one build: one sketch -/// algorithm and grouping at any sizes, or one exact accumulator. -fn same_build_family(a: &FieldDataType, b: &FieldDataType) -> bool { - match (a, b) { - (FieldDataType::Sketch(a, grouping), FieldDataType::Sketch(b, grouping_b)) => { - a.algorithm() == b.algorithm() && grouping == grouping_b - } - _ => a == b, - } -} - -/// One build serving every member: the first member with the members' -/// shared size ([`size_for_all_readers`]). -fn sized_for_all(members: &[&Rc]) -> Result>, CseError> { - // An exact accumulator has one size: the first member serves them all. - if matches!( - members[0].asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), - .. - }) - ) { - return Ok(Some(Rc::clone(members[0]))); - } - let kinds: Vec<&SketchKind> = members - .iter() - .filter_map(|member| match member.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind), - _ => None, - }) - .collect(); - let params: Vec<_> = kinds.iter().map(|kind| kind.params()).collect(); - let Some(params) = size_for_all_readers(¶ms) else { - return Ok(None); - }; - let first = members[0]; - let Some(ASAPOp::SummaryAgg { - child, - family: FieldDataType::Sketch(kind, grouping_strategy), - input, - reduction, - grouping, - filter, - }) = first.asap() - else { - return Ok(None); - }; - let operator = Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::Sketch( - SketchKind::new(*kind.algorithm(), params), - grouping_strategy.clone(), - ), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }); - let mut build = OperatorNode::new(operator)?; - // Keep the member's explicit output names. - for (field, retained) in build.schema.fields.iter_mut().zip(&first.schema.fields) { - field.name = retained.name.clone(); - field.table = retained.table.clone(); - } - Ok(Some(Rc::new(build))) -} - -/// One summary build the window-composition rule can build as tumbling -/// tumbling windows: a `SummaryAgg` over a range `TimeRange(W)` whose family merges -/// (the family-capability table: exact accumulators included, and families -/// whose merged readouts are only approximate), read only by queries that -/// repeat at a fixed interval. Its tumbling windows are `tumbling_window_count = W / w` back-to-back -/// windows of `w = gcd(W, E)` over the queries' intervals `E` (and, for a -/// query with a range evaluation, its step), at least two. -#[derive(Debug, Clone)] -pub struct TumblingWindowCandidate { - /// The `SummaryAgg` over `TimeRange(W)` built as tumbling windows. - pub build: Rc, - /// The width `w` of one tumbling window. - pub tumbling_window_width_ms: u64, - /// The number of tumbling windows, `W / w`, at least two. - pub tumbling_window_count: u32, -} - -impl TumblingWindowCandidate { - /// The window `W` its tumbling windows cover, `tumbling_window_width_ms · tumbling_window_count`. - pub fn window_ms(&self) -> u64 { - self.tumbling_window_width_ms * u64::from(self.tumbling_window_count) - } -} - -/// A Pass 2 candidate of the window-composition rule: the candidate's roots -/// with the build of each of `tumbling_window_candidates` built as its tumbling windows, -/// merged before every read, and every guarantee composed again; the error -/// when that fails, which leaves the other Pass 2 candidates unaffected. -#[derive(Debug)] -pub struct WindowCompositionPass2Candidate { - /// The builds built as tumbling windows, in [`tumbling_window_candidates`] order. - /// Only tests read it. - #[cfg_attr(not(any(test, feature = "test-support")), allow(dead_code))] - pub tumbling_window_candidates: Vec, - /// The chains each shared by several of `tumbling_window_candidates` - /// ([`nested_window_groups`] or [`gcd_window_groups`]); empty when each - /// build has its own chain. - pub shared_chains: Vec, - /// The candidate's roots with those builds in tumbling windows. - pub roots: Result, CseError>, -} - -/// One tumbling-window chain several tumbling-window candidates read (window-composition rule): -/// tumbling windows of `tumbling_window_width_ms`, a width that divides every member's own tumbling window -/// width, and `tumbling_window_count` of them, covering the longest member's window. -/// Each member merges the newest `W / tumbling_window_width_ms` tumbling windows of its window `W`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SharedTumblingWindowChain { - /// Positions in the Pass 2 candidate's `tumbling_window_candidates`, two or more. - pub members: Vec, - /// The width of one tumbling window of the shared chain. - pub tumbling_window_width_ms: u64, - /// The number of tumbling windows of the shared chain. - pub tumbling_window_count: u32, -} - -/// The most tumbling-window candidates one candidate may have: every non-empty subset -/// is a Pass 2 candidate, and with shared grids up to three, about three -/// million at 20. -pub(crate) const MAX_TUMBLING_WINDOW_CANDIDATES: usize = 20; - -/// The tumbling-window candidates of the candidate with query roots `roots`, children -/// first. `evaluation_grid_ms[q]` is the grid query `q`'s evaluations fall -/// on: how often it repeats, or with a range evaluation the greatest common -/// divisor of that and its step, and of its phase if declared -/// (`QueryWorkloadEntry::evaluation_grid_ms`); `None` when it does not -/// repeat at a fixed interval. Each tumbling-window width divides every reader's grid, -/// so with tumbling windows anchored at the epoch every evaluation, every range step -/// included, ends on a tumbling-window boundary. -pub fn tumbling_window_candidates( - roots: &[QueryRoot], - evaluation_grid_ms: &[Option], -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let reach: Vec> = roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .map(|node| Rc::as_ptr(&node)) - .collect() - }) - .collect(); - nodes - .into_iter() - .filter_map(|build| { - let window_ms = splittable_window_ms(&build)?; - let readers: Vec = (0..roots.len()) - .filter(|&q| reach[q].contains(&Rc::as_ptr(&build))) - .collect(); - let mut width = window_ms; - for &q in &readers { - width = asap_types::workload::greatest_common_divisor( - width, - evaluation_grid_ms.get(q).copied().flatten()?, - ); - } - let tumbling_window_count = u32::try_from(window_ms / width).ok()?; - (!readers.is_empty() && tumbling_window_count >= 1).then_some(TumblingWindowCandidate { - build, - tumbling_window_width_ms: width, - tumbling_window_count, - }) - }) - .collect() -} - -/// The window `W` of a build the rule can split: a `SummaryAgg` whose -/// family merges, over a range `TimeRange(W)` over an instant vector or a -/// SQL table's rows, read at the query's time or a fixed offset from it: no `@` anchor on any of -/// the time shifts stacked below the range. -fn splittable_window_ms(build: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryAgg { child, family, .. }) = build.asap() else { - return None; - }; - FamilyCapability::of(family).merge_inputs?; - let Some(NonASAPOp::TimeRange { - range, - kind: TimeRangeKind::Range, - child: input, - }) = child.non_asap() - else { - return None; - }; - // An instant vector (PromQL `m[w]`) or a SQL table's rows (a trailing - // window); not a range vector, whose rows are already windows. - if !matches!( - input.result_kind, - OperatorResultKind::InstantVector | OperatorResultKind::Relation - ) { - return None; - } - // Refuse an `@` anchor on any of the shifts stacked below the range. - let mut below = input; - while let Some(NonASAPOp::TimeShift { shift, child }) = below.non_asap() { - if shift.at.is_some() { - return None; - } - below = child; - } - u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0) -} - -/// Every Pass 2 candidate of the window-composition rule for the candidate -/// with query roots `roots` and tumbling-window candidates `candidates` -/// ([`tumbling_window_candidates`]): one per non-empty subset of them, in -/// binary-counter order (the first candidate the lowest bit). Each build in -/// the subset becomes a `SummaryMerge` over one [`ASAPOp::SummaryTumblingWindowChain`] of -/// its tumbling windows; every other build is untouched. A subset whose builds can -/// share chains is followed by one Pass 2 candidate per shared grid, in -/// which each group reads one shared chain, each member merging its newest -/// tumbling windows: the finest member's tumbling windows (`nested_window_groups`), then the -/// gcd of the members' widths when that groups them differently -/// (`gcd_window_groups`); and by one error candidate per group whose -/// sizing fails. Each grid groups every build of a subset at once: a -/// shared gcd grid over only some of the builds that could share it is not -/// listed. An error only for more than -/// `MAX_TUMBLING_WINDOW_CANDIDATES` tumbling-window candidates; a Pass 2 candidate whose -/// composition fails carries its own error. -pub fn window_composition_pass2_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Result, CseError> { - if candidates.len() > MAX_TUMBLING_WINDOW_CANDIDATES { - return Err(CseError::TooManyCandidates { - counted: "summary builds that could be built in tumbling windows", - found: candidates.len(), - max: MAX_TUMBLING_WINDOW_CANDIDATES, - }); - } - let rollable = rollable_tumbling_window_candidates(roots, candidates); - let mut pass2_candidates = Vec::new(); - for mask in 1u64..(1u64 << candidates.len()) { - let chosen: Vec = (0..candidates.len()) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| candidates[i].clone()) - .collect(); - pass2_candidates.push(WindowCompositionPass2Candidate { - roots: in_tumbling_windows(roots, &chosen, &[]), - tumbling_window_candidates: chosen.clone(), - shared_chains: Vec::new(), - }); - let chosen_rollable: Vec = (0..candidates.len()) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| rollable[i]) - .collect(); - push_every_shared_chain(roots, &chosen, &chosen_rollable, &mut pass2_candidates); - } - Ok(pass2_candidates) -} - -/// The Pass 2 candidates of [`window_composition_pass2_candidates`] in -/// which every one of `candidates` is in tumbling windows and reads a shared chain: -/// that function's candidates of the whole set with shared chains, in its -/// order, without building the other subsets'. -pub fn window_composition_shared_chain_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Vec { - let rollable = rollable_tumbling_window_candidates(roots, candidates); - let mut pass2_candidates = Vec::new(); - push_every_shared_chain(roots, candidates, &rollable, &mut pass2_candidates); - pass2_candidates -} - -/// Push the shared-chain Pass 2 candidates of the subset `chosen` (each -/// rollable or not, `chosen_rollable`) of [`window_composition_pass2_candidates`]. -fn push_every_shared_chain( - roots: &[QueryRoot], - chosen: &[TumblingWindowCandidate], - chosen_rollable: &[bool], - pass2_candidates: &mut Vec, -) { - // Shared chains on the members' own finest tumbling windows, then, when it - // groups them differently, on the gcd of their widths. - let on_member_tumbling_windows = nested_window_groups(chosen); - let on_gcd_tumbling_windows = gcd_window_groups(chosen); - push_shared_chains(roots, chosen, &on_member_tumbling_windows, pass2_candidates); - if on_gcd_tumbling_windows != on_member_tumbling_windows { - push_shared_chains(roots, chosen, &on_gcd_tumbling_windows, pass2_candidates); - } - // Coarser groupings read a finer grouping's chain, rolled up. - push_shared_chains( - roots, - chosen, - &rollup_window_groups(chosen, chosen_rollable), - pass2_candidates, - ); -} - -/// Push the Pass 2 candidate in which each of `groups` reads its one shared -/// chain: the groups whose members have a shared size, and the error of -/// each group whose sizing fails. Nothing when no group has a shared size. -fn push_shared_chains( - roots: &[QueryRoot], - chosen: &[TumblingWindowCandidate], - groups: &[SharedTumblingWindowChain], - pass2_candidates: &mut Vec, -) { - // Each group with a size that serves every member, and that build. - let mut shared = Vec::new(); - for group in groups { - match shared_chain_build(chosen, &group.members) { - Ok(Some(build)) => shared.push((group.clone(), build)), - Ok(None) => {} - Err(error) => pass2_candidates.push(WindowCompositionPass2Candidate { - roots: Err(error), - tumbling_window_candidates: chosen.to_vec(), - shared_chains: vec![group.clone()], - }), - } - } - if !shared.is_empty() { - pass2_candidates.push(WindowCompositionPass2Candidate { - roots: in_tumbling_windows(roots, chosen, &shared), - tumbling_window_candidates: chosen.to_vec(), - shared_chains: shared.into_iter().map(|(group, _)| group).collect(), - }); - } -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// shared chain at the gcd of their tumbling-window widths, each group with its -/// chain: every build over one input ([`same_build_over_one_input`]) in one -/// group, whatever its tumbling-window width, on tumbling windows of the gcd of the members' -/// widths over the longest window. Finer tumbling windows than any member builds mean -/// more states and more merges; Stage 3's pricing decides whether sharing -/// pays. Groups of members with the same tumbling windows and window are left to the -/// other rules. -fn gcd_window_groups(candidates: &[TumblingWindowCandidate]) -> Vec { - let mut groups: Vec> = Vec::new(); - for (i, candidate) in candidates.iter().enumerate() { - match groups - .iter_mut() - .find(|group| same_build_over_one_input(&candidates[group[0]], candidate)) - { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - groups - .into_iter() - .filter_map(|members| { - let tumbling_windows = |i: usize| { - ( - candidates[i].tumbling_window_width_ms, - candidates[i].tumbling_window_count, - ) - }; - if members - .iter() - .all(|&i| tumbling_windows(i) == tumbling_windows(members[0])) - { - return None; - } - let tumbling_window_width_ms = members.iter().fold(0, |width, &i| { - asap_types::workload::greatest_common_divisor( - width, - candidates[i].tumbling_window_width_ms, - ) - }); - let longest = longest(candidates, &members); - let tumbling_window_count = - u32::try_from(candidates[longest].window_ms() / tumbling_window_width_ms).ok()?; - Some(SharedTumblingWindowChain { - members, - tumbling_window_width_ms, - tumbling_window_count, - }) - }) - .collect() -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// shared chain, each group with its chain: builds of one family -/// ([`same_family`], at any size) that read the same input below their -/// ranges the same way (item and weight, grouping and filter), with the -/// same output column names (each reader reads the shared chain's columns -/// by position). Their windows end at the same time, so each is the newest -/// tumbling windows of the longest. The shared tumbling windows are the group's finest member's -/// own tumbling windows: a build joins a group only when that width divides its own -/// tumbling-window width (the gcd of the widths, never finer than a chain some member -/// builds anyway), and otherwise starts its own group. Members at different -/// repeat intervals read the same tumbling windows, each at its own cadence. One chain -/// over the longest window, sized for every reader -/// ([`size_for_all_readers`]), serves them all. Groups of members with the -/// same tumbling windows and window are left to the other rules. -fn nested_window_groups(candidates: &[TumblingWindowCandidate]) -> Vec { - // Finest tumbling windows first: each build joins the first group over its input - // whose (finest) tumbling-window width divides its own, so a build whose tumbling windows no - // group's divide starts its own group instead of breaking another. - let mut by_width: Vec = (0..candidates.len()).collect(); - by_width.sort_by_key(|&i| candidates[i].tumbling_window_width_ms); - let mut groups: Vec> = Vec::new(); - for i in by_width { - let candidate = &candidates[i]; - match groups.iter_mut().find(|group| { - let finest = &candidates[group[0]]; - candidate - .tumbling_window_width_ms - .is_multiple_of(finest.tumbling_window_width_ms) - && same_build_over_one_input(finest, candidate) - }) { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - let mut shared = groups - .into_iter() - .filter_map(|mut members| { - let tumbling_windows = |i: usize| { - ( - candidates[i].tumbling_window_width_ms, - candidates[i].tumbling_window_count, - ) - }; - if members - .iter() - .all(|&i| tumbling_windows(i) == tumbling_windows(members[0])) - { - return None; - } - let tumbling_window_width_ms = candidates[members[0]].tumbling_window_width_ms; - let longest = longest(candidates, &members); - let tumbling_window_count = - u32::try_from(candidates[longest].window_ms() / tumbling_window_width_ms).ok()?; - members.sort_unstable(); - Some(SharedTumblingWindowChain { - members, - tumbling_window_width_ms, - tumbling_window_count, - }) - }) - .collect::>(); - // In the order of their first members, as [`gcd_window_groups`] lists - // its groups, so equal groupings compare equal. - shared.sort_by_key(|group| group.members[0]); - shared -} - -/// Whether each of `candidates` may be read rolled up from a finer -/// grouping's chain: a sketch build over a SQL relation's rows (a PromQL -/// rollup over newest tumbling windows is not offered yet) whose every reader in -/// `roots` is an estimate whose merged readout keeps a guarantee, as for -/// the rollup rule. -fn rollable_tumbling_window_candidates( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); - for node in &nodes { - for child in node.children() { - readers.entry(Rc::as_ptr(child)).or_default().push(node); - } - } - candidates - .iter() - .map(|candidate| match candidate.build.asap() { - Some(ASAPOp::SummaryAgg { - child: range, - family: family @ FieldDataType::Sketch(..), - .. - }) => { - range.children()[0].result_kind == OperatorResultKind::Relation - && readers - .get(&Rc::as_ptr(&candidate.build)) - .is_some_and(|readers| read_with_merged_guarantee(family, readers)) - } - _ => false, - }) - .collect() -} - -/// The tumbling-window candidates (positions in `candidates`) that can read one -/// chain of a finer grouping's tumbling windows, each group with its chain: a host, -/// and every candidate whose group keys are the host's or fewer -/// ([`rolls_up_into`]), whose window is at most the host's and whose tumbling window -/// width the host's divides. Each member merges the newest tumbling windows of its -/// window, and a coarser member then rolls the host's groups up into its -/// own (a regrouping `SummaryMerge`): `p90 per city over 20 minutes` in -/// 10 s tumbling windows serves `p90 over 5 minutes` as the newest 30 tumbling windows merged -/// across cities. A coarser member is one `rollable[i]` allows -/// ([`rollable_tumbling_window_candidates`]). Hosts are tried by most group keys, -/// then longest window, and each candidate joins one group. Only groups -/// with a coarser member are listed; the others are -/// [`nested_window_groups`]'s. -fn rollup_window_groups( - candidates: &[TumblingWindowCandidate], - rollable: &[bool], -) -> Vec { - let mut hosts: Vec = (0..candidates.len()).collect(); - hosts.sort_by_key(|&i| { - ( - std::cmp::Reverse(group_key_count(&candidates[i].build)), - std::cmp::Reverse(candidates[i].window_ms()), - i, - ) - }); - let mut grouped = vec![false; candidates.len()]; - let mut groups = Vec::new(); - for host in hosts { - if grouped[host] { - continue; - } - let coarser = |i: usize| { - group_key_count(&candidates[i].build) < group_key_count(&candidates[host].build) - }; - let reads_host = |i: usize| { - let (member, chain) = (&candidates[i], &candidates[host]); - i == host - || (!grouped[i] - && member.window_ms() <= chain.window_ms() - && member - .tumbling_window_width_ms - .is_multiple_of(chain.tumbling_window_width_ms) - && rolls_up_into(member, chain) - && (!coarser(i) || rollable[i])) - }; - let members: Vec = (0..candidates.len()).filter(|&i| reads_host(i)).collect(); - if !members.iter().any(|&i| coarser(i)) { - continue; - } - for &i in &members { - grouped[i] = true; - } - groups.push(SharedTumblingWindowChain { - members, - tumbling_window_width_ms: candidates[host].tumbling_window_width_ms, - tumbling_window_count: candidates[host].tumbling_window_count, - }); - } - groups.sort_by_key(|group| group.members[0]); - groups -} - -/// The number of group keys of a build grouped by listed keys; 0 for any -/// other build. -fn group_key_count(build: &OperatorNode) -> usize { - match build.asap() { - Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() => by.len(), - _ => 0, - } -} - -/// Whether `member`'s build is `host`'s ([`same_build_over_one_input`]), -/// or the same but grouped by fewer of `host`'s keys, of a family whose -/// states merge across groups ([`FamilyCapability::merges_groups`]), so a -/// rollup of `host`'s states answers it. -fn rolls_up_into(member: &TumblingWindowCandidate, host: &TumblingWindowCandidate) -> bool { - if same_build_over_one_input(member, host) { - return true; - } - let ( - Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction: Reduction::Reduce(by), - grouping, - filter, - }), - Some(ASAPOp::SummaryAgg { - child: host_range, - family: host_family, - input: host_input, - reduction: Reduction::Reduce(host_by), - grouping: host_grouping, - filter: host_filter, - }), - ) = (member.build.asap(), host.build.asap()) - else { - return false; - }; - let (below, host_below) = (range.children()[0], host_range.children()[0]); - // Both ranges read the same rows below, so a key names one column. - !by.is_without() - && !host_by.is_without() - && by.len() < host_by.len() - && by.keys().iter().all(|key| host_by.keys().contains(key)) - && FamilyCapability::merges_groups(family) - && same_build_family(family, host_family) - && (Rc::ptr_eq(below, host_below) || below == host_below) - && input == host_input - && grouping == host_grouping - && filter == host_filter -} - -/// Whether two tumbling-window candidates build one family (at any size) over the -/// same input below their ranges, read the same way, with the same output -/// column names. -fn same_build_over_one_input(a: &TumblingWindowCandidate, b: &TumblingWindowCandidate) -> bool { - let ( - Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }), - Some(ASAPOp::SummaryAgg { - child: range_b, - family: family_b, - input: input_b, - reduction: reduction_b, - grouping: grouping_b, - filter: filter_b, - }), - ) = (a.build.asap(), b.build.asap()) - else { - return false; - }; - let (below, below_b) = (range.children()[0], range_b.children()[0]); - let names = |build: &OperatorNode| -> Vec<(String, Option)> { - build - .schema - .fields - .iter() - .map(|f| (f.name.clone(), f.table.clone())) - .collect() - }; - names(&a.build) == names(&b.build) - && same_build_family(family, family_b) - && (Rc::ptr_eq(below, below_b) || below == below_b) - && input == input_b - && reduction == reduction_b - && grouping == grouping_b - && filter == filter_b -} - -/// The member of `group` (positions in `candidates`) with the longest -/// window. -fn longest(candidates: &[TumblingWindowCandidate], group: &[usize]) -> usize { - *group - .iter() - .max_by_key(|&&i| candidates[i].window_ms()) - .expect("a group has members") -} - -/// The build one chain shared by `group` (positions in `candidates`) is -/// made of: the longest member's, sized for every member -/// ([`sized_for_all`]); `None` when their sizes have no shared size. -fn shared_chain_build( - candidates: &[TumblingWindowCandidate], - group: &[usize], -) -> Result>, CseError> { - // Of the longest windows, the finest grouping: a rollup group's host. - let host = *group - .iter() - .max_by_key(|&&i| { - ( - candidates[i].window_ms(), - group_key_count(&candidates[i].build), - ) - }) - .expect("a group has members"); - let members: Vec<&Rc> = std::iter::once(host) - .chain(group.iter().copied().filter(|&i| i != host)) - .map(|i| &candidates[i].build) - .collect(); - sized_for_all(&members) -} - -/// `roots` with the build of each of `candidates` built as its tumbling -/// tumbling windows, and every guarantee composed again. The members of each -/// [`SharedTumblingWindowChain`] (with its [`shared_chain_build`]) read that one -/// chain: each merges the newest tumbling windows covering its own window. -fn in_tumbling_windows( - roots: &[QueryRoot], - candidates: &[TumblingWindowCandidate], - shared_chains: &[(SharedTumblingWindowChain, Rc)], -) -> Result, CseError> { - let mut replaced: HashMap<*const OperatorNode, Rc> = HashMap::new(); - for (shared, build) in shared_chains { - let chain = tumbling_window_chain(&TumblingWindowCandidate { - build: Rc::clone(build), - tumbling_window_width_ms: shared.tumbling_window_width_ms, - tumbling_window_count: shared.tumbling_window_count, - })?; - for &i in &shared.members { - let newest = u32::try_from(candidates[i].window_ms() / shared.tumbling_window_width_ms) - .expect("a member's window is at most the chain's"); - let merge = Rc::new(OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&chain)], - newest_tumbling_windows: (newest < shared.tumbling_window_count).then_some(newest), - group_by: None, - }))?); - // A coarser member rolls the chain's groups up into its own. - let member = &candidates[i].build; - let read = if group_key_count(member) == group_key_count(build) { - merge - } else { - regrouped(member, &merge).ok_or(CseError::Unsupported( - "a coarser tumbling-window build whose keys the shared chain does not hold", - ))? - }; - replaced.insert(Rc::as_ptr(member), read); - } - } - for candidate in candidates { - if let std::collections::hash_map::Entry::Vacant(entry) = - replaced.entry(Rc::as_ptr(&candidate.build)) - { - entry.insert(Rc::new(OperatorNode::new(Operator::ASAP( - ASAPOp::SummaryMerge { - children: vec![tumbling_window_chain(candidate)?], - newest_tumbling_windows: None, - group_by: None, - }, - ))?)); - } - } - let mut rebuilt: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// `roots` with the `SummaryMerge` `merge` (over one [`ASAPOp::SummaryTumblingWindowChain`] -/// of `roots`) reading the newest tumbling windows of another chain over the same -/// input: `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of family `family` (a size -/// that serves every reader of the chain). The merge reads the newest -/// tumbling windows covering its own window (its newest tumbling windows, or its whole chain), -/// which `tumbling_window_width_ms` divides and the new chain covers; every guarantee -/// is composed again. -pub(crate) fn with_tumbling_window_chain( - roots: &[QueryRoot], - merge: &Rc, - family: FieldDataType, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Result, CseError> { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: own_newest, - .. - }) = merge.asap() - else { - unreachable!("a merge over a tumbling-window chain"); - }; - let own = &children[0]; - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - input, - reduction, - grouping, - filter, - tumbling_window_width_ms: own_width, - tumbling_window_count: own_count, - .. - }) = own.asap() - else { - unreachable!("a merge over a tumbling-window chain"); - }; - // The merge's window: its newest tumbling windows, or the whole chain. - let window_ms = own_width * u64::from(own_newest.unwrap_or(*own_count)); - let newest = u32::try_from(window_ms / tumbling_window_width_ms) - .expect("a reader's window is at most the chain's"); - let mut chain = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(child), - family, - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms, - tumbling_window_count, - }))?; - for (field, kept) in chain.schema.fields.iter_mut().zip(&own.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - let read = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::new(chain)], - newest_tumbling_windows: (newest < tumbling_window_count).then_some(newest), - group_by: None, - }))?; - let replaced = HashMap::from([(Rc::as_ptr(merge), Rc::new(read))]); - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// `roots` with the summary build `build` (a `SummaryAgg` of `roots`) -/// built at family `family` instead (the same algorithm at another size), -/// its output names kept; every guarantee is composed again. -pub(crate) fn with_resized_build( - roots: &[QueryRoot], - build: &Rc, - family: FieldDataType, -) -> Result, CseError> { - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - grouping, - filter, - .. - }) = build.asap() - else { - unreachable!("a summary build"); - }; - let mut resized = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family, - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }))?; - for (field, kept) in resized.schema.fields.iter_mut().zip(&build.schema.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - let replaced = HashMap::from([(Rc::as_ptr(build), Rc::new(resized))]); - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(rebuild(node, &replaced, &mut rebuilt)?) - } - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), rebuild(node, &replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// The [`ASAPOp::SummaryTumblingWindowChain`] of `candidate`'s tumbling windows, with the build's -/// output names. -fn tumbling_window_chain( - candidate: &TumblingWindowCandidate, -) -> Result, CseError> { - let Some(ASAPOp::SummaryAgg { - child: range, - family, - input, - reduction, - grouping, - filter, - }) = candidate.build.asap() - else { - unreachable!("a tumbling-window candidate's build is a SummaryAgg"); - }; - let mut chain = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(range.children()[0]), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - tumbling_window_width_ms: candidate.tumbling_window_width_ms, - tumbling_window_count: candidate.tumbling_window_count, - }))?; - for (field, kept) in chain - .schema - .fields - .iter_mut() - .zip(&candidate.build.schema.fields) - { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - Ok(Rc::new(chain)) -} - -/// `node` with every replaced build below it swapped for its shared build; -/// the same `Rc` when nothing below changed. -fn rebuild( - node: &Rc, - replaced: &HashMap<*const OperatorNode, Rc>, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, -) -> Result, CseError> { - if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let out = match replaced.get(&Rc::as_ptr(node)) { - Some(build) => Rc::clone(build), - None => { - let mut changed = false; - let mut children = HashMap::new(); - for child in node.children() { - let new = rebuild(child, replaced, rebuilt)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - if changed { - Rc::new(node.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) - } else { - Rc::clone(node) - } - } - }; - rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); - Ok(out) -} - -/// One coarse build the rollup rule can answer from a finer one: the -/// coarse `SummaryAgg`, the finer node it reads, and the node that replaces -/// the coarse value or state. -#[derive(Debug, Clone)] -pub struct RollupCandidate { - /// The coarse build (`SummaryAgg`). An exact accumulator's every - /// reader is a `FinalizeExactAccumulator`, which the rollup replaces; a - /// sketch's every reader is an estimate its family answers, and the - /// rollup replaces the build itself. - pub coarse: Rc, - /// For an exact accumulator, the finer build's - /// `FinalizeExactAccumulator`, whose values the rollup reads; for a - /// sketch, the finest build of the same algorithm over the same rows - /// (a `SummaryAgg`), whose states the rollup merges. Both are already - /// in the candidate. - pub finer: Rc, - /// For an exact accumulator, the finalized exact accumulator over - /// `finer`'s values, grouped by the coarse keys, with the coarse value's - /// column names; for a sketch, the regrouping `SummaryMerge` of - /// `finer`'s states by the coarse keys, at `finer`'s size (a Pass 2 - /// candidate resizes `finer` for every reader). - pub rollup: Rc, -} - -/// A Pass 2 candidate of the rollup rule: the candidate's roots with the -/// coarse build of each chosen rollup candidate answered from its finer -/// build, and every guarantee composed again; the error when that fails. -#[derive(Debug)] -pub struct RollupPass2Candidate { - /// The candidate's roots with those builds rolled up and every guarantee - /// composed again; `Err` when the rebuilt DAG does not derive or - /// compose, which drops only this candidate. - pub roots: Result, CseError>, -} - -/// The most rollup or top-k reuse candidates one candidate may have: every -/// non-empty subset is a Pass 2 candidate, about a million at 20. -pub(crate) const MAX_REUSE_CANDIDATES: usize = 20; - -/// The exact accumulator that merges finalized values of `family` across -/// disjoint groups, by the family-capability table: a family whose states -/// merge over disjoint (or overlapping) selections into an exact state. -/// `None` for a family without such a merge: counter states merge only -/// consecutive windows, and sketch states have no regrouping merge in the IR. -pub(crate) fn rollup_kind(family: &FieldDataType) -> Option { - let FieldDataType::ExactAggregate(kind, _) = family else { - return None; - }; - // An exact accumulator's merged state is exact; a counter state merges - // only consecutive windows. - if FamilyCapability::of(family).merge_inputs? == MergeInputRule::DisjointInTime { - return None; - } - match kind { - // Counts add up. - ExactKind::Sum | ExactKind::Count => Some(ExactKind::Sum), - ExactKind::Min => Some(ExactKind::Min), - ExactKind::Max => Some(ExactKind::Max), - ExactKind::Rate | ExactKind::IRate | ExactKind::Increase => None, - } -} - -/// The rollup candidates of the candidate with query roots `roots`, children -/// first: each coarse exact accumulator build read only through its -/// finalized value, with its coarsest finer build of the same family over -/// the same rows that the candidate finalizes; then each coarse sketch build -/// read only by estimates, with the finest build of its algorithm over the -/// same rows whose keys include its own (`sketch_rollup_candidates`). -#[cfg(any(test, feature = "test-support"))] -pub fn rollup_candidates(roots: &[QueryRoot]) -> Vec { - rollup_candidates_of(roots, None) -} - -/// The rollup candidate of coarse build `coarse` among -/// [`rollup_candidates`]`(roots)`, the first when several, computed for that -/// build alone (each coarse build's candidate depends only on `roots`, not -/// on the other coarse builds). -pub(crate) fn rollup_candidate_for( - roots: &[QueryRoot], - coarse: *const OperatorNode, -) -> Option { - rollup_candidates_of(roots, Some(coarse)).into_iter().next() -} - -/// [`rollup_candidates`], over the coarse builds that are `only` when set. -fn rollup_candidates_of( - roots: &[QueryRoot], - only: Option<*const OperatorNode>, -) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut readers: HashMap<*const OperatorNode, Vec<&Rc>> = HashMap::new(); - for node in &nodes { - for child in node.children() { - readers.entry(Rc::as_ptr(child)).or_default().push(node); - } - } - let finalized = |build: &Rc| -> Vec<&Rc> { - readers - .get(&Rc::as_ptr(build)) - .map_or_else(Vec::new, |readers| { - readers - .iter() - .filter(|r| matches!(r.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .copied() - .collect() - }) - }; - let builds: Vec<&Rc> = nodes.iter().filter(|n| grouped_exact(n)).collect(); - let wanted = |coarse: &Rc| only.is_none_or(|only| Rc::as_ptr(coarse) == only); - let mut candidates = Vec::new(); - for &coarse in builds.iter().filter(|coarse| wanted(coarse)) { - let finals = finalized(coarse); - // Every reader of the coarse state reads its finalized value. - if finals.is_empty() || finals.len() != readers[&Rc::as_ptr(coarse)].len() { - continue; - } - let mut best: Option<(usize, RollupCandidate)> = None; - for &fine in &builds { - let Some(keys) = finer_keys(coarse, fine) else { - continue; - }; - if best.as_ref().is_some_and(|(k, _)| *k <= keys) { - continue; - } - let Some(&finer) = finalized(fine).first() else { - continue; - }; - if let Some(rollup) = rolled_up(coarse, finer, finals[0]) { - best = Some(( - keys, - RollupCandidate { - coarse: Rc::clone(coarse), - finer: Rc::clone(finer), - rollup, - }, - )); - } - } - candidates.extend(best.map(|(_, candidate)| candidate)); - } - candidates.extend(sketch_rollup_candidates(&nodes, &readers, only)); - candidates -} - -/// The sketch rollup candidates among `nodes` (children first), whose -/// readers are `readers`: each per-group sketch build grouped by listed keys -/// and read only by estimates its family answers, answered from the finest -/// build ([`finer_keys`] with the most keys, and no build finer than it) of -/// the same algorithm (at any size), update and filter over the same rows, -/// whose family's states merge into a state with a guarantee -/// ([`regroupable`]). Every coarse build of one finest build is one -/// choice in [`rollup_pass2_candidates`], so the choices stay few however -/// many groupings a workload asks for. -fn sketch_rollup_candidates( - nodes: &[Rc], - readers: &HashMap<*const OperatorNode, Vec<&Rc>>, - only: Option<*const OperatorNode>, -) -> Vec { - let builds: Vec<&Rc> = nodes.iter().filter(|n| regroupable(n)).collect(); - // The finer build with the most keys; among several (groupings that - // are not nested), the one whose sorted key names come first, so the - // choice does not depend on query order. - let key_names = |build: &OperatorNode| -> Vec { - let mut names: Vec = build - .schema - .fields - .iter() - .filter(|field| field.is_plain()) - .map(column_name) - .collect(); - names.sort(); - names - }; - let finest = |coarse: &OperatorNode| { - builds - .iter() - .filter_map(|&fine| Some((finer_keys(coarse, fine)?, fine))) - .min_by(|(keys, fine), (other_keys, other)| { - other_keys - .cmp(keys) - .then_with(|| key_names(fine).cmp(&key_names(other))) - }) - .map(|(_, fine)| fine) - }; - let mut candidates = Vec::new(); - for &coarse in builds - .iter() - .filter(|coarse| only.is_none_or(|only| Rc::as_ptr(coarse) == only)) - { - let Some(ASAPOp::SummaryAgg { family, .. }) = coarse.asap() else { - continue; - }; - if !readers - .get(&Rc::as_ptr(coarse)) - .is_some_and(|readers| read_with_merged_guarantee(family, readers)) - { - continue; - } - // A build finer than the one with the most keys would have more. - // Over more rows, the finer build's states are first filtered by - // the coarse build's extra conjuncts (a compensating filter). - let rollup = match finest(coarse) { - Some(fine) => regrouped(coarse, fine).map(|rollup| (fine, rollup)), - None => builds - .iter() - .filter_map(|&fine| Some((filtered_finer(coarse, fine)?, fine))) - // The same choice as `finest`'s, whatever the query order. - .min_by(|((keys, _), fine), ((other_keys, _), other)| { - other_keys - .cmp(keys) - .then_with(|| key_names(fine).cmp(&key_names(other))) - }) - .and_then(|((_, pred), fine)| { - let kept = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: Rc::clone(fine), - })) - .ok()?; - Some((fine, regrouped(coarse, &kept)?)) - }), - }; - if let Some((fine, rollup)) = rollup { - candidates.push(RollupCandidate { - coarse: Rc::clone(coarse), - finer: Rc::clone(fine), - rollup, - }); - } - } - candidates -} - -/// Whether every one of `readers`, the readers of a build of `family`, is -/// an estimate whose merged readout keeps a guarantee: not a heap's top-k -/// or a UnivMon's entropy. -fn read_with_merged_guarantee(family: &FieldDataType, readers: &[&Rc]) -> bool { - let capability = FamilyCapability::of(family); - readers.iter().all(|reader| match reader.asap() { - Some(ASAPOp::SummaryEstimate { query, .. }) => { - capability.answers(query) - && matches!( - capability.merged_error(query), - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) - } - _ => false, - }) -} - -/// Whether `node` is a per-group sketch build grouped by listed keys whose -/// states a rollup may merge across groups -/// ([`FamilyCapability::merges_groups`]). -fn regroupable(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: family @ FieldDataType::Sketch(..), - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() && FamilyCapability::merges_groups(family) - ) -} - -/// The regrouping `SummaryMerge` of `fine`'s states by `coarse`'s keys, in -/// `coarse`'s order and with its column names; `None` when a key is not in -/// `fine`'s state. -fn regrouped(coarse: &OperatorNode, fine: &Rc) -> Option> { - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(by), - .. - }) = coarse.asap() - else { - return None; - }; - let keys = (0..by.keys().len()) - .map(|key| { - let name = column_name(&coarse.schema.fields[key]); - fine.schema - .fields - .iter() - .position(|field| field.is_plain() && column_name(field) == name) - }) - .collect::>>()?; - regroup_merge(fine, keys, coarse).ok() -} - -/// `SummaryMerge { [state], group_by: keys }`, its columns named as -/// `named`'s (the coarse build it stands for). -fn regroup_merge( - state: &Rc, - keys: Vec, - named: &OperatorNode, -) -> Result, asap_types::ir::SchemaDerivationError> { - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(state)], - group_by: Some(keys), - newest_tumbling_windows: None, - }))?; - let mut schema = merge.schema.clone(); - for (field, coarse) in schema.fields.iter_mut().zip(&named.schema.fields) { - field.name = coarse.name.clone(); - field.table = coarse.table.clone(); - } - Ok(Rc::new(OperatorNode::with_schema(merge.operator, schema))) -} - -/// Whether `node` is an exact accumulator build grouped by listed keys whose -/// family merges across groups ([`rollup_kind`]). -fn grouped_exact(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family, - reduction: Reduction::Reduce(by), - grouping: GroupingStrategy::PerSubpopulationInstance, - .. - }) if !by.is_without() && rollup_kind(family).is_some() - ) -} - -/// The number of group keys of `fine` when it can answer `coarse`: the same -/// family (a sketch's algorithm at any size), update and filter over the -/// same rows, and group keys (by column) that strictly include the coarse -/// ones. -fn finer_keys(coarse: &OperatorNode, fine: &OperatorNode) -> Option { - let ( - Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - filter, - .. - }), - Some(ASAPOp::SummaryAgg { - child: fine_child, - family: fine_family, - input: fine_input, - reduction: Reduction::Reduce(fine_by), - filter: fine_filter, - .. - }), - ) = (coarse.asap(), fine.asap()) - else { - return None; - }; - let names = |node: &OperatorNode, keys: &[usize]| -> Option> { - keys.iter() - .map(|&k| node.schema.fields.get(k).map(column_name)) - .collect() - }; - let coarse_keys = names(child, by.keys())?; - let fine_keys = names(fine_child, fine_by.keys())?; - let same_filter = match (filter, fine_filter) { - (None, None) => true, - // A filter indexes its input's columns: equal only over equal rows. - (a, b) => a == b && (Rc::ptr_eq(child, fine_child) || child == fine_child), - }; - (same_build_family(family, fine_family) - && input == fine_input - && same_filter - && fine_keys.len() > coarse_keys.len() - && coarse_keys.iter().all(|key| fine_keys.contains(key)) - && same_rows(child, fine_child)) - .then_some(fine_keys.len()) -} - -/// The number of group keys of `fine` and the compensating filter over its -/// states when `fine` answers `coarse` from more rows: the same family, -/// update and no build filter, over rows that hold `coarse`'s, whose extra -/// conjuncts read only `fine`'s keys ([`match_finer`]), with group keys that -/// include the coarse ones. `None` over the same rows ([`finer_keys`]). -fn filtered_finer(coarse: &OperatorNode, fine: &OperatorNode) -> Option<(usize, Predicate)> { - let ( - Some(ASAPOp::SummaryAgg { - child, - family, - input, - reduction: Reduction::Reduce(by), - filter: None, - .. - }), - Some(ASAPOp::SummaryAgg { - child: fine_child, - family: fine_family, - input: fine_input, - reduction: Reduction::Reduce(fine_by), - filter: None, - .. - }), - ) = (coarse.asap(), fine.asap()) - else { - return None; - }; - if by.is_without() - || fine_by.is_without() - || !same_build_family(family, fine_family) - || input != fine_input - { - return None; - } - let matched = - crate::pass2::finer_aggregate::match_finer(child, by.keys(), fine_child, fine_by.keys())?; - // The filter reads the finer keys as the state's columns of those names. - let state_columns = fine_by - .keys() - .iter() - .map(|&key| { - // A state's key column keeps the input column's name, not - // always its table qualifier. - let name = &fine_child.schema.fields.get(key)?.name; - fine.schema - .fields - .iter() - .position(|field| field.is_plain() && field.name == *name) - }) - .collect::>>()?; - let pred = matched.compensating_filter?; - Some(( - fine_by.keys().len(), - Predicate(pred.0.map_columns(&mut |key| state_columns[key])), - )) -} - -type ColumnName = (Option, String); - -fn column_name(field: &asap_types::ir::schema::Field) -> ColumnName { - (field.table.clone(), field.name.clone()) -} - -/// Whether `a` and `b` output the same rows: the same or equal nodes, or -/// time-series scans of one metric without predicates (projecting perhaps -/// other labels) under the same time ranges and shifts. -fn same_rows(a: &Rc, b: &Rc) -> bool { - if Rc::ptr_eq(a, b) || a == b { - return true; - } - match (a.non_asap(), b.non_asap()) { - ( - Some(NonASAPOp::Scan { - source: source @ Source::TimeSeries { .. }, - predicates, - .. - }), - Some(NonASAPOp::Scan { - source: other, - predicates: other_predicates, - .. - }), - ) => source == other && predicates.is_empty() && other_predicates.is_empty(), - ( - Some(NonASAPOp::TimeRange { range, kind, child }), - Some(NonASAPOp::TimeRange { - range: other_range, - kind: other_kind, - child: other, - }), - ) => range == other_range && kind == other_kind && same_rows(child, other), - ( - Some(NonASAPOp::TimeShift { shift, child }), - Some(NonASAPOp::TimeShift { - shift: other_shift, - child: other, - }), - ) => shift == other_shift && same_rows(child, other), - ( - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - having: None, - child, - .. - }), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: other_measures, - filters: other_filters, - having: None, - child: other, - .. - }), - ) => { - // These reductions preserve series identity and read the implicit - // sample value. Extra projected labels do not change their values. - // Explicit column indices and filtered measures need remapping and - // are deliberately excluded. - measures == other_measures - && matches!( - measures.as_slice(), - [AggIntent::Avg { col: None } - | AggIntent::Sum { col: None } - | AggIntent::Min { col: None } - | AggIntent::Max { col: None }] - ) - && filters.iter().all(Option::is_none) - && other_filters.iter().all(Option::is_none) - && same_rows(child, other) - } - _ => false, - } -} - -/// The finalized value of `coarse` (`coarse_value`) as an exact -/// accumulator over `finer` (the finer build's finalized values), grouped by -/// the coarse keys, finalized, with `coarse_value`'s column names; `None` -/// when it cannot be built, a non-count finer value is not Float64 (the -/// supported accumulator update type), or it differs from -/// `coarse_value` in type. -fn rolled_up( - coarse: &OperatorNode, - finer: &Rc, - coarse_value: &OperatorNode, -) -> Option> { - let Some(ASAPOp::SummaryAgg { - child, - family, - reduction, - .. - }) = coarse.asap() - else { - return None; - }; - let Some(ASAPOp::FinalizeExactAccumulator { child: fine }) = finer.asap() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(fine_by), - .. - }) = fine.asap() - else { - return None; - }; - // A grouped accumulator outputs its keys, then its value. - let value = fine_by.len(); - // Merge count states directly: converting finalized Int64 counts through - // a Float64 sum would lose integer precision above 2^53. - if matches!(family, FieldDataType::ExactAggregate(ExactKind::Count, _)) { - let merged = regrouped(coarse, fine)?; - let finalized = - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: merged, - })) - .ok()?; - if finalized.schema.fields.len() != coarse_value.schema.fields.len() - || finalized - .schema - .fields - .iter() - .zip(&coarse_value.schema.fields) - .any(|(a, b)| a.dtype != b.dtype || a.nullable != b.nullable) - || finalized.schema.unique_keys != coarse_value.schema.unique_keys - { - return None; - } - // Finalization preserves the count's type; only the reader's SQL alias - // differs from the canonical state readout name. - return Some(Rc::new(OperatorNode::with_schema( - finalized.operator.clone(), - coarse_value.schema.clone(), - ))); - } - if finer.schema.fields.get(value).map(|f| &f.dtype) - != Some(&FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - )) - { - return None; - } - let regrouped = reduction.partitions_over(&child.schema, &finer.schema)?; - if regrouped - .group_keys() - .is_some_and(|keys| keys.iter().any(|&k| k >= value)) - { - return None; - } - let kind = rollup_kind(family)?; - // An exact accumulator over the finer value column: what Pass 1 builds - // for a Sum, Min or Max of that column (no item, signed weights). - let weight = finer.schema.fields[value].column_ref(); - resolve_column_ref(&weight, &finer.schema).ok()?; - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(finer), - input: SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(weight), - weight_domain: WeightDomain::UnknownOrSigned, - }, - family: FieldDataType::exact(kind), - reduction: regrouped, - grouping: GroupingStrategy::default(), - filter: None, - })) - .ok()?; - let mut finalized = - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - })) - .ok()?; - let types = |node: &OperatorNode| -> Vec { - node.schema.fields.iter().map(|f| f.dtype.clone()).collect() - }; - if types(&finalized) != types(coarse_value) { - return None; - } - // A count summed is named after the sum: rename to the coarse columns. - if finalized.schema.fields != coarse_value.schema.fields { - let cols = coarse_value - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| asap_types::ir::ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(); - finalized = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: finalized, - })) - .ok()?; - } - Some(finalized) -} - -/// The independent choices among rollup candidates `candidates` -/// ([`rollup_candidates`]), in order: each exact accumulator's alone, and -/// every sketch candidate of one finest build together, since they share -/// that build's size. -pub(crate) fn rollup_choices(candidates: &[RollupCandidate]) -> Vec> { - let mut choices: Vec> = Vec::new(); - for candidate in candidates { - let sketch = is_sketch_build(&candidate.finer); - match choices.iter_mut().find(|choice| { - sketch - && is_sketch_build(&choice[0].finer) - && Rc::ptr_eq(&choice[0].finer, &candidate.finer) - }) { - Some(choice) => choice.push(candidate.clone()), - None => choices.push(vec![candidate.clone()]), - } - } - choices -} - -fn is_sketch_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) -} - -/// Every Pass 2 candidate of the rollup rule for the candidate with query -/// roots `roots` and rollup candidates `candidates` ([`rollup_candidates`]): -/// one per non-empty subset of their choices (`rollup_choices`), in -/// binary-counter order (the first the lowest bit). A chosen sketch choice -/// resizes its finest build for every reader, its own and each coarse -/// build's (`sized_for_all`: the strictest consumer's size), and each -/// coarse build becomes the regrouping merge of the resized build's -/// states. An error only for more than `MAX_REUSE_CANDIDATES` choices; a -/// Pass 2 candidate whose composition fails carries its own error. -pub fn rollup_pass2_candidates( - roots: &[QueryRoot], - candidates: &[RollupCandidate], -) -> Result, CseError> { - Ok(subsets(&rollup_choices(candidates), "rollup")? - .into_iter() - .map(|choices| { - let values: HashMap<*const OperatorNode, &RollupCandidate> = choices - .iter() - .flatten() - .filter(|candidate| !is_sketch_build(&candidate.finer)) - .map(|candidate| (Rc::as_ptr(&candidate.coarse), candidate)) - .collect(); - // Every finalized value of a chosen coarse exact build is - // replaced. - let mut replaced: HashMap<*const OperatorNode, Rc> = roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|node| match node.asap() { - Some(ASAPOp::FinalizeExactAccumulator { child }) => { - let candidate = values.get(&Rc::as_ptr(child))?; - Some((Rc::as_ptr(&node), Rc::clone(&candidate.rollup))) - } - _ => None, - }) - .collect(); - let roots = sketch_rollups(&choices).and_then(|sketches| { - replaced.extend(sketches); - replace_all(roots, &replaced) - }); - RollupPass2Candidate { roots } - }) - .collect()) -} - -/// The replacements of the sketch rollup choices among `choices` -/// ([`rollup_choices`]): each finest build by the build sized for it and -/// every coarse build it serves, and each coarse build by the regrouping -/// merge of that build's states. An error when a finest build and its -/// coarse builds have no shared size, or a node does not derive. -fn sketch_rollups( - choices: &[Vec], -) -> Result)>, CseError> { - let unsupported = || { - CseError::Unsupported("a sketch rollup whose finest and coarse builds have no shared size") - }; - // A finest build can also be the coarse build of another choice, read - // through a compensating filter. Each build is replaced once: as a - // finest build, sized for its readers. - let finest: HashSet<*const OperatorNode> = choices - .iter() - .filter(|choice| is_sketch_build(&choice[0].finer)) - .map(|choice| Rc::as_ptr(&choice[0].finer)) - .collect(); - let mut replaced = Vec::new(); - for choice in choices { - let fine = &choice[0].finer; - if !is_sketch_build(fine) { - continue; - } - let mut members = vec![fine]; - members.extend(choice.iter().map(|candidate| &candidate.coarse)); - let sized = sized_for_all(&members)?.ok_or_else(unsupported)?; - replaced.push((Rc::as_ptr(fine), Rc::clone(&sized))); - for candidate in choice { - if finest.contains(&Rc::as_ptr(&candidate.coarse)) { - continue; - } - let Some(ASAPOp::SummaryMerge { - group_by: Some(keys), - .. - }) = candidate.rollup.asap() - else { - unreachable!("a sketch rollup candidate's rollup is a regrouping merge"); - }; - // A compensating filter is kept over the resized states. - let states = match candidate.rollup.children()[0].non_asap() { - Some(NonASAPOp::Filter { pred, .. }) => { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: pred.clone(), - child: Rc::clone(&sized), - }))? - } - _ => Rc::clone(&sized), - }; - let merge = regroup_merge(&states, keys.clone(), &candidate.coarse)?; - replaced.push((Rc::as_ptr(&candidate.coarse), merge)); - } - } - Ok(replaced) -} - -/// Every non-empty subset of `candidates`, in binary-counter order (the -/// first candidate the lowest bit); an error past [`MAX_REUSE_CANDIDATES`]. -fn subsets(candidates: &[T], rule: &'static str) -> Result>, CseError> { - let n = candidates.len(); - if n > MAX_REUSE_CANDIDATES { - return Err(CseError::TooManyCandidates { - counted: match rule { - "rollup" => "rollup choices", - _ => "top-k reuse candidates", - }, - found: n, - max: MAX_REUSE_CANDIDATES, - }); - } - Ok((1u64..(1u64 << n)) - .map(|mask| { - (0..n) - .filter(|&i| mask >> i & 1 == 1) - .map(|i| candidates[i].clone()) - .collect() - }) - .collect()) -} - -/// `roots` with every node in `replaced` swapped for its replacement, and -/// the nodes a replacement reads rebuilt too (a rollup may read a build -/// another rollup replaces); every guarantee composed again. -fn replace_all( - roots: &[QueryRoot], - replaced: &HashMap<*const OperatorNode, Rc>, -) -> Result, CseError> { - fn walk( - node: &Rc, - replaced: &HashMap<*const OperatorNode, Rc>, - rebuilt: &mut HashMap<*const OperatorNode, Rc>, - ) -> Result, CseError> { - if let Some(done) = rebuilt.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let source = replaced.get(&Rc::as_ptr(node)).unwrap_or(node); - let mut changed = !Rc::ptr_eq(source, node); - let mut children = HashMap::new(); - for child in source.children() { - let new = walk(child, replaced, rebuilt)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - let out = if !changed { - Rc::clone(node) - } else if children - .iter() - .all(|(old, new)| std::ptr::eq(*old, Rc::as_ptr(new))) - { - Rc::clone(source) - } else { - Rc::new(source.with_new_children(|child| children[&Rc::as_ptr(child)].clone())?) - }; - rebuilt.insert(Rc::as_ptr(node), Rc::clone(&out)); - Ok(out) - } - let mut rebuilt = HashMap::new(); - let mut out = Vec::new(); - for root in roots { - out.push(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, replaced, &mut rebuilt)?), - QueryRoot::Scalar(expr) => { - let mut nodes = HashMap::new(); - for node in expr.operator_refs() { - nodes.insert(Rc::as_ptr(node), walk(node, replaced, &mut rebuilt)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| nodes[&Rc::as_ptr(node)].clone()), - ) - } - }); - } - Ok(recompose_guarantees(&out)?) -} - -/// Exact top-k computations of one input and partition with at least two -/// different `k`, in the order a children-first walk meets them, and the -/// ranking they all read in a reuse candidate: `Limit(k_max)` over `Sort` -/// of the input, partitioned as they are. -#[derive(Debug, Clone)] -pub(crate) struct TopKReuseCandidate { - /// The plain `Aggregate` top-k nodes of one input and partition, in - /// children-first order; at least two different `k`. - pub members: Vec>, - /// `Limit(k_max)` over a descending `Sort` of their input, partitioned - /// as they are: what every member reads in a reuse candidate. - pub ranking: Rc, -} - -/// The exact top-k reuse candidates of the candidate with query roots `roots`: -/// plain (exact) `Aggregate` top-k nodes over an exact input, unfiltered and -/// grouped by listed keys, grouped by equal input and partition. -pub(crate) fn topk_reuse_candidates(roots: &[QueryRoot]) -> Vec { - let mut nodes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for node in roots.iter().flat_map(QueryRoot::operator_refs) { - collect_children_first(node, &mut seen, &mut nodes); - } - let mut groups: Vec>> = Vec::new(); - for node in nodes.into_iter().filter(|n| exact_topk(n).is_some()) { - let same = |member: &Rc| { - let (Some((child, by, _)), Some((other, other_by, _))) = - (exact_topk(member), exact_topk(&node)) - else { - return false; - }; - by == other_by && (Rc::ptr_eq(child, other) || child == other) - }; - match groups.iter_mut().find(|g| same(&g[0])) { - Some(group) => group.push(node), - None => groups.push(vec![node]), - } - } - groups - .into_iter() - .filter_map(|members| { - let ks: Vec = members - .iter() - .filter_map(|m| exact_topk(m)) - .map(|t| t.2) - .collect(); - let largest = *ks.iter().max()?; - if ks.iter().all(|&k| k == largest) { - return None; - } - let (child, by, _) = exact_topk(&members[0])?; - let ranking = ranking(child, by, largest)?; - Some(TopKReuseCandidate { members, ranking }) - }) - .collect() -} - -/// The input, partition and `k` of an exact top-k over an exact input. -fn exact_topk(node: &OperatorNode) -> Option<(&Rc, &Reduction, usize)> { - let Some(NonASAPOp::Aggregate { - child, - reduction: reduction @ Reduction::Reduce(by), - measures, - filters, - having: None, - .. - }) = node.non_asap() - else { - return None; - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return None; - }; - let exact = child - .guarantee - .as_ref() - .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact); - (exact && !by.is_without() && filters.iter().all(Option::is_none)) - .then_some((child, reduction, *k)) -} - -/// The `k` highest-ranked rows of `child` per partition: `Limit` over a -/// descending `Sort` by its ranked column, as exact top-k runs. -fn ranking(child: &Rc, by: &Reduction, k: usize) -> Option> { - let Reduction::Reduce(keys) = by else { - return None; - }; - let value = asap_types::ir::schema::topk::ranked_column(child).ok()?; - let sorted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child: Rc::clone(child), - })) - .ok()?; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { - n: Some(k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - })) - .ok() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::schema::schema_support::with_promql_series_identity; - use asap_types::ir::schema::SketchAlgorithm; - - fn roots(queries: &[&str]) -> Vec { - queries - .iter() - .map(|q| { - let root = lower_promql(q, AccuracyTarget::exact()); - QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) - }) - .collect() - } - - /// The positions in `all` of the items of `chosen`, matched by the node - /// `node` gives for each. - fn positions(all: &[T], chosen: &[T], node: impl Fn(&T) -> &Rc) -> Vec { - chosen - .iter() - .map(|item| { - all.iter() - .position(|other| Rc::ptr_eq(node(other), node(item))) - .expect("a chosen item is listed") - }) - .collect() - } - - /// `sum by (job) (m)` and `max by (host, job) (m)` scan `m` with other - /// labels; widened to `job, host`, both scans project the same - /// columns, and each root keeps its columns. - #[test] - fn scans_of_one_metric_with_other_labels_share_once_widened() { - let roots = roots(&["sum by (job) (m)", "max by (host, job) (m)"]); - let widened = with_union_scan_labels(&roots).unwrap().unwrap(); - let scan_labels = |root: &QueryRoot| { - let QueryRoot::Operator(node) = root else { - unreachable!() - }; - OperatorNode::reachable(node) - .iter() - .find_map(|n| match n.non_asap() { - Some(NonASAPOp::Scan { schema, .. }) => Some( - schema - .fields - .iter() - .map(|f| f.name.clone()) - .collect::>(), - ), - _ => None, - }) - .unwrap() - }; - assert_eq!( - scan_labels(&widened[0]), - ["ts", "value", "job", "host", PROMQL_SERIES_IDENTITY] - ); - assert_eq!(scan_labels(&widened[0]), scan_labels(&widened[1])); - // The unique key still names the series identity and ts. - let QueryRoot::Operator(node) = &widened[0] else { - unreachable!() - }; - let scan = OperatorNode::reachable(node) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - assert_eq!(scan.schema.unique_keys, [vec![4, 0]]); - for (root, wide) in roots.iter().zip(&widened) { - let (QueryRoot::Operator(root), QueryRoot::Operator(wide)) = (root, wide) else { - unreachable!() - }; - assert_eq!(root.schema, wide.schema); - } - // Scans that already agree are left alone. - assert!(with_union_scan_labels(&widened).unwrap().is_none()); - } - - /// A count over a window, read from a Count-Min built in tumbling - /// tumbling windows, keeps the guarantee of one build over the window: Pass 1 - /// offers Count-Min for counts, and its merged counters add. - #[test] - fn a_count_from_count_min_tumbling_windows_keeps_its_guarantee() { - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let root = lower_promql( - "count_over_time(m[5m])", - asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator( - with_promql_series_identity(&root).unwrap(), - )]), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - *c == asap_types::stages::HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::Cms, - ) - }) - .expect("Count-Min is offered for a count") - }) - .collect(); - let roots: Vec = compose_logical_candidate(&inventory, &choice).unwrap(); - let candidates = tumbling_window_candidates(&roots, &[Some(60_000)]); - let pass2_candidates = window_composition_pass2_candidates(&roots, &candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate: {pass2_candidates:?}") - }; - let (QueryRoot::Operator(whole), QueryRoot::Operator(windowed)) = - (&roots[0], &pass2_candidate.roots.as_ref().unwrap()[0]) - else { - panic!("operator roots") - }; - assert!(whole.guarantee.is_some()); - assert_eq!(windowed.guarantee, whole.guarantee); - } - - /// The composed candidate of `queries` (each at its own ε, δ = 0.01) - /// in which every target chose `family`. - fn chose(queries: &[(&str, f64)], family: SketchAlgorithm) -> Vec { - use asap_types::stages::HowToComputeAggregateMeasure; - chose_where(queries, |c| { - *c == HowToComputeAggregateMeasure::Summary(family) - }) - } - - /// The composed candidate of `queries` (each at its own ε, δ = 0.01) - /// in which every target chose the first candidate `pick` accepts. - fn chose_where( - queries: &[(&str, f64)], - pick: impl Fn(&asap_types::stages::HowToComputeAggregateMeasure) -> bool, - ) -> Vec { - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let roots: Vec<_> = queries - .iter() - .map(|(q, epsilon)| { - let root = lower_promql( - q, - asap_types::ir::properties::AccuracyTarget::max_error(*epsilon) - .failure_probability(0.01) - .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), - ); - QueryRoot::Operator(with_promql_series_identity(&root).unwrap()) - }) - .collect(); - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(&pick) - .expect("the family is a candidate") - }) - .collect(); - compose_logical_candidate(&inventory, &choice).unwrap() - } - - /// The summary builds `roots` read, by address, with each one's KLL `k`. - fn kll_builds(roots: &[QueryRoot]) -> Vec<(*const OperatorNode, u32)> { - let mut out: Vec<(*const OperatorNode, u32)> = Vec::new(); - for node in roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - { - if let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = node.asap() - { - let asap_types::ir::schema::SketchParams::Kll { k, .. } = kind.params() else { - panic!("{kind:?}") - }; - if !out.iter().any(|(p, _)| *p == Rc::as_ptr(&node)) { - out.push((Rc::as_ptr(&node), *k)); - } - } - } - out - } - - fn root_bound(root: &QueryRoot) -> f64 { - let QueryRoot::Operator(node) = root else { - panic!("operator root") - }; - node.guarantee - .as_ref() - .and_then(|g| g.spec.error.evaluate()) - .expect("a constant bound") - } - - /// The tumbling-window candidates of `roots` as (tumbling-window width, tumbling-window count). - fn tumbling_windows(roots: &[QueryRoot], interval_ms: &[Option]) -> Vec<(u64, u32)> { - tumbling_window_candidates(roots, interval_ms) - .iter() - .map(|rule_candidate| { - ( - rule_candidate.tumbling_window_width_ms, - rule_candidate.tumbling_window_count, - ) - }) - .collect() - } - - /// A 5-min window evaluated every 2 min, 90 s or 1 min is built in tumbling windows - /// of gcd(window, interval). A once-per-window reader has one tumbling window, - /// which can reuse the newest tumbling window of a longer chain. - #[test] - fn tumbling_window_width_is_the_gcd_of_the_window_and_the_interval() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - assert_eq!(tumbling_windows(&roots, &[Some(120_000)]), [(60_000, 5)]); - assert_eq!(tumbling_windows(&roots, &[Some(90_000)]), [(30_000, 10)]); - assert_eq!(tumbling_windows(&roots, &[Some(60_000)]), [(60_000, 5)]); - assert_eq!(tumbling_windows(&roots, &[Some(300_000)]), [(300_000, 1)]); - assert!(tumbling_windows(&roots, &[None]).is_empty()); - } - - /// Past their caps, the tumbling-window rule and the reuse subsets name how many - /// candidates they found. - #[test] - fn too_many_tumbling_window_or_reuse_candidates_name_the_count_found() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - let tumbling_windows: Vec<_> = tumbling_window_candidates(&roots, &[Some(60_000)]) - .into_iter() - .cycle() - .take(MAX_TUMBLING_WINDOW_CANDIDATES + 1) - .collect(); - let error = window_composition_pass2_candidates(&roots, &tumbling_windows).unwrap_err(); - assert_eq!( - error.to_string(), - format!( - "found {} summary builds that could be built in tumbling windows, more than \ - {MAX_TUMBLING_WINDOW_CANDIDATES}", - MAX_TUMBLING_WINDOW_CANDIDATES + 1 - ) - ); - let error = subsets(&[0u8; MAX_REUSE_CANDIDATES + 1], "rollup").unwrap_err(); - assert_eq!( - error.to_string(), - format!( - "found {} rollup choices, more than {MAX_REUSE_CANDIDATES}", - MAX_REUSE_CANDIDATES + 1 - ) - ); - } - - /// A range read at an `@` anchor has no tumbling windows, however many shifts - /// stack above the anchored one: `m[5m] @ 100` with an extra - /// `offset 1m` shift between the range and the anchor is still anchored. - #[test] - fn an_anchor_below_stacked_shifts_has_no_tumbling_windows() { - use asap_types::ir::operator::TimeShift; - let roots = chose( - &[("quantile_over_time(0.99, m[5m] @ 100)", 0.01)], - SketchAlgorithm::Kll, - ); - let QueryRoot::Operator(root) = &roots[0] else { - panic!("operator root") - }; - let build = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a build"); - assert_eq!(splittable_window_ms(&build), None, "one anchored shift"); - // Put an `offset 1m` shift between the range and its anchored shift. - let restacked = build - .with_new_children(|range| { - Rc::new( - range - .with_new_children(|anchored| { - assert!(matches!( - anchored.non_asap(), - Some(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_some() - )); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 60_000, - at: None, - }, - child: anchored.clone(), - })) - .unwrap() - }) - .unwrap(), - ) - }) - .unwrap(); - assert_eq!( - splittable_window_ms(&restacked), - None, - "anchor below an offset" - ); - } - - /// Exact accumulators whose states merge get tumbling windows; IRate, which reads - /// the last two samples of the window, does not. - #[test] - fn exact_accumulators_that_merge_get_tumbling_windows() { - use asap_types::stages::HowToComputeAggregateMeasure; - let accumulator = |c: &HowToComputeAggregateMeasure| { - matches!(c, HowToComputeAggregateMeasure::ExactAccumulator(_)) - }; - for query in [ - "rate(m[5m])", - "increase(m[5m])", - "sum_over_time(m[5m])", - "max_over_time(m[5m])", - ] { - let roots = chose_where(&[(query, 0.01)], accumulator); - assert_eq!( - tumbling_windows(&roots, &[Some(60_000)]), - [(60_000, 5)], - "{query}" - ); - } - let roots = chose_where(&[("irate(m[5m])", 0.01)], accumulator); - assert!(tumbling_windows(&roots, &[Some(60_000)]).is_empty()); - } - - /// The rule's candidate reads a `SummaryMerge` over one `SummaryTumblingWindowChain` of - /// the build's family and size over the range's input, keeps the - /// build's output names, and keeps the whole build's guarantee (a KLL - /// merges into the state of one build over the union). - #[test] - fn tumbling_windows_replace_the_build_with_a_merged_chain() { - let roots = chose( - &[("quantile_over_time(0.99, m[5m])", 0.01)], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(60_000)]); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate: {pass2_candidates:?}") - }; - assert_eq!( - positions( - &rule_candidates, - &pass2_candidate.tumbling_window_candidates, - |c| &c.build - ), - [0] - ); - let tumbling_windows = pass2_candidate.roots.as_ref().unwrap(); - let QueryRoot::Operator(root) = &tumbling_windows[0] else { - panic!("operator root") - }; - root.validate_structure().unwrap(); - let QueryRoot::Operator(whole) = &roots[0] else { - panic!("operator root") - }; - assert_eq!(root.guarantee, whole.guarantee); - assert!(root.guarantee.is_some()); - let merge = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))) - .expect("a merge"); - let Some(ASAPOp::SummaryMerge { children, .. }) = merge.asap() else { - unreachable!() - }; - let [chain] = children.as_slice() else { - panic!("one chain") - }; - let Some(ASAPOp::SummaryTumblingWindowChain { - child, - family, - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, - .. - }) = chain.asap() - else { - panic!("{:?}", chain.operator) - }; - let Some(ASAPOp::SummaryAgg { - child: range, - family: whole_family, - .. - }) = rule_candidates[0].build.asap() - else { - unreachable!() - }; - assert_eq!(family, whole_family); - assert_eq!(child.operator, range.children()[0].operator); - assert_eq!(chain.schema, rule_candidates[0].build.schema); - assert_eq!(merge.coverage(), rule_candidates[0].build.coverage()); - } - - /// Two builds that could be split give three candidates: each alone and - /// both, in binary-counter order. Their windows (5 and 2 min) nest on - /// 1-min tumbling windows, so both together are also offered as one shared chain. - #[test] - fn every_non_empty_subset_of_the_tumbling_window_candidates_is_a_pass2_candidate() { - let roots = chose( - &[ - ("quantile_over_time(0.99, m[5m])", 0.01), - ("quantile_over_time(0.5, m[2m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(60_000), Some(60_000)]); - assert_eq!(rule_candidates.len(), 2); - let subsets: Vec<_> = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .iter() - .map(|v| { - assert!(v.roots.is_ok()); - ( - positions(&rule_candidates, &v.tumbling_window_candidates, |c| { - &c.build - }), - v.shared_chains.clone(), - ) - }) - .collect(); - assert_eq!( - subsets, - [ - (vec![0], vec![]), - (vec![1], vec![]), - (vec![0, 1], vec![]), - ( - vec![0, 1], - vec![SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5 - }] - ) - ] - ); - } - - /// The tumbling-window chains in `roots`, one per distinct node. - fn chains(roots: &[QueryRoot]) -> Vec> { - let mut found: Vec> = Vec::new(); - for root in roots { - for node in OperatorNode::reachable(operator(root)) { - if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - && !found.iter().any(|c| Rc::ptr_eq(c, &node)) - { - found.push(node); - } - } - } - found - } - - /// The `newest_tumbling_windows` of the one merge `root` reads. - fn newest_tumbling_windows(root: &QueryRoot) -> Option { - OperatorNode::reachable(operator(root)) - .into_iter() - .find_map(|n| match n.asap() { - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => Some(*newest_tumbling_windows), - _ => None, - }) - .expect("a merge") - } - - /// A 1-min and a 5-min KLL quantile over one metric, both every 10 s, - /// are 6 and 30 tumbling windows of 10 s; the 1-min window is the newest 6 of the - /// 5-min one. Besides each query's own chain, the rule offers one chain - /// of 30 tumbling windows read by both: the 5-min query merges every tumbling window, the - /// 1-min one only the newest 6, and each keeps the guarantee of its own - /// whole-window build (a KLL merges exactly). - #[test] - fn nested_windows_on_equal_tumbling_windows_share_one_chain() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[1m])", 0.01), - ("quantile_over_time(0.9, m[5m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(10_000), Some(10_000)]); - assert_eq!( - tumbling_windows(&roots, &[Some(10_000), Some(10_000)]), - [(10_000, 6), (10_000, 30)] - ); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared: Vec<_> = pass2_candidates - .iter() - .filter(|c| !c.shared_chains.is_empty()) - .collect(); - let [shared] = shared.as_slice() else { - panic!("one shared-chain candidate: {pass2_candidates:?}") - }; - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 10_000, - tumbling_window_count: 30 - }] - ); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert!(matches!( - chain.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 10_000, - tumbling_window_count: 30, - .. - }) - )); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), Some(6)); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - for (root, whole) in shared_roots.iter().zip(&roots) { - let root = operator(root); - root.validate_structure().unwrap(); - assert!(root.guarantee.is_some()); - assert_eq!(root.guarantee, operator(whole).guarantee); - } - // The per-query chains stay candidates: two chains, each read in full. - let separate = pass2_candidates - .iter() - .find(|c| c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty()) - .expect("both builds in their own tumbling windows"); - assert_eq!(chains(separate.roots.as_ref().unwrap()).len(), 2); - } - - /// A 15-min quantile every minute and the same quantile every 5 min - /// are 15 × 1-min and 3 × 5-min tumbling windows. The rule also offers one chain - /// at the finer width, 15 × 1 min, that both read in full, each at its - /// own interval; a 1-hour quantile every 5 min (12 × 5 min) reads the - /// newest 60 of one 60 × 1-min chain with them. - #[test] - fn queries_at_different_intervals_share_one_chain_at_the_finer_width() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(300_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(60_000, 15), (300_000, 3)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared = pass2_candidates - .iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 60_000, - tumbling_window_count: 15 - }] - ); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert!(matches!( - chain.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 15, - .. - }) - )); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), None); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - for (root, whole) in shared_roots.iter().zip(&roots) { - operator(root).validate_structure().unwrap(); - assert_eq!(operator(root).guarantee, operator(whole).guarantee); - } - - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[1h])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - let shared_roots = shared.roots.unwrap(); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), Some(15)); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), None); - assert!(matches!( - chains(&shared_roots)[0].asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 60, - .. - }) - )); - } - - /// A build whose tumbling windows the group's finest tumbling windows do not divide gets its - /// own chain without breaking the group: 20-min and 10-min quantiles - /// every 2 min (2-min tumbling windows) share one chain, and a 15-min quantile - /// every 3 min (3-min tumbling windows) stays apart. - #[test] - fn a_build_on_an_incompatible_grid_does_not_break_a_shared_chain() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[20m])", 0.01), - ("quantile_over_time(0.9, m[10m])", 0.01), - ("quantile_over_time(0.99, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(120_000), Some(120_000), Some(180_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(120_000, 10), (120_000, 5), (180_000, 5)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| c.tumbling_window_candidates.len() == 3 && !c.shared_chains.is_empty()) - .expect("a shared chain among all three builds in tumbling windows"); - assert_eq!( - shared.shared_chains, - [SharedTumblingWindowChain { - members: vec![0, 1], - tumbling_window_width_ms: 120_000, - tumbling_window_count: 10 - }] - ); - assert_eq!(chains(shared.roots.as_ref().unwrap()).len(), 2); - } - - /// The shared chains (width, count) of every Pass 2 candidate that - /// windowed all of `rule_candidates`. - fn shared_grids( - roots: &[QueryRoot], - rule_candidates: &[TumblingWindowCandidate], - ) -> Vec> { - window_composition_pass2_candidates(roots, rule_candidates) - .unwrap() - .into_iter() - .filter(|c| c.tumbling_window_candidates.len() == rule_candidates.len()) - .map(|c| { - c.shared_chains - .iter() - .map(|shared| { - ( - shared.tumbling_window_width_ms, - shared.tumbling_window_count, - ) - }) - .collect() - }) - .collect() - } - - /// Tumbling windows neither member's divide (3-min and 5-min tumbling windows of a 15-min - /// window) are shared at their gcd, 1 min, as one more candidate beside - /// the per-query chains; no member's own width serves both. - #[test] - fn a_shared_grid_at_the_gcd_of_the_members_widths_is_offered() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[15m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(180_000), Some(300_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(180_000, 5), (300_000, 3)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(60_000, 15)]] - ); - } - - /// A 15-min quantile every minute (15 × 1-min tumbling windows) and a 6-min one - /// every 90 s (4 × 90-s tumbling windows): one chain of 30-s tumbling windows over 15 min - /// serves both, the 6-min query merging the newest 12. It comes beside - /// the per-query chains, and Stage 3 prices which is cheaper. - #[test] - fn queries_at_60_and_90_seconds_share_one_chain_of_30_second_tumbling_windows() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[15m])", 0.01), - ("quantile_over_time(0.9, m[6m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(90_000)]; - assert_eq!( - tumbling_windows(&roots, &grids), - [(60_000, 15), (90_000, 4)] - ); - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(30_000, 30)]] - ); - let shared = window_composition_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .into_iter() - .find(|c| !c.shared_chains.is_empty()) - .unwrap(); - let shared_roots = shared.roots.unwrap(); - assert_eq!(newest_tumbling_windows(&shared_roots[0]), None); - assert_eq!(newest_tumbling_windows(&shared_roots[1]), Some(12)); - for (root, whole) in shared_roots.iter().zip(&roots) { - operator(root).validate_structure().unwrap(); - assert_eq!(operator(root).guarantee, operator(whole).guarantee); - } - } - - /// When the members' finest width divides the others, the gcd is that - /// width: one shared candidate, not two equal ones. With 1-min, 2-min - /// and 3-min tumbling windows, the divisible group (1 and 2 min, the 3-min build - /// joining at 1 min too) is already the gcd grid. - #[test] - fn the_gcd_grid_is_not_listed_twice_when_a_member_width_is_the_gcd() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[12m])", 0.01), - ("quantile_over_time(0.9, m[12m])", 0.01), - ("quantile_over_time(0.99, m[12m])", 0.01), - ], - SketchAlgorithm::Kll, - ); - let grids = [Some(60_000), Some(120_000), Some(180_000)]; - let rule_candidates = tumbling_window_candidates(&roots, &grids); - assert_eq!( - shared_grids(&roots, &rule_candidates), - [vec![], vec![(60_000, 12)]] - ); - } - - /// The shared chain is sized for its strictest reader - /// ([`size_for_all_readers`]): a 1-min quantile at ε = 0.01 and a - /// 5-min one at ε = 0.05 read one chain of the 1-min query's `k`. - #[test] - fn a_shared_chain_is_sized_for_its_strictest_reader() { - let roots = chose( - &[ - ("quantile_over_time(0.5, m[1m])", 0.01), - ("quantile_over_time(0.9, m[5m])", 0.05), - ], - SketchAlgorithm::Kll, - ); - let family = |node: &OperatorNode| match node.asap() { - Some( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => family.clone(), - _ => unreachable!(), - }; - let strict = family(&builds(&roots[0])[0]); - assert_ne!(strict, family(&builds(&roots[1])[0])); - let rule_candidates = tumbling_window_candidates(&roots, &[Some(10_000), Some(10_000)]); - let pass2_candidates = - window_composition_pass2_candidates(&roots, &rule_candidates).unwrap(); - let shared = pass2_candidates - .iter() - .find(|c| !c.shared_chains.is_empty()) - .expect("a shared-chain candidate"); - let shared_roots = shared.roots.as_ref().unwrap(); - let [chain] = chains(shared_roots) - .try_into() - .unwrap_or_else(|c: Vec<_>| panic!("one chain, found {}", c.len())); - assert_eq!(family(&chain), strict); - // The 1-min query keeps its own guarantee; the 5-min one reads the - // stricter state, so its guarantee is the 1-min build's size's, - // not its own looser one. - let guarantee = |root: &QueryRoot| operator(root).guarantee.clone().unwrap(); - assert_eq!(guarantee(&shared_roots[0]), guarantee(&roots[0])); - assert_ne!(guarantee(&shared_roots[1]), guarantee(&roots[1])); - assert_eq!( - guarantee(&shared_roots[1]).spec, - guarantee(&shared_roots[0]).spec - ); - } - /// The composed candidate of `queries` with every target at its exact - /// option (an exact accumulator, or plain exact execution). - fn exact(queries: &[&str]) -> Vec { - use asap_types::stages::HowToComputeAggregateMeasure as C; - let queries: Vec<(&str, f64)> = queries.iter().map(|q| (*q, 0.01)).collect(); - chose_where(&queries, |c| matches!(c, C::Exact | C::ExactAccumulator(_))) - } - - fn operator(root: &QueryRoot) -> &Rc { - let QueryRoot::Operator(node) = root else { - panic!("operator root") - }; - node - } - - fn builds(root: &QueryRoot) -> Vec> { - OperatorNode::reachable(operator(root)) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect() - } - - /// One coarse build's rollup candidate, computed alone, is the one - /// [`rollup_candidates`] lists for it, and needs a finer build to exist. - #[test] - fn one_coarse_builds_rollup_candidate_is_the_listed_one() { - let roots = exact(&[ - "sum by (job) (m)", - "sum by (job, instance) (m)", - "sum by (job, instance, zone) (m)", - ]); - let listed = rollup_candidates(&roots); - assert!(!listed.is_empty()); - for candidate in &listed { - let alone = rollup_candidate_for(&roots, Rc::as_ptr(&candidate.coarse)).unwrap(); - assert!(Rc::ptr_eq(&alone.finer, &candidate.finer)); - } - let coarse = &builds(&roots[0])[0]; - assert!(rollup_candidate_for(&roots[..1], Rc::as_ptr(coarse)).is_none()); - } - - /// `sum by (job)` is answered from `sum by (job, instance)`, which the - /// workload computes anyway: the coarse query reads the finer query's - /// finalized sums, summed by `job`, so it builds nothing over the raw - /// rows; both values stay exact and keep their columns. - #[test] - fn rollup_from_a_finer_exact_sum() { - let roots = exact(&["sum by (job) (m)", "sum by (job, instance) (m)"]); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(Rc::ptr_eq(&rule_candidate.coarse, &builds(&roots[0])[0])); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - let coarse = builds(&rolled[0]); - let fine = builds(&rolled[1]); - assert_eq!(fine.len(), 1); - // The finer build, then the sum over its values. - assert_eq!(coarse.len(), 2); - assert!(Rc::ptr_eq(&coarse[0], &fine[0]) || Rc::ptr_eq(&coarse[1], &fine[0])); - let rollup = coarse.iter().find(|b| !Rc::ptr_eq(b, &fine[0])).unwrap(); - let Some(ASAPOp::SummaryAgg { child, family, .. }) = rollup.asap() else { - unreachable!() - }; - assert!(matches!( - child.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )); - assert_eq!( - *family, - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum) - ); - for (before, after) in roots.iter().zip(rolled) { - let (before, after) = (operator(before), operator(after)); - after.validate_structure().unwrap(); - assert_eq!(before.schema, after.schema); - assert!(after - .guarantee - .as_ref() - .is_some_and(asap_types::ir::properties::AccuracyGuarantee::is_exact)); - } - } - - /// Identical per-series averages retain their values under label projection; - /// different windows, functions, and source predicates must stay separate. - #[test] - fn rollup_crosses_only_identical_per_series_reductions() { - let input = |query: &str| { - let root = lower_promql(query, asap_types::ir::properties::AccuracyTarget::exact()); - OperatorNode::reachable(&root) - .into_iter() - .find(|node| { - matches!( - node.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - .. - }) - ) - }) - .unwrap() - }; - let fine = input("sum by (job, instance) (avg_over_time(m[1m]))"); - assert!(same_rows( - &input("sum by (job) (avg_over_time(m[1m]))"), - &fine - )); - for coarse in [ - "sum by (job) (avg_over_time(m[2m]))", - "sum by (job) (max_over_time(m[1m]))", - "sum by (job) (avg_over_time(m{zone=\"east\"}[1m]))", - ] { - assert!(!same_rows(&input(coarse), &fine), "{coarse}"); - } - } - - /// Minima and maxima roll up as extrema; counts merge their integer states. - #[test] - fn extrema_and_integer_counts_roll_up() { - assert!(!rollup_candidates(&exact(&[ - "count by (job) (m)", - "count by (job, instance) (m)" - ])) - .is_empty()); - for (query, kind) in [ - ("min by (job) (m)", ExactKind::Min), - ("max by (job) (m)", ExactKind::Max), - ] { - let finer = query.replace("(job)", "(job, instance)"); - let roots = exact(&[query, &finer]); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("{query}: one candidate: {rule_candidates:?}") - }; - let mut value = Rc::clone(&rule_candidate.rollup); - if let Some(NonASAPOp::Project { child, .. }) = value.non_asap() { - value = Rc::clone(child); - } - let Some(ASAPOp::FinalizeExactAccumulator { child: state }) = value.asap() else { - panic!("a finalized value") - }; - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(rolled, _), - .. - }) = state.asap() - else { - panic!("an exact accumulator") - }; - assert_eq!(*rolled, kind, "{query}"); - let pass2_candidate = rollup_pass2_candidates(&roots, &rule_candidates) - .unwrap() - .remove(0); - assert_eq!( - operator(&pass2_candidate.roots.unwrap()[0]).schema, - operator(&roots[0]).schema, - "{query}" - ); - } - } - - /// The rule reuses only a finer computation the workload already has: - /// keys that are not a superset, another metric, window or measure, or - /// a lone query, give no tumbling-window candidate. - #[test] - fn rollup_needs_a_finer_computation_of_the_same_rows() { - for queries in [ - &["sum by (job) (m)"][..], - &["sum by (job) (m)", "sum by (instance) (m)"], - &["sum by (job) (m)", "sum by (job, instance) (n)"], - &["sum by (job) (m)", "max by (job, instance) (m)"], - &[ - "sum by (job) (sum_over_time(m[5m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - &["sum by (job) (m)", "sum by (job) (m)"], - &["sum by (job) (m)", "sum without (instance) (m)"], - &["sum without (job) (m)", "sum by (job, instance) (m)"], - ] { - assert!(rollup_candidates(&exact(queries)).is_empty(), "{queries:?}"); - } - } - - /// Rollup follows the family-capability table: sums, counts and - /// extrema merge exactly across groups; counter states merge only in - /// time, IRate not at all; a heap sketch's merged top-k has no - /// guarantee, and no sketch's states can be regrouped in the IR. - #[test] - fn rollup_is_refused_for_families_without_an_exact_merge_across_groups() { - use asap_types::ir::schema::{ExactParams as P, SketchParams}; - let exact = |kind, params| FieldDataType::ExactAggregate(kind, params); - assert_eq!( - rollup_kind(&exact(ExactKind::Sum, P::Sum)), - Some(ExactKind::Sum) - ); - assert_eq!( - rollup_kind(&exact(ExactKind::Count, P::Count)), - Some(ExactKind::Sum) - ); - for family in [ - exact(ExactKind::Rate, P::Rate), - exact(ExactKind::Increase, P::Increase), - exact(ExactKind::IRate, P::IRate), - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 5, - heap_size: 10, - }, - ), - GroupingStrategy::default(), - ), - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ), - ] { - assert_eq!(rollup_kind(&family), None, "{family:?}"); - } - } - - /// #840's `quantile by (host, job)` and `quantile by (job)`, both KLL, - /// the coarse one at a tighter ε: the coarse query reads the regrouping - /// merge of the finer KLL's states by `job`, so one KLL is built, sized - /// for its strictest reader (the larger `k`), and the coarse query keeps - /// a guarantee within its own target. - #[test] - fn a_coarse_sketch_rolls_up_from_the_finer_build() { - let roots = chose( - &[ - ("quantile by (host, job) (0.5, data)", 0.01), - ("quantile by (job) (0.5, data)", 0.001), - ], - SketchAlgorithm::Kll, - ); - let ks: Vec = kll_builds(&roots).into_iter().map(|(_, k)| k).collect(); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(Rc::ptr_eq(&rule_candidate.coarse, &builds(&roots[1])[0])); - assert!(Rc::ptr_eq(&rule_candidate.finer, &builds(&roots[0])[0])); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - // One KLL, at the larger k, read by both queries. - assert_eq!( - kll_builds(rolled) - .into_iter() - .map(|(_, k)| k) - .collect::>(), - [*ks.iter().max().unwrap()] - ); - let merge = OperatorNode::reachable(operator(&rolled[1])) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))) - .expect("the coarse query reads a rollup"); - let Some(ASAPOp::SummaryMerge { - children, group_by, .. - }) = merge.asap() - else { - unreachable!() - }; - assert_eq!(group_by, &Some(vec![1])); - assert!(Rc::ptr_eq(&children[0], &builds(&rolled[0])[0])); - for (before, after) in roots.iter().zip(rolled) { - let (before, after) = (operator(before), operator(after)); - after.validate_structure().unwrap(); - assert_eq!(before.schema, after.schema); - } - // Each query's guarantee is the shared KLL's, within its target. - assert!( - root_bound(&rolled[1]) <= 0.001, - "{}", - root_bound(&rolled[1]) - ); - assert!(root_bound(&rolled[0]) <= 0.001); - } - - /// Every coarser grouping of one finest build is one choice: two coarse - /// builds of a three-grouping chain give one Pass 2 candidate, in which - /// both read the finest build's states. A finer build that is itself - /// coarse is not a source. - #[test] - fn coarse_groupings_of_one_finest_build_roll_up_together() { - let roots = chose( - &[ - ("quantile by (job) (0.9, m)", 0.01), - ("quantile by (job, instance) (0.9, m)", 0.01), - ("quantile by (job, instance, zone) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = rollup_candidates(&roots); - assert_eq!(rule_candidates.len(), 2); - let finest = &builds(&roots[2])[0]; - assert!(rule_candidates - .iter() - .all(|candidate| Rc::ptr_eq(&candidate.finer, finest))); - assert_eq!(rollup_choices(&rule_candidates).len(), 1); - let pass2_candidates = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let [pass2_candidate] = pass2_candidates.as_slice() else { - panic!("one Pass 2 candidate") - }; - let rolled = pass2_candidate.roots.as_ref().unwrap(); - assert_eq!(kll_builds(rolled).len(), 1); - } - - /// A quantile over every series rolls up from the KLLs by job: a - /// rollup to no keys. - #[test] - fn a_global_quantile_rolls_up_to_no_keys() { - let roots = chose( - &[ - ("quantile(0.9, m)", 0.01), - ("quantile by (job) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - assert!(matches!( - rule_candidate.rollup.asap(), - Some(ASAPOp::SummaryMerge { group_by: Some(keys), .. }) if keys.is_empty() - )); - let pass2 = rollup_pass2_candidates(&roots, &rule_candidates).unwrap(); - let rolled = pass2[0].roots.as_ref().unwrap(); - assert_eq!(kll_builds(rolled).len(), 1); - operator(&rolled[0]).validate_structure().unwrap(); - } - - /// Groupings that are not nested: `by (job)` rolls up from `by (job, - /// instance)` or `by (job, zone)`; the source is the one whose sorted - /// key names come first, whatever the query order. - #[test] - fn the_rollup_source_does_not_depend_on_query_order() { - for queries in [ - [ - "quantile by (job) (0.9, m)", - "quantile by (job, zone) (0.9, m)", - "quantile by (job, instance) (0.9, m)", - ], - [ - "quantile by (job) (0.9, m)", - "quantile by (job, instance) (0.9, m)", - "quantile by (job, zone) (0.9, m)", - ], - ] { - let roots = chose(&queries.map(|q| (q, 0.01)), SketchAlgorithm::Kll); - let rule_candidates = rollup_candidates(&roots); - let [rule_candidate] = rule_candidates.as_slice() else { - panic!("one candidate: {rule_candidates:?}") - }; - let instance = queries.iter().position(|q| q.contains("instance")).unwrap(); - assert!(Rc::ptr_eq( - &rule_candidate.finer, - &builds(&roots[instance])[0] - )); - } - } - - /// A rollup needs the same rows: a coarse KLL over another metric is - /// none. - #[test] - fn sketch_rollup_needs_the_same_rows() { - let other_metric = chose( - &[ - ("quantile by (job) (0.9, n)", 0.01), - ("quantile by (job, instance) (0.9, m)", 0.01), - ], - SketchAlgorithm::Kll, - ); - assert!(rollup_candidates(&other_metric).is_empty()); - } - - /// Top-k over other partitions or inputs, or of one `k`, share no - /// ranking; a top-k a heap sketch answers is not reused. - #[test] - fn topk_reuse_needs_exact_topk_of_one_input_and_partition() { - for queries in [ - &[ - "topk(5, sum by (job) (m))", - "topk by (job) (10, sum by (job, instance) (m))", - ][..], - &["topk(5, sum by (job) (m))", "topk(10, sum by (job) (n))"], - &["topk(5, sum by (job) (m))", "topk(5, max by (job) (m))"], - &[ - "topk(5, sum by (job) (m))", - "topk by (job) (10, sum by (job) (m))", - ], - ] { - assert!( - topk_reuse_candidates(&exact(queries)).is_empty(), - "{queries:?}" - ); - } - use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure as C; - let inventory = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups( - &[ - "topk by (job) (5, sum_over_time(m[1m]))", - "topk by (job) (10, sum_over_time(m[1m]))", - ] - .iter() - .map(|q| { - let root = lower_promql( - q, - asap_types::ir::properties::AccuracyTarget::max_error(0.01) - .failure_probability(0.01), - ); - QueryRoot::Operator(root) - }) - .collect::>(), - ), - ) - .unwrap(); - // The CountSketch heap, the top-k's fused whole-expression one. - let heap = |c: &C| matches!(c, C::Summary(SketchAlgorithm::CountSketchWithHeap)); - let choice: Vec = inventory - .targets - .iter() - .map(|t| { - (0..t.choice_count()) - .position(|i| t.computation_for(i).is_some_and(|(c, _)| heap(c))) - .unwrap_or(0) - }) - .collect(); - let heaps: Vec = compose_logical_candidate(&inventory, &choice).unwrap(); - assert!(OperatorNode::reachable(operator(&heaps[0])) - .iter() - .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. })))); - assert!(topk_reuse_candidates(&heaps).is_empty()); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs b/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs deleted file mode 100644 index df857c35c..000000000 --- a/crates/logical-asap-optimization/src/pass2/finer_aggregate.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Whether a query aggregate can be computed from a finer aggregate over the -//! same source and window, and with what on top (view matching for -//! aggregation queries, as in *Optimizing Queries Using Materialized Views: -//! A Practical, Scalable Solution*, SIGMOD 2001, §3.3, and Apache Calcite's -//! `SubstitutionVisitor.unifyAggregates`). -//! -//! The finer result holds at least the rows the query needs. A -//! *compensating* operation closes the gap: a compensating filter removes -//! the extra rows, and a rollup merges finer groups into the query's. A -//! filter that reads only finer group-by columns is constant within a finer -//! group, so filtering finer groups equals filtering input rows. -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; - -use asap_accuracy::propagation::{pinned_column, split_and}; - -/// How a query aggregate is computed from a finer one: keep the finer rows -/// that pass `compensating_filter`, then, when `needs_rollup`, merge them -/// into the query's groups. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FinerAggregateMatch { - /// The query's conjuncts that the finer aggregate lacks, over the finer - /// group-by (column `i` is the finer aggregate's `i`-th key); `None` - /// when the finer aggregate has every conjunct. - pub compensating_filter: Option, - /// Each query group key, as an index into the finer group-by. - pub query_keys: Vec, - /// Whether finer groups merge into query groups: a finer key the query - /// drops is not pinned by an equality in `compensating_filter`. When - /// `false`, each query group is at most one finer row. - pub needs_rollup: bool, -} - -/// How a query aggregate over `query_rows` grouped by `query_keys` is -/// computed from a finer aggregate over `finer_rows` grouped by -/// `finer_keys` (keys are columns of the rows); `None` when it cannot be. -/// -/// Both rows must be the same scan (source and schema) under the same time -/// ranges; the finer conjuncts (scan predicates and filters) must be among -/// the query's; every other query conjunct must read only finer group-by -/// columns; and the query keys must be finer keys. -pub(crate) fn match_finer( - query_rows: &OperatorNode, - query_keys: &[usize], - finer_rows: &OperatorNode, - finer_keys: &[usize], -) -> Option { - let query = Rows::of(query_rows)?; - let finer = Rows::of(finer_rows)?; - if query.spine != finer.spine { - return None; - } - let mut extra = query.conjuncts; - for conjunct in finer.conjuncts { - let position = extra.iter().position(|other| *other == conjunct)?; - extra.remove(position); - } - let finer_key = |column: usize| finer_keys.iter().position(|&key| key == column); - let query_key_positions = query_keys - .iter() - .map(|&key| finer_key(key)) - .collect::>>()?; - if !extra.iter().all(|conjunct| { - conjunct.operator_refs().is_empty() - && conjunct - .columns_referenced() - .into_iter() - .all(|column| finer_key(column).is_some()) - }) { - return None; - } - let pinned: Vec = extra.iter().filter_map(pinned_column).collect(); - let needs_rollup = finer_keys - .iter() - .any(|key| !query_keys.contains(key) && !pinned.contains(key)); - // Without extra conjuncts or a rollup, the two aggregates have the same - // groups, perhaps in another key order: they are one computation, not - // one read from the other (each read from the other would be a cycle). - if extra.is_empty() && !needs_rollup { - return None; - } - let compensating_filter = (!extra.is_empty()).then(|| { - let remapped: Vec = extra - .iter() - .map(|conjunct| conjunct.map_columns(&mut |column| finer_key(column).unwrap())) - .collect(); - Predicate(match <[ScalarExpr; 1]>::try_from(remapped) { - Ok([one]) => one, - Err(many) => ScalarExpr::BoolAnd(many), - }) - }); - Some(FinerAggregateMatch { - compensating_filter, - query_keys: query_key_positions, - needs_rollup, - }) -} - -/// An aggregate's input rows: the scan and time ranges they come from, and -/// the conjuncts that select them. -struct Rows { - /// Each time range above the scan, outermost first, then the scan - /// without its predicates. - spine: Vec>, - /// Scan predicates and filter predicates, split at `AND`. - conjuncts: Vec, -} - -impl Rows { - /// The rows of `node`, a chain of `TimeRange`s and `Filter`s over a - /// `Scan`, none changing the scan's columns; `None` for anything else. - fn of(node: &OperatorNode) -> Option { - let mut rows = Rows { - spine: Vec::new(), - conjuncts: Vec::new(), - }; - let mut node = node; - loop { - match node.non_asap()? { - NonASAPOp::TimeRange { child, .. } if child.schema == node.schema => { - rows.spine.push(node.operator.map_children(|_| ())); - node = child; - } - NonASAPOp::Filter { pred, child } if child.schema == node.schema => { - split_and(&pred.0, &mut rows.conjuncts); - node = child; - } - NonASAPOp::Scan { - source, - schema, - predicates, - } => { - for predicate in predicates { - split_and(&predicate.0, &mut rows.conjuncts); - } - rows.spine.push(Operator::NonASAP(NonASAPOp::Scan { - source: source.clone(), - schema: schema.clone(), - predicates: Vec::new(), - })); - return Some(rows); - } - _ => return None, - } - } - } -} diff --git a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs index 678ba5c0d..02fa8f6c5 100644 --- a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs @@ -1,71 +1,53 @@ -//! Stage 1, Pass 2's entry and output: the Pass 1 memo extended with the -//! candidates and groups that share work across computations -//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 implements. +//! Stage 1, Pass 2's entry and output: the Pass 1 memo extended by the +//! rules over groups, and the logical edges +//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 expands. use crate::pass2::memo::rules::{ - finer_aggregate, rollup, scan_labels, shared_configuration_readers, summary_inputs, - summary_results, topk, -}; -use crate::pass2::memo::state_groups::{ - add_shared_summary_state_groups, check_unexpanded_memo, Pass2RuleError, + finer_aggregate, rollup, scan_labels, summary_inputs, topk, Pass2RuleError, }; use crate::pass2::memo::window_candidates::add_local_window_composition; -use asap_types::stages::Memo; -use asap_types::stages::{LogicalASAPMemoCandidates, PerComputationLogicalMemoCandidates}; +use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; use asap_types::workload::QueryWorkloadEntry; -/// Stage 1, Pass 2: adds to Pass 1's memo the local tumbling-window -/// candidates for every legal tumbling-window width (rule 1), then runs -/// rules 2–9 and `add_shared_summary_state_groups`. +/// Stage 1, Pass 2: rule 1 ([`pass2_local_window_composition`]), then rules +/// 2–7 and the logical edges ([`pass2_after_local_window_composition`]). /// `entries[i]` is query `i`'s workload entry. pub fn generate_logical_asap_memo_candidates( - pass1: PerComputationLogicalMemoCandidates, + pass1: Memo, entries: &[QueryWorkloadEntry], ) -> Result { - pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)) + pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?) } -/// Pass 2's rule 1 alone: Pass 1's memo with the local tumbling-window -/// candidates for every legal width (`asap_planner` prints this step). +/// Pass 2's rule 1 alone: Pass 1's memo with the tumbling-window layouts +/// of each summary state group (`asap_planner` prints this step). pub fn pass2_local_window_composition( - pass1: PerComputationLogicalMemoCandidates, + pass1: Memo, entries: &[QueryWorkloadEntry], -) -> Memo { - let mut memo = pass1.memo; +) -> Result { + let mut memo = pass1; // 1. Local window composition. - add_local_window_composition(&mut memo, entries); - memo + add_local_window_composition(&mut memo, entries)?; + Ok(memo) } /// The rest of Pass 2 on [`pass2_local_window_composition`]'s memo: rules -/// 2–9 and `add_shared_summary_state_groups`. +/// 2–7, then the logical edges with their pruning caps. pub fn pass2_after_local_window_composition( mut memo: Memo, ) -> Result { - apply_sharing_rules(&mut memo)?; - // 10. One group per shared summary build, read by its reader groups. - add_shared_summary_state_groups(&mut memo)?; - Ok(LogicalASAPMemoCandidates { memo }) -} - -/// Pass 2's rules 2–9, in order, on a memo after local window composition. -pub(crate) fn apply_sharing_rules(memo: &mut Memo) -> Result<(), Pass2RuleError> { - check_unexpanded_memo(memo)?; // 2. Scan widening. - scan_labels::add_scan_label_candidates(memo)?; + scan_labels::add_scan_label_candidates(&mut memo)?; // 3. Summary inputs. - summary_inputs::factor_summary_inputs(memo)?; + summary_inputs::factor_summary_inputs(&mut memo)?; // 4. Top-k reuse. - topk::add_topk_candidates(memo)?; + topk::add_topk_candidates(&mut memo)?; // 5. Finer aggregate. - finer_aggregate::add_finer_aggregate_candidates(memo)?; + finer_aggregate::add_finer_aggregate_candidates(&mut memo)?; // 6. Rollup. - rollup::add_rollup_candidates(memo)?; + rollup::add_rollup_candidates(&mut memo)?; // 7. Window rollup. - rollup::add_window_rollup_candidates(memo)?; - // 8. Summary results. - summary_results::factor_summary_results(memo)?; - // 9. Shared configurations. - shared_configuration_readers::add_state_configuration_candidates(memo)?; - Ok(()) + rollup::add_window_rollup_candidates(&mut memo)?; + let _ = memo; + todo!("#1207 PR 4: the logical edges, each input's floor tightened by its pruning cap") } diff --git a/crates/logical-asap-optimization/src/pass2/memo.rs b/crates/logical-asap-optimization/src/pass2/memo.rs index c9bc66a80..754a06a5c 100644 --- a/crates/logical-asap-optimization/src/pass2/memo.rs +++ b/crates/logical-asap-optimization/src/pass2/memo.rs @@ -1,12 +1,6 @@ -//! The logical memo's candidates, rules and enumeration, over the memo -//! groups Stage 0 builds (`crate::stage0`). +//! The logical memo's rules over the memo groups Stage 0 builds +//! (`crate::stage0`). -/// The candidate DAGs a memo stands for: choices and their composition. -pub(crate) mod enumeration; pub(crate) mod rules; -/// Shared finite state configuration enumeration. -pub(crate) mod shared_configurations; -/// Concrete summary-state groups exposed by sharing rules. -pub(crate) mod state_groups; -/// Local tumbling-window candidates with selected-reader grid configurations. +/// Rule 1: tumbling-window layouts of a summary state group. pub(crate) mod window_candidates; diff --git a/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs b/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs deleted file mode 100644 index d849a5e75..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/enumeration.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! The candidate DAGs a memo stands for: each choice of one implementation -//! candidate per needed group, and the complete DAG that choice composes. -//! -//! A group is needed when a query's root is in it or a chosen candidate reads -//! it. A group read by several chosen candidates takes one candidate, so its -//! nodes are built once and shared; which groups are needed depends on the -//! candidates chosen above them. The count is therefore not a product over -//! groups. -//! -//! A choice is counted, not the DAG it composes: two candidates of a group -//! that differ only in what recomposing guarantees re-derives compose equal -//! DAGs. Composing checks only that a choice composes; input contracts, -//! accuracy at the roots, Stage 2 legality and Stage 3 cost stay with the -//! validator and the later stages. The memo's groups are acyclic, as every -//! memo step builds them, but candidates may read each other (a rollup of -//! a finer grouping whose candidate reads the coarser one): a choice taking -//! such candidates together composes no DAG and is not enumerated. - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{OperatorNode, QueryRoot, SchemaDerivationError}; - -use super::rules::reuse::replace; -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; - -/// One candidate (an index into `Memo::candidates[group]`) for each -/// group it names. [`build_logical_asap_dags_from_chosen_candidates`] gives a group it does not name its first -/// candidate. -pub type ChosenCandidatePerMemoGroup = BTreeMap; - -/// The choices [`enumerate`] found, in depth-first order. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AllChosenCandidatesPerMemoGroup { - /// Each choice names exactly the groups it needs. - pub all_chosen_candidates: Vec, - /// False when the limit stopped the enumeration before every choice was found. - pub complete: bool, -} - -/// A choice that does not compose. -#[derive(Debug, thiserror::Error)] -pub enum ErrorBuildingLogicalASAPDagsFromChosenCandidates { - #[error("group {group:?} has no candidate {candidate}")] - NoSuchCandidate { - group: MemoGroupId, - candidate: usize, - }, - #[error("group {group:?}'s candidate {candidate} reads group {child:?} but has no node of it to replace")] - ChildNotInCandidate { - group: MemoGroupId, - candidate: usize, - child: MemoGroupId, - }, - /// The chosen candidates read each other in a cycle (e.g. a coarse - /// group's rollup reads a finer group whose chosen candidate reads it). - #[error("group {group:?} reads itself through the chosen candidates")] - Cycle { group: MemoGroupId }, - /// Re-deriving the composed nodes' schemas and guarantees failed. - #[error("recomposing the composed DAG: {0}")] - Recompose(#[from] SchemaDerivationError), -} - -/// Every choice of one candidate per needed group, starting from the query -/// roots, stopping after `limit` choices. A choice whose candidates read -/// each other in a cycle composes no DAG and is left out. -pub fn enumerate(memo: &Memo, limit: usize) -> AllChosenCandidatesPerMemoGroup { - fn walk( - memo: &Memo, - mut pending: Vec, - chosen_candidates: &mut ChosenCandidatePerMemoGroup, - limit: usize, - all_chosen_candidates: &mut Vec, - ) -> bool { - let group = loop { - match pending.pop() { - None => { - if all_chosen_candidates.len() == limit { - return false; - } - all_chosen_candidates.push(chosen_candidates.clone()); - return true; - } - Some(group) if !chosen_candidates.contains_key(&group) => break group, - Some(_) => {} - } - }; - for (index, candidate) in memo.candidates[group.0 as usize].iter().enumerate() { - // A candidate reading a group that already reads this one closes - // a cycle; every edge is checked as its group is chosen. - if candidate - .input_groups - .iter() - .any(|&child| reaches(memo, chosen_candidates, child, group)) - { - continue; - } - chosen_candidates.insert(group, index); - let mut next = pending.clone(); - next.extend(candidate.input_groups.iter().rev()); - let finished = walk(memo, next, chosen_candidates, limit, all_chosen_candidates); - chosen_candidates.remove(&group); - if !finished { - return false; - } - } - true - } - let roots = root_groups(memo).rev().collect(); - let mut all_chosen_candidates = Vec::new(); - let complete = walk( - memo, - roots, - &mut ChosenCandidatePerMemoGroup::new(), - limit, - &mut all_chosen_candidates, - ); - AllChosenCandidatesPerMemoGroup { - all_chosen_candidates, - complete, - } -} - -/// Each root group (one with a query's root) and its DAG under `chosen_candidates`: -/// each needed group implemented by its chosen candidate over its children's -/// composed DAGs, a group read several times composed once and shared. Every -/// node is then rebuilt with its derived schema and composed guarantee, as -/// in plan reconstruction. A scalar query's expression is not rebuilt: its -/// operator roots are root groups like any other query's. -pub fn build_logical_asap_dags_from_chosen_candidates( - memo: &Memo, - chosen_candidates: &ChosenCandidatePerMemoGroup, -) -> Result)>, ErrorBuildingLogicalASAPDagsFromChosenCandidates> -{ - fn composed( - memo: &Memo, - group: MemoGroupId, - chosen_candidates: &ChosenCandidatePerMemoGroup, - done: &mut HashMap>, - composing: &mut HashSet, - ) -> Result, ErrorBuildingLogicalASAPDagsFromChosenCandidates> { - if let Some(found) = done.get(&group) { - return Ok(Rc::clone(found)); - } - if !composing.insert(group) { - return Err(ErrorBuildingLogicalASAPDagsFromChosenCandidates::Cycle { group }); - } - let index = chosen_candidates.get(&group).copied().unwrap_or(0); - let candidate = memo.candidates[group.0 as usize].get(index).ok_or( - ErrorBuildingLogicalASAPDagsFromChosenCandidates::NoSuchCandidate { - group, - candidate: index, - }, - )?; - let own_nodes: HashSet<_> = OperatorNode::reachable(&candidate.root_node) - .iter() - .map(Rc::as_ptr) - .collect(); - let mut replacements = HashMap::new(); - for &child in &candidate.input_groups { - let old = &memo.groups[child.0 as usize].logical_node; - if !own_nodes.contains(&Rc::as_ptr(old)) { - return Err( - ErrorBuildingLogicalASAPDagsFromChosenCandidates::ChildNotInCandidate { - group, - candidate: index, - child, - }, - ); - } - replacements.insert( - Rc::as_ptr(old), - composed(memo, child, chosen_candidates, done, composing)?, - ); - } - let node = replace(&candidate.root_node, &replacements); - done.insert(group, Rc::clone(&node)); - Ok(node) - } - let mut done = HashMap::new(); - let mut composing = HashSet::new(); - let (groups, roots): (Vec<_>, Vec<_>) = root_groups(memo) - .map(|group| { - Ok(( - group, - QueryRoot::Operator(composed( - memo, - group, - chosen_candidates, - &mut done, - &mut composing, - )?), - )) - }) - .collect::, ErrorBuildingLogicalASAPDagsFromChosenCandidates>>()? - .into_iter() - .unzip(); - let roots = recompose_guarantees(&roots)?; - Ok(groups - .into_iter() - .zip(roots) - .map(|(group, root)| match root { - QueryRoot::Operator(node) => (group, node), - QueryRoot::Scalar(_) => unreachable!("composed from operator roots"), - }) - .collect()) -} - -/// Whether `from` is `to` or reads it through the candidates `chosen_candidates` names. -fn reaches( - memo: &Memo, - chosen_candidates: &ChosenCandidatePerMemoGroup, - from: MemoGroupId, - to: MemoGroupId, -) -> bool { - let mut pending = vec![from]; - let mut seen = HashSet::new(); - while let Some(group) = pending.pop() { - if group == to { - return true; - } - if !seen.insert(group) { - continue; - } - if let Some(&index) = chosen_candidates.get(&group) { - pending.extend(&memo.candidates[group.0 as usize][index].input_groups); - } - } - false -} - -fn root_groups(memo: &Memo) -> impl DoubleEndedIterator { - asap_types::stages::root_groups(&memo.query_roots).into_iter() -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules.rs b/crates/logical-asap-optimization/src/pass2/memo/rules.rs index e8eec23f0..c9b38ed85 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules.rs @@ -1,11 +1,19 @@ -//! Bounded logical memo rule expansion. Rules expose computations and typed -//! dependencies; physical configuration and cost selection live downstream. +//! Pass 2's rules over groups (concepts doc, section 3): each reads the +//! memo and adds groups (`Memo::find_or_add_group`) or candidates +//! (`Memo::add_candidate`); none copies a consumer. pub(crate) mod finer_aggregate; -pub(super) mod reuse; pub(crate) mod rollup; pub(crate) mod scan_labels; -pub(crate) mod shared_configuration_readers; -pub(super) mod shared_producer; pub(crate) mod summary_inputs; -pub(crate) mod summary_results; pub(crate) mod topk; + +/// Why a Pass 2 rule failed on a memo. +#[derive(Debug, thiserror::Error)] +pub enum Pass2RuleError { + #[error("memo contains an out-of-range group reference")] + InvalidGroupReference, + #[error("memo group capacity exceeds the group identifier range")] + TooManyGroups, + #[error("memo dependencies contain a cycle")] + CyclicDependencies, +} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs index 78aee9654..95c8c7c0a 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs @@ -1,177 +1,12 @@ //! Read a SQL aggregate from a finer aggregate's result //! ([`match_finer`](crate::pass2::finer_aggregate::match_finer)): filter the finer //! rows, then merge them into the query's groups with each measure's rollup. -use super::reuse::append_reader; -use crate::pass2::finer_aggregate::match_finer; -use crate::pass2::memo::state_groups::Pass2RuleError; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, OperatorResultKind}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -/// A memo group's SQL aggregate: no `without`, measure filter or `HAVING`. -struct Aggregate<'a> { - keys: &'a [usize], - measures: &'a [AggIntent], - output_names: &'a [String], - rows: &'a Rc, -} - -impl<'a> Aggregate<'a> { - fn of(node: &'a OperatorNode) -> Option { - if node.result_kind != OperatorResultKind::Relation { - return None; - } - let NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - output_names, - filters, - having: None, - child, - } = node.non_asap()? - else { - return None; - }; - // A same-level read takes the one row of each group, so a measure - // that gives several rows per group (a top-k, a count of values) - // does not qualify. - let one_row_per_group = !measures.iter().any(|measure| { - matches!( - measure, - AggIntent::TopK { .. } | AggIntent::CountValues { .. } - ) - }); - (!by.is_without() && filters.iter().all(Option::is_none) && one_row_per_group).then_some( - Self { - keys: by.keys(), - measures, - output_names, - rows: child, - }, - ) - } -} - -/// The measure that merges finer values of `measure` (at `col` of the finer -/// result) into a coarser group (Calcite's `SqlAggFunction.getRollup`); -/// `None` when its values do not roll up. A count rolls up as a sum only -/// into grouped queries: a global count of no rows is 0, a sum NULL. -fn value_rollup(measure: &AggIntent, col: usize, global: bool) -> Option { - match measure { - AggIntent::Sum { .. } => Some(AggIntent::Sum { col: Some(col) }), - AggIntent::Count { .. } if !global => Some(AggIntent::Sum { col: Some(col) }), - AggIntent::Min { .. } => Some(AggIntent::Min { col: Some(col) }), - AggIntent::Max { .. } => Some(AggIntent::Max { col: Some(col) }), - _ => None, - } -} - -pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result { - // Only aggregates of equal measures can match; bucket by them (the - // first levels of the paper's filter tree) instead of trying all pairs. - let mut buckets: HashMap> = HashMap::new(); - for (group, entry) in memo.groups.iter().enumerate() { - if let Some(aggregate) = Aggregate::of(&entry.logical_node) { - buckets - .entry(format!("{:?}", aggregate.measures)) - .or_default() - .push(group); - } - } - let mut added = 0; - for groups in buckets.values() { - for &owner in groups { - for &finer in groups { - if owner != finer { - added += add_read(memo, owner, finer)?; - } - } - } - } - Ok(added) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -/// Adds to group `owner` a read of group `finer`'s result when it computes -/// `owner`'s aggregate; the number of candidates added (0 or 1). -fn add_read(memo: &mut Memo, owner: usize, finer: usize) -> Result { - let query_node = Rc::clone(&memo.groups[owner].logical_node); - let finer_node = Rc::clone(&memo.groups[finer].logical_node); - let (Some(query), Some(fine)) = (Aggregate::of(&query_node), Aggregate::of(&finer_node)) else { - return Ok(0); - }; - let Some(matched) = match_finer(query.rows, query.keys, fine.rows, fine.keys) else { - return Ok(0); - }; - let global = query.keys.is_empty(); - // A finer measure's value column follows the finer keys. When each query - // group is at most one finer row, `MIN` reads that row's value, and over - // no row gives a global query SQL's one NULL row. - let measures = query - .measures - .iter() - .enumerate() - .map(|(index, measure)| { - let col = fine.keys.len() + index; - if !matched.needs_rollup && !(global && matches!(measure, AggIntent::Count { .. })) { - Some(AggIntent::Min { col: Some(col) }) - } else if matched.needs_rollup { - value_rollup(measure, col, global) - } else { - None - } - }) - .collect::>>(); - let Some(measures) = measures else { - return Ok(0); - }; - let reuse_error = - |error: asap_types::ir::SchemaDerivationError| Pass2RuleError::ReuseRule(error.to_string()); - let rows = match matched.compensating_filter { - Some(pred) => OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: Rc::clone(&finer_node), - })) - .map_err(reuse_error)?, - None => Rc::clone(&finer_node), - }; - let reader = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(matched.query_keys), - measures, - output_names: query.output_names.to_vec(), - filters: vec![], - having: None, - child: rows, - })) - .map_err(reuse_error)?; - let Some(original) = memo.candidates[owner] - .iter() - .find(|c| c.origin == OriginOfCandidateForMemoGroup::Pass1) - .cloned() - else { - return Ok(0); - }; - // The reader's columns take the query's qualifiers. A rollup whose type - // or nullability differs from the query's (a sum of counts) is not - // offered. - let mut schema = reader.schema.clone(); - for (field, expected) in schema - .fields - .iter_mut() - .zip(&original.root_node.schema.fields) - { - field.table = expected.table.clone(); - } - if schema != original.root_node.schema || reader.result_kind != original.root_node.result_kind { - return Ok(0); - } - let reader = Rc::new(OperatorNode::with_schema(reader.operator.clone(), schema)); - Ok(usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::Rollup, - )?)) +/// Rule 5, finer aggregate: a candidate of a coarse group that filters and rolls up a finer group. +pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 5 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs deleted file mode 100644 index 5eac3feb1..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/reuse.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Shared boundary handling for producer-level reuse rules. Rule outputs retain -//! canonical input identities after guarantee recomposition rebuilds their DAGs. -use crate::pass2::memo::state_groups::{boundary_groups, Pass2RuleError}; -use asap_types::ir::OperatorNode; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -pub(in crate::pass2::memo) fn eligible(candidate: &CandidateForMemoGroup) -> bool { - candidate.accuracy_budget_parts > 0 -} - -pub(in crate::pass2::memo) fn restore_inputs( - node: &Rc, - memo: &Memo, - inputs: &[MemoGroupId], -) -> Rc { - // These are the same declared logical input groups, rebound after a rule - // cloned the DAG. Value guarantees are recomposed from selected producers - // during reconstruction; state inputs additionally require the same error - // contract because the intermediate state's guarantee is part of its API. - let canonical: Vec<_> = inputs - .iter() - .flat_map(|id| OperatorNode::reachable(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn visit( - node: &Rc, - canonical: &[Rc], - done: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(found) = done.get(&Rc::as_ptr(node)) { - return Rc::clone(found); - } - let operator = node - .operator - .map_children(|child| visit(child, canonical, done)); - let rebuilt = Rc::new(OperatorNode { - operator, - ..(**node).clone() - }); - let result = canonical - .iter() - .find(|known| { - same_node( - known, - &rebuilt, - known.result_kind == asap_types::ir::OperatorResultKind::State, - ) - }) - .cloned() - .unwrap_or(rebuilt); - done.insert(Rc::as_ptr(node), Rc::clone(&result)); - result - } - visit(node, &canonical, &mut HashMap::new()) -} - -pub(in crate::pass2::memo) fn same_node( - left: &OperatorNode, - right: &OperatorNode, - guarantees: bool, -) -> bool { - left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) - && left.schema == right.schema - && left.result_kind == right.result_kind - && left.timing == right.timing - && left.materialization == right.materialization - && left.children().len() == right.children().len() - && left - .children() - .iter() - .zip(right.children()) - .all(|(a, b)| Rc::ptr_eq(a, b)) - && serde_json::to_string(&( - left.operator.map_children(|_| ()), - &left.schema, - left.result_kind, - left.timing, - left.materialization, - )) - .ok() - == serde_json::to_string(&( - right.operator.map_children(|_| ()), - &right.schema, - right.result_kind, - right.timing, - right.materialization, - )) - .ok() - && (!guarantees || left.guarantee == right.guarantee) -} - -pub(in crate::pass2::memo) fn append_producer( - memo: &mut Memo, - node: Rc, - queries: &[usize], - origin: OriginOfCandidateForMemoGroup, - accuracy_budget_parts: usize, -) -> Result { - // Reuse only dedicated producer groups. A logical result group can contain - // other implementations and cannot enforce this producer's exact contract. - let root_groups = asap_types::stages::root_groups(&memo.query_roots); - if let Some((index, _)) = memo.groups.iter().enumerate().find(|(index, group)| { - !root_groups.contains(&MemoGroupId(*index as u32)) - && memo.candidates[*index].len() == 1 - && memo.candidates[*index][0].origin == origin - && memo.candidates[*index][0].accuracy_budget_parts == accuracy_budget_parts - && same_node(&group.logical_node, &node, true) - }) { - return Ok(MemoGroupId(index as u32)); - } - let id = - MemoGroupId(u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?); - let node = Rc::new((*node).clone()); - let boundaries = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let children = boundary_groups(&node, &boundaries); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&node), - child_groups: children.clone(), - parents: Vec::new(), - queries: queries.to_vec(), - }); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: node, - input_groups: children, - origin, - accuracy_budget_parts, - }]); - Ok(id) -} - -pub(in crate::pass2::memo) fn append_reader( - memo: &mut Memo, - owner: usize, - original: &CandidateForMemoGroup, - node: Rc, - origin: OriginOfCandidateForMemoGroup, -) -> Result { - if original.root_node.schema.fields != node.schema.fields - || original.root_node.schema.time_index != node.schema.time_index - || original.root_node.schema.unique_keys != node.schema.unique_keys - || (original.root_node.schema.closed && !node.schema.closed) - || original.root_node.result_kind != node.result_kind - { - return Err(Pass2RuleError::ReuseRule(format!( - "reuse changed the reader output contract ({origin:?}): expected {:?}, got {:?}", - original.root_node.schema, node.schema - ))); - } - let boundaries = memo - .groups - .iter() - .enumerate() - .filter(|(id, _)| *id != owner) - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let children = boundary_groups(&node, &boundaries); - if memo.candidates[owner].iter().any(|known| { - known.input_groups == children - && known.accuracy_budget_parts == original.accuracy_budget_parts - && *known.root_node == *node - && serde_json::to_string(&known.root_node).ok() == serde_json::to_string(&node).ok() - }) { - return Ok(false); - } - let mut candidate = original.clone(); - candidate.root_node = node; - candidate.input_groups = children; - candidate.origin = origin; - memo.candidates[owner].push(candidate); - Ok(true) -} - -pub(in crate::pass2::memo) fn replace( - node: &Rc, - replacements: &HashMap<*const OperatorNode, Rc>, -) -> Rc { - fn visit( - node: &Rc, - replacements: &HashMap<*const OperatorNode, Rc>, - done: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(found) = replacements - .get(&Rc::as_ptr(node)) - .or_else(|| done.get(&Rc::as_ptr(node))) - { - return Rc::clone(found); - } - let mut changed = false; - let operator = node.operator.map_children(|child| { - let replacement = visit(child, replacements, done); - changed |= !Rc::ptr_eq(child, &replacement); - replacement - }); - let rebuilt = if changed { - Rc::new(OperatorNode { - operator, - ..(**node).clone() - }) - } else { - Rc::clone(node) - }; - done.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt - } - visit(node, replacements, &mut HashMap::new()) -} - -pub(in crate::pass2::memo) fn combined_inputs( - left: &CandidateForMemoGroup, - right: &CandidateForMemoGroup, -) -> Vec { - let mut seen = HashSet::new(); - left.input_groups - .iter() - .chain(&right.input_groups) - .copied() - .filter(|id| seen.insert(*id)) - .collect() -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs index 1f684e04f..de56a825b 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs @@ -1,360 +1,18 @@ //! Producer-level exact and sketch rollups, using the existing capability rules. -use super::reuse::{ - append_producer, append_reader, combined_inputs, eligible, replace, restore_inputs, -}; -use crate::pass2::asap_aware_cse::{rollup_candidate_for, rollup_pass2_candidates}; -use crate::pass2::memo::state_groups::{simple_build, Pass2RuleError}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ - ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, -}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result { - let mut candidates = Vec::new(); - for (group, entries) in memo.candidates.iter().enumerate() { - for candidate in entries { - if eligible(candidate) - && matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::WidenedScans - ) - { - if let Some((build, _)) = simple_build(memo, candidate) { - candidates.push((group, candidate.clone(), build)); - } - } - } - } - let mut added = 0; - for (coarse_group, coarse, coarse_build) in &candidates { - for (fine_group, fine, _) in &candidates { - if coarse_group == fine_group { - continue; - } - let roots = [ - QueryRoot::Operator(Rc::clone(&coarse.root_node)), - QueryRoot::Operator(Rc::clone(&fine.root_node)), - ]; - let Some(rule) = rollup_candidate_for(&roots, Rc::as_ptr(coarse_build)) else { - continue; - }; - if !OperatorNode::reachable(&fine.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &rule.finer)) - { - continue; - } - let state_rollup = OperatorNode::reachable(&rule.rollup).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - ) - }); - let is_sketch = matches!( - rule.finer.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ); - let rewritten = rollup_pass2_candidates(&roots, &[rule]) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let Some(result) = rewritten.into_iter().next() else { - continue; - }; - // Existing rule API marks individually noncomposable rewrites as - // invalid candidates; retain the local implementation in that case. - let Ok(rewritten) = result.roots else { - continue; - }; - let [QueryRoot::Operator(coarse_node), QueryRoot::Operator(fine_node)] = - rewritten.as_slice() - else { - unreachable!("two operator roots") - }; - let inputs = combined_inputs(coarse, fine); - let coarse_node = restore_inputs(coarse_node, memo, &inputs); - let fine_node = restore_inputs(fine_node, memo, &inputs); - if is_sketch || state_rollup { - // Concrete resized states become ordinary state groups after all - // producer-level configurations have been expanded. - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } else { - let Some(value) = - OperatorNode::reachable(&coarse_node) - .into_iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { child, .. }) - if matches!( - child.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - ) => - { - Some(Rc::clone(child)) - } - _ => None, - }) - else { - continue; - }; - // When the finer query's result is the value itself and every - // implementation of it is exact, read the finer query's group: - // one stored result then serves both queries, as the - // whole-workload rewrite does, without a separate producer the - // finer query would read through an identity projection. - // Exactness is checked when this rule runs; a later - // implementation of the group is still checked by plan - // reconstruction (input contract) and Stage 3 (guarantee). - let fine_result_is_value = *fine_node == *value - && memo.groups[*fine_group].logical_node.schema == value.schema - && memo.candidates[*fine_group].iter().all(|candidate| { - candidate - .root_node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - }); - if fine_result_is_value { - let fine_result = Rc::clone(&memo.groups[*fine_group].logical_node); - let coarse_node = replace( - &coarse_node, - &HashMap::from([(Rc::as_ptr(&value), fine_result)]), - ); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - continue; - } - let mut queries = memo.groups[*coarse_group].queries.clone(); - queries.extend(&memo.groups[*fine_group].queries); - queries.sort_unstable(); - queries.dedup(); - let producer = append_producer( - memo, - Rc::clone(&value), - &queries, - OriginOfCandidateForMemoGroup::Rollup, - 1, - )?; - let canonical = Rc::clone(&memo.groups[producer.0 as usize].logical_node); - let coarse_node = replace( - &coarse_node, - &HashMap::from([(Rc::as_ptr(&value), Rc::clone(&canonical))]), - ); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - if let Some(own_value) = OperatorNode::reachable(&fine_node) - .into_iter() - .find(|node| **node == *value) - { - let fine_reader = if Rc::ptr_eq(&own_value, &fine_node) { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: canonical - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: Rc::clone(&canonical), - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))? - } else { - replace( - &fine_node, - &HashMap::from([(Rc::as_ptr(&own_value), Rc::clone(&canonical))]), - ) - }; - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_reader, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } - } - } - } - Ok(added) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -/// Existing tumbling window rollup rules applied to producer pairs on already generated -/// legal local grids. No whole-workload choice combinations are constructed. -pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result { - use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_shared_chain_candidates, - }; - let mut grids = vec![std::collections::BTreeSet::new(); memo.groups.len()]; - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates.iter().filter(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows - && eligible(candidate) - }) { - if let Some((build, _)) = simple_build(memo, candidate) { - if let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = build.asap() - { - grids[group].insert(*tumbling_window_width_ms); - } - } - } - } - let candidates: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(group, candidates)| { - candidates - .iter() - .filter(|candidate| { - eligible(candidate) && candidate.origin == OriginOfCandidateForMemoGroup::Pass1 - }) - .map(move |candidate| (group, candidate.clone())) - }) - .collect(); - let mut added = 0; - for (coarse_group, coarse) in &candidates { - for (fine_group, fine) in &candidates { - if coarse_group == fine_group || grids[*fine_group].is_empty() { - continue; - } - for grid in &grids[*coarse_group] { - let roots = [ - QueryRoot::Operator(Rc::clone(&coarse.root_node)), - QueryRoot::Operator(Rc::clone(&fine.root_node)), - ]; - let tumbling_windows = - tumbling_window_candidates(&roots, &[Some(*grid), Some(*grid)]); - if tumbling_windows.len() != 2 { - continue; - } - for rewritten in - window_composition_shared_chain_candidates(&roots, &tumbling_windows) - { - if rewritten.shared_chains.is_empty() { - continue; - } - let Ok(nodes) = rewritten.roots else { continue }; - let [QueryRoot::Operator(coarse_node), QueryRoot::Operator(fine_node)] = - nodes.as_slice() - else { - continue; - }; - if !OperatorNode::reachable(coarse_node).iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - ) - }) { - continue; - } - let inputs = combined_inputs(coarse, fine); - let coarse_node = restore_inputs(coarse_node, memo, &inputs); - let fine_node = restore_inputs(fine_node, memo, &inputs); - added += usize::from(append_reader( - memo, - *coarse_group, - coarse, - coarse_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - added += usize::from(append_reader( - memo, - *fine_group, - fine, - fine_node, - OriginOfCandidateForMemoGroup::Rollup, - )?); - } - } - } - } - Ok(added) +/// Rule 6, rollup: a candidate of a coarse aggregate or summary group reading +/// a finer group whose state rolls up (exact or sketch merge). +pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 6 over groups") } -#[cfg(test)] -mod tests { - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{lower_promql, sharing_rules_and_shared_state_groups}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::QueryRoot; - use asap_types::stages::OriginOfCandidateForMemoGroup; - - // When the finer query's own result is the finalized value the coarse - // query rolls up, the rollup reads the finer query's group, so one - // stored result serves both queries without a copy or an identity - // projection. - #[test] - fn a_rollup_reads_the_finer_query_result_itself() { - let roots: Vec<_> = ["sum by (job) (m)", "sum by (job, instance) (m)"] - .into_iter() - .map(|text| QueryRoot::Operator(lower_promql(text, AccuracyTarget::exact()))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = - sharing_rules_and_shared_state_groups(try_populate_pass1(initial, &inventory).unwrap()) - .unwrap(); - let root_of = |query: usize| memo.query_roots[query].as_operator().unwrap().0 as usize; - let (coarse, fine) = (root_of(0), root_of(1)); - assert!( - memo.candidates[coarse].iter().any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::Rollup - && candidate - .input_groups - .iter() - .any(|child| child.0 as usize == fine) - }), - "the coarse rollup reads the finer query's group" - ); - assert!( - !memo.candidates[fine] - .iter() - .any(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup), - "the finer query needs no reader of a separate producer" - ); - } +/// Rule 7, window rollup: a candidate of a longer-range group merging a +/// shorter-range group's windows. +pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 7 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs index 1410c426a..a808eb48f 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs @@ -1,237 +1,11 @@ //! Scan-label widening for explicit memo state sharing. Reuses the existing //! Pass 2 widening rule; only its source-chain boundaries are interned here. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::ir::operator::Reduction; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot}; +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -use crate::pass2::memo::state_groups::{boundary_groups, simple_build, Pass2RuleError}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; - -/// Preserve local implementations and append readers over union-label inputs. -/// Parent/query reachability is rebuilt by the enclosing state-group expansion. -pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result { - let mut owners = Vec::new(); - let mut roots = Vec::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - // Source-group candidates expose scan reuse to every downstream - // operator, including exact/non-summary computations. Their public - // schemas stay narrow through the existing projection rewrite. - if is_source_chain(&candidate.root_node) && super::reuse::eligible(candidate) { - owners.push((group, index)); - roots.push(QueryRoot::Operator(Rc::clone(&candidate.root_node))); - continue; - } - let Some((build, children)) = simple_build(memo, candidate) else { - continue; - }; - if !matches!( - build.asap(), - Some( - ASAPOp::SummaryAgg { - reduction: Reduction::PerEntity, - filter: None, - .. - } | ASAPOp::SummaryTumblingWindowChain { - reduction: Reduction::PerEntity, - filter: None, - .. - } - ) - ) || candidate - .input_groups - .iter() - .any(|child| !children.contains(child)) - || children - .iter() - .any(|child| !is_source_chain(&memo.groups[child.0 as usize].logical_node)) - { - continue; - } - owners.push((group, index)); - roots.push(QueryRoot::Operator(Rc::clone(&candidate.root_node))); - } - } - let Some(widened) = crate::pass2::asap_aware_cse::with_union_scan_labels(&roots) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))? - else { - return Ok(0); - }; - // Widening recomposes every root's guarantees, also those it left - // unchanged, so a root is unchanged when it equals its recomposed - // original. Compared with the stored original instead, an unchanged root - // (a scan that already has every label) that differs only in its - // guarantee annotation would pass for a widened reader, and interning - // would map it back to its own group: a candidate reading the group it - // implements. - let unwidened = recompose_guarantees(&roots) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))?; - let mut added = 0; - let mut canonical = HashMap::new(); - for (((group, index), root), unwidened) in owners.into_iter().zip(widened).zip(unwidened) { - let (QueryRoot::Operator(node), QueryRoot::Operator(unwidened)) = (root, unwidened) else { - unreachable!("operator readers") - }; - let original = memo.candidates[group][index].clone(); - if *node == *unwidened || !same_output_contract(&original.root_node, &node) { - continue; - } - let queries = memo.groups[group].queries.clone(); - let mut node = intern_inputs(&node, memo, &queries, &mut canonical)?; - if memo - .groups - .iter() - .any(|group| Rc::ptr_eq(&group.logical_node, &node)) - { - // Interning an equivalent range/source can return another group's - // canonical value. Keep a concrete reader instead of a root alias. - node = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: node - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| asap_types::ir::ProjectItem { - alias: Some(field.name.clone()), - expr: asap_types::ir::ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: node, - })) - .map_err(|error| Pass2RuleError::ScanLabelWidening(error.to_string()))?; - if !same_output_contract(&original.root_node, &node) { - continue; - } - } - let boundaries = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - let mut reader = original; - reader.root_node = node; - reader.input_groups = boundary_groups(&reader.root_node, &boundaries); - reader.origin = OriginOfCandidateForMemoGroup::WidenedScans; - if memo.candidates[group].iter().any(|known| { - known.origin == OriginOfCandidateForMemoGroup::WidenedScans - && known.input_groups == reader.input_groups - && *known.root_node == *reader.root_node - }) { - continue; - } - memo.candidates[group].push(reader); - added += 1; - } - Ok(added) -} - -fn same_output_contract(expected: &OperatorNode, actual: &OperatorNode) -> bool { - expected.result_kind == actual.result_kind - && expected.schema.fields == actual.schema.fields - && expected.schema.time_index == actual.schema.time_index - && expected.schema.unique_keys == actual.schema.unique_keys - // A projection may enumerate every output of an originally open schema. - && (!expected.schema.closed || actual.schema.closed) -} - -fn is_source_chain(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { .. }) => true, - Some(NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }) => { - is_source_chain(child) - } - _ => false, - } -} - -fn same_source_node(left: &OperatorNode, right: &OperatorNode) -> bool { - // These operators preserve exact source rows; recomposition's exact - // guarantee annotation does not change their logical/execution contract. - left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) - && left.schema == right.schema - && left.result_kind == right.result_kind - && left.timing == right.timing - && left.materialization == right.materialization - && left - .children() - .iter() - .zip(right.children()) - .all(|(a, b)| Rc::ptr_eq(a, b)) - && left.children().len() == right.children().len() -} - -fn intern_inputs( - node: &Rc, - memo: &mut Memo, - queries: &[usize], - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Result, Pass2RuleError> { - if let Some(known) = done.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(known)); - } - let mut children = HashMap::new(); - for child in node.children() { - children.insert( - Rc::as_ptr(child), - intern_inputs(child, memo, queries, done)?, - ); - } - let rewritten = Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), - ..(**node).clone() - }); - let result = if is_source_chain(&rewritten) { - if let Some(existing) = memo - .groups - .iter() - .find(|group| same_source_node(&group.logical_node, &rewritten)) - { - Rc::clone(&existing.logical_node) - } else { - // The new group's id must fit a `MemoGroupId`. - u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?; - let mut seen = HashSet::new(); - let inputs: Vec<_> = rewritten - .children() - .iter() - .map(|child| { - MemoGroupId( - memo.groups - .iter() - .position(|group| Rc::ptr_eq(child, &group.logical_node)) - .expect("source-chain children were interned") - as u32, - ) - }) - .filter(|input| seen.insert(*input)) - .collect(); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&rewritten), - child_groups: inputs.clone(), - parents: Vec::new(), - queries: queries.to_vec(), - }); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&rewritten), - input_groups: inputs, - origin: OriginOfCandidateForMemoGroup::WidenedScans, - accuracy_budget_parts: 1, - }]); - rewritten - } - } else { - rewritten - }; - done.insert(Rc::as_ptr(node), Rc::clone(&result)); - Ok(result) +/// Rule 2, scan widening: a group for the widened scan, and in each original scan group a candidate `Project(widened)`. +pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 2 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs deleted file mode 100644 index 5d45eac0d..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs +++ /dev/null @@ -1,637 +0,0 @@ -//! Expand compatible readers at finite shared sizes and tumbling-window horizons before -//! interning concrete states. Local candidates remain available. -use crate::stage0::cse::same_value; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::flat::{flatten, FlatDag}; -use asap_types::ir::schema::{sketch::SketchKind, FieldDataType}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; - -use super::shared_producer::{ - add_shared_producer_candidates, spans_groups, Reader, SharedProducer, -}; -use crate::pass2::asap_aware_cse::{with_resized_build, with_tumbling_window_chain}; -use crate::pass2::memo::shared_configurations::{ - shared_size, state_size_configurations, tumbling_window_configurations, - TumblingWindowConfiguration, TumblingWindowRequirement, -}; -use crate::pass2::memo::state_groups::{ - boundary_ptrs, same_range_input, simple_build, Pass2RuleError, -}; -use asap_types::stages::Memo; - -/// `resized` (a rewrite of `original` with the same shape) with each node -/// that is a child group's node in `original` put back, and the nodes above -/// rebuilt over them, keeping their own schema and guarantee. -fn with_child_nodes( - original: &Rc, - resized: &Rc, - child_nodes: &HashSet<*const OperatorNode>, - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - if child_nodes.contains(&Rc::as_ptr(original)) { - return Rc::clone(original); - } - if let Some(found) = done.get(&Rc::as_ptr(resized)) { - return Rc::clone(found); - } - let pairs: HashMap<*const OperatorNode, Rc> = original - .children() - .into_iter() - .zip(resized.children()) - .map(|(o, r)| (Rc::as_ptr(r), with_child_nodes(o, r, child_nodes, done))) - .collect(); - let mut changed = false; - let operator = resized.operator.map_children(|child| { - let new = Rc::clone(&pairs[&Rc::as_ptr(child)]); - changed |= !Rc::ptr_eq(&new, child); - new - }); - let out = match changed { - false => Rc::clone(resized), - true => Rc::new( - OperatorNode::with_schema(operator, resized.schema.clone()) - .with_guarantee(resized.guarantee.clone()) - .with_timing(resized.timing) - .with_materialization(resized.materialization), - ), - }; - done.insert(Rc::as_ptr(resized), Rc::clone(&out)); - out -} - -/// A summary build one candidate reads: its own need of the shared producer. -struct StateNeed { - candidate: usize, - build: Rc, - merge: Option>, - family: FieldDataType, - /// The reader candidate's `accuracy_budget_parts`: the shares its build - /// is sized for. - accuracy_budget_parts: usize, - /// The reader's normalized tumbling-window layout, when its set shares tumbling windows: equal - /// layouts of one set have one index, so a configuration rewrites them once. - normalized_layout: Option, -} - -/// A shared build's need: one tumbling-window layout (its index among the set's tumbling window -/// configurations, for request deduplication) or one size, with the shares -/// of the budget its build is sized for ([`covered_parts`]). -enum StateCovering { - TumblingWindows { - configuration_index: usize, - configuration: TumblingWindowConfiguration, - accuracy_budget_parts: usize, - }, - Size { - family: FieldDataType, - accuracy_budget_parts: usize, - }, -} - -/// Summary builds of one family over one input at different sizes or tumbling window -/// layouts: each reader is rewritten to a build at each covering size or -/// layout that serves it, and equal concrete builds become one state group -/// later ([`super::super::state_groups`]). -struct SummaryStates { - deduplicate_requests: bool, - /// Every candidate's graph signature by group, so no rewrite repeats one. - known: HashMap<(usize, String), Vec>>, - rewrite_count: usize, - /// The (normalized layout, configuration index) rewrites already made. - requests: HashSet<(usize, usize)>, -} - -pub(crate) fn add_state_configuration_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - expand_configurations(memo, true).map(|_| ()) -} - -fn expand_configurations( - memo: &mut Memo, - deduplicate_requests: bool, -) -> Result { - let mut states = SummaryStates { - deduplicate_requests, - known: HashMap::new(), - rewrite_count: 0, - requests: HashSet::new(), - }; - add_shared_producer_candidates(memo, &mut states)?; - Ok(states.rewrite_count) -} - -impl SharedProducer for SummaryStates { - type Need = StateNeed; - type Covering = StateCovering; - - fn sets(&mut self, memo: &Memo) -> Result>>, Pass2RuleError> { - let mut sets: Vec>> = Vec::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (candidate, entry) in candidates.iter().enumerate() { - let Some((build, _)) = simple_build(memo, entry) else { - continue; - }; - // Configuration expansion precedes Stage 2. The rewrite helpers - // create unassigned state; preserve any already assigned build. - if build.timing.is_some() - || build.materialization - != asap_types::physical::Materialization::NotMaterialized - { - continue; - } - let family = match build.asap() { - Some( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => family.clone(), - _ => unreachable!(), - }; - let merge = if matches!( - build.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - let merges: Vec<_> = OperatorNode::reachable(&entry.root_node).into_iter().filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryMerge {children, group_by: None, ..}) - if children.len() == 1 && Rc::ptr_eq(&children[0], &build)) - }).collect(); - // The existing rewrite changes one plain merge. Other tumbling window uses - // need their own explicit transformation, not guessed semantics. - if merges.len() != 1 { - continue; - } - Some(Rc::clone(&merges[0])) - } else { - None - }; - let reader = Reader { - group, - need: StateNeed { - candidate, - build, - merge, - family, - accuracy_budget_parts: entry.accuracy_budget_parts, - normalized_layout: None, - }, - }; - match sets - .iter_mut() - .find(|set| compatible(&set[0].need, &reader.need)) - { - Some(set) => set.push(reader), - None => sets.push(vec![reader]), - } - } - } - for (group, candidates) in memo.candidates.iter().enumerate() { - for candidate in candidates { - self.known - .entry((group, graph_signature(&candidate.root_node)?)) - .or_default() - .push(Rc::clone(&candidate.root_node)); - } - } - let mut layouts: Vec<(usize, usize, FlatDag)> = Vec::new(); - for set in &mut sets { - if spans_groups(set) { - assign_normalized_layouts(memo, set, &mut layouts)?; - } - } - Ok(sets) - } - - fn coverings(&self, set: &[Reader]) -> Result, Pass2RuleError> { - let tumbling_window_requirements: Option> = set - .iter() - .map(|reader| tumbling_window_requirement(&reader.need)) - .collect(); - let Some(requirements) = tumbling_window_requirements else { - let requirements = set - .iter() - .map(|reader| match &reader.need.family { - FieldDataType::Sketch(kind, _) => Some(kind.params().clone()), - _ => None, - }) - .collect(); - let first = &set[0].need.family; - return Ok(state_size_configurations(requirements) - .into_iter() - .map(|configuration| { - let family = match (first, configuration) { - (FieldDataType::Sketch(kind, grouping), Some(params)) => { - FieldDataType::Sketch( - SketchKind::new(*kind.algorithm(), params), - grouping.clone(), - ) - } - _ => first.clone(), - }; - StateCovering::Size { - accuracy_budget_parts: covered_parts(&family, set), - family, - } - }) - .collect()); - }; - Ok(tumbling_window_configurations(&requirements) - .into_iter() - .enumerate() - .map( - |(configuration_index, configuration)| StateCovering::TumblingWindows { - configuration_index, - accuracy_budget_parts: covered_parts(&configuration.family, set), - configuration, - }, - ) - .collect()) - } - - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader, - covering: &StateCovering, - ) -> Result { - let need = &reader.need; - let serves = match covering { - StateCovering::TumblingWindows { configuration, .. } => configuration - .serves(&tumbling_window_requirement(need).expect("tumbling window set")), - StateCovering::Size { family, .. } => serves_size(family, &need.family), - }; - if !serves { - return Ok(0); - } - let original = memo.candidates[reader.group][need.candidate].clone(); - let roots = [QueryRoot::Operator(Rc::clone(&original.root_node))]; - let result = match covering { - StateCovering::TumblingWindows { - configuration_index, - configuration, - .. - } => { - let Ok(count) = u32::try_from( - configuration.longest_window_ms / configuration.tumbling_window_width_ms, - ) else { - return Ok(0); - }; - if self.deduplicate_requests - && need - .normalized_layout - .is_some_and(|layout| !self.requests.insert((layout, *configuration_index))) - { - return Ok(0); - } - self.rewrite_count += 1; - with_tumbling_window_chain( - &roots, - need.merge.as_ref().unwrap(), - configuration.family.clone(), - configuration.tumbling_window_width_ms, - count, - ) - } - StateCovering::Size { family, .. } => { - self.rewrite_count += 1; - with_resized_build(&roots, &need.build, family.clone()) - } - }; - let boundaries = boundary_ptrs(memo, &original); - let roots = - result.map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let [QueryRoot::Operator(node)] = roots.as_slice() else { - unreachable!("operator rewrite") - }; - let node = with_child_nodes(&original.root_node, node, &boundaries, &mut HashMap::new()); - // Resizing affects internal state types, never public output fields. - if original.root_node.schema.fields != node.schema.fields - || original.root_node.schema.time_index != node.schema.time_index - || original.root_node.schema.unique_keys != node.schema.unique_keys - || (original.root_node.schema.closed && !node.schema.closed) - { - return Err(Pass2RuleError::StateConfiguration( - "state rewrite changed the reader output contract".into(), - )); - } - let previous = self - .known - .entry((reader.group, graph_signature(&node)?)) - .or_default(); - if previous.iter().any(|known| **known == *node) { - return Ok(0); - } - previous.push(Rc::clone(&node)); - let mut candidate = original; - candidate.root_node = node; - // The rewritten build is sized for the covering's shares, at least - // the reader's own. - candidate.accuracy_budget_parts = match covering { - StateCovering::TumblingWindows { - accuracy_budget_parts, - .. - } - | StateCovering::Size { - accuracy_budget_parts, - .. - } => (*accuracy_budget_parts).max(candidate.accuracy_budget_parts), - }; - memo.candidates[reader.group].push(candidate); - Ok(1) - } -} - -/// Numbers the normalized tumbling-window layouts of `set`'s readers, when it shares -/// tumbling windows: different original layouts can produce the same requested -/// reader/configuration, so each is normalized once and its rewrite made once, -/// instead of once for every existing tumbling-window width. Numbers continue across -/// sets (`layouts`), so a configuration index of one set never matches -/// another's. -fn assign_normalized_layouts( - memo: &Memo, - set: &mut [Reader], - layouts: &mut Vec<(usize, usize, FlatDag)>, -) -> Result<(), Pass2RuleError> { - let Some(requirements) = set - .iter() - .map(|reader| tumbling_window_requirement(&reader.need)) - .collect::>>() - else { - return Ok(()); - }; - let mut layout_keys: HashMap<(usize, String), Vec> = HashMap::new(); - for (reader, requirement) in set.iter_mut().zip(&requirements) { - let Ok(count) = u32::try_from(requirement.window_ms) else { - // Normalization must not narrow the legal long-window domain. - continue; - }; - let need = &reader.need; - let original = &memo.candidates[reader.group][need.candidate]; - let roots = [QueryRoot::Operator(Rc::clone(&original.root_node))]; - let normalized = with_tumbling_window_chain( - &roots, - need.merge.as_ref().unwrap(), - need.family.clone(), - 1, - count, - ) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let flat = flatten(&normalized).0; - let signature = serde_json::to_string(&flat) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string()))?; - let indices = layout_keys.entry((reader.group, signature)).or_default(); - let existing = indices.iter().copied().find(|&index| { - let (group, candidate, previous) = &layouts[index]; - let previous_candidate = &memo.candidates[*group][*candidate]; - previous == &flat - && previous_candidate.origin == original.origin - && previous_candidate.input_groups == original.input_groups - && previous_candidate.fused_groups == original.fused_groups - && previous_candidate.accuracy_budget_parts == original.accuracy_budget_parts - }); - reader.need.normalized_layout = Some(existing.unwrap_or_else(|| { - let index = layouts.len(); - layouts.push((reader.group, need.candidate, flat)); - indices.push(index); - index - })); - } - Ok(()) -} - -fn graph_signature(node: &Rc) -> Result { - serde_json::to_string(&flatten(&[QueryRoot::Operator(Rc::clone(node))]).0) - .map_err(|error| Pass2RuleError::StateConfiguration(error.to_string())) -} - -fn tumbling_window_requirement(reader: &StateNeed) -> Option { - let ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - } = reader.build.asap()? - else { - return None; - }; - let ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - } = reader.merge.as_ref()?.asap()? - else { - return None; - }; - Some(TumblingWindowRequirement { - tumbling_window_width_ms: *tumbling_window_width_ms, - window_ms: tumbling_window_width_ms.checked_mul(u64::from( - newest_tumbling_windows.unwrap_or(*tumbling_window_count), - ))?, - family: reader.family.clone(), - }) -} - -/// Whether a build of `family` serves a reader of `reader`: the size both -/// share is `family` itself. -fn serves_size(family: &FieldDataType, reader: &FieldDataType) -> bool { - shared_size(family, reader).is_some_and(|joined| same_value(&joined, family)) -} - -/// The shares of the budget a build of `family` is sized for: the most of -/// any reader in `set` it serves (it is at least as large as that reader's -/// own build, sized for those shares); 1 when it serves none. -fn covered_parts(family: &FieldDataType, set: &[Reader]) -> usize { - set.iter() - .filter(|reader| serves_size(family, &reader.need.family)) - .map(|reader| reader.need.accuracy_budget_parts) - .max() - .unwrap_or(1) -} - -fn compatible(left: &StateNeed, right: &StateNeed) -> bool { - if shared_size(&left.family, &right.family).is_none() { - return false; - } - let left_inputs = left.build.children(); - let right_inputs = right.build.children(); - if left_inputs.len() != right_inputs.len() - || !left_inputs - .iter() - .zip(right_inputs) - .all(|(a, b)| Rc::ptr_eq(a, b) || same_range_input(a, b)) - { - return false; - } - // Ignore only the fields this rule rewrites. Build guarantees are derived - // from these operators and are recomposed by the existing rewrite helpers. - let signature = |reader: &StateNeed| { - let mut operator = reader.build.operator.map_children(|_| ()); - match &mut operator { - Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) => *family = left.family.clone(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - *family = left.family.clone(); - *tumbling_window_width_ms = 1; - *tumbling_window_count = 1; - } - _ => unreachable!(), - } - let mut schema = reader.build.schema.clone(); - for field in &mut schema.fields { - if same_value(&field.dtype, &reader.family) { - field.dtype = left.family.clone(); - } - } - serde_json::to_string(&( - operator, - schema, - reader.build.result_kind, - reader.build.timing, - reader.build.materialization, - )) - .ok() - }; - signature(left).is_some_and(|signature_left| Some(signature_left) == signature(right)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - // Request deduplication preserves the entire declared finite configuration - // set while avoiding repeated rewrites through every initial tumbling-window layout. - #[test] - fn equivalent_tumbling_window_layouts_expand_each_configuration_once() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let roots: Vec<_> = [0.95, 0.99] - .into_iter() - .map(|quantile| { - QueryRoot::Operator(lower_promql( - &format!("quantile_over_time({quantile}, latency[5m])"), - target.clone(), - )) - }) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - let entries = vec![ - entry_with( - target, - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(6))) - ); - 2 - ]; - crate::pass2::memo::window_candidates::add_local_window_composition(&mut memo, &entries); - let mut unoptimized = memo.clone(); - let optimized_rewrites = expand_configurations(&mut memo, true).unwrap(); - let exhaustive_rewrites = expand_configurations(&mut unoptimized, false).unwrap(); - assert!( - optimized_rewrites < exhaustive_rewrites, - "{optimized_rewrites} versus {exhaustive_rewrites}" - ); - let signatures = |memo: &Memo| -> Vec> { - memo.candidates - .iter() - .map(|candidates| { - candidates - .iter() - .map(|candidate| graph_signature(&candidate.root_node).unwrap()) - .collect() - }) - .collect() - }; - assert_eq!(signatures(&memo), signatures(&unoptimized)); - let widths: HashSet<_> = memo - .candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => Some(*tumbling_window_width_ms), - _ => None, - }) - .collect(); - assert_eq!(widths, HashSet::from([1, 2, 3, 6])); - } - - /// The KLL sizes (`k`) of the builds `candidate` reads. - fn kll_sizes(candidate: &asap_types::stages::CandidateForMemoGroup) -> Vec { - OperatorNode::reachable(&candidate.root_node) - .iter() - .filter_map(|node| match node.asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => match kind.params() { - asap_types::ir::schema::SketchParams::Kll { k, .. } => Some(*k), - _ => None, - }, - _ => None, - }) - .collect() - } - - // A rewrite to a size another reader's build was sized for carries that - // build's shares of the budget: in `q0.5 / q0.9` each operand has a KLL - // sized for the whole budget and one for half of it, and the copy of the - // whole-budget reader rewritten to the half-budget size is sized for two - // shares, not one. - #[test] - fn a_rewrite_to_a_shared_size_carries_its_parts() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let roots = vec![QueryRoot::Operator(lower_promql( - "quantile_over_time(0.5, latency[5m]) / quantile_over_time(0.9, latency[5m])", - target.clone(), - ))]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let mut memo = try_populate_pass1(initial, &inventory).unwrap(); - // Pass 1's parts per KLL size. - let mut parts_of_size: HashMap = HashMap::new(); - for candidate in memo.candidates.iter().flatten() { - for k in kll_sizes(candidate) { - let parts = parts_of_size.entry(k).or_insert(0); - *parts = (*parts).max(candidate.accuracy_budget_parts); - } - } - assert!( - parts_of_size.values().any(|&parts| parts == 2), - "{parts_of_size:?}" - ); - let before = memo.candidates.iter().map(Vec::len).sum::(); - expand_configurations(&mut memo, true).unwrap(); - assert!(memo.candidates.iter().map(Vec::len).sum::() > before); - for candidate in memo.candidates.iter().flatten() { - for k in kll_sizes(candidate) { - if let Some(&parts) = parts_of_size.get(&k) { - assert!( - candidate.accuracy_budget_parts >= parts, - "k {k}: parts {} below {parts}", - candidate.accuracy_budget_parts - ); - } - } - } - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs deleted file mode 100644 index ea01f1956..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_producer.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! The shared-producer rule: readers that need one computation at different -//! values (a top-k `k`, a sketch size, a window and step) share one producer -//! built at a need that covers them, and each reader derives its own result -//! from it as one more candidate; its own candidates stay, so plan selection -//! decides. Identical expressions are the trivial case, shared by memo group -//! identity, and add no candidates here. -use crate::pass2::memo::state_groups::Pass2RuleError; -use asap_types::stages::Memo; - -/// One reader of a kind of shared producer: its memo group and its need. -pub(in crate::pass2::memo) struct Reader { - pub group: usize, - pub need: N, -} - -/// A kind of producer several readers can share. -pub(in crate::pass2::memo) trait SharedProducer { - /// What one reader requires of the producer. - type Need; - /// A producer's need that serves several readers at once. - type Covering; - /// The memo's readers of this kind, in sets whose members can all read - /// one producer; in memo order. - fn sets(&mut self, memo: &Memo) -> Result>>, Pass2RuleError>; - /// The coverings to offer the readers of `set`. - fn coverings(&self, set: &[Reader]) -> Result, Pass2RuleError>; - /// Adds `reader`'s derivation from the producer at `covering`, if that - /// producer serves it; the number of candidates added. - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader, - covering: &Self::Covering, - ) -> Result; -} - -/// Whether `set`'s readers belong to two or more groups; a set inside one -/// group shares nothing. -pub(in crate::pass2::memo) fn spans_groups(set: &[Reader]) -> bool { - set.iter().any(|reader| reader.group != set[0].group) -} - -/// Gives every reader of a set spanning two or more groups its derivation -/// from each of the set's coverings; the number of candidates added. -pub(in crate::pass2::memo) fn add_shared_producer_candidates( - memo: &mut Memo, - producer: &mut P, -) -> Result { - let mut added = 0; - for set in producer.sets(memo)? { - if !spans_groups(&set) { - continue; - } - let coverings = producer.coverings(&set)?; - for reader in &set { - for covering in &coverings { - added += producer.derive(memo, reader, covering)?; - } - } - } - Ok(added) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs index e2bb0e84f..956141fbf 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs @@ -1,100 +1,12 @@ //! Expose private value projections and filters below summary builds as canonical inputs. //! This preserves their full expressions and schema, including opaque series //! identity, rather than assuming that a projection is an identity operation. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use crate::pass2::memo::state_groups::{boundary_ptrs, nodes_within, Pass2RuleError}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; -pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let originals: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(owner, candidates)| { - candidates - .iter() - .filter(|candidate| eligible(candidate)) - .map(move |candidate| (owner, candidate.clone())) - }) - .collect(); - for (owner, original) in originals { - let boundaries = boundary_ptrs(memo, &original); - let mut inputs = Vec::new(); - for node in nodes_within(&original.root_node, &boundaries) { - if matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - for child in node.children() { - if !boundaries.contains(&Rc::as_ptr(child)) { - inputs.push(Rc::clone(child)); - } - } - } - } - let queries = memo.groups[owner].queries.clone(); - let mut replacements = HashMap::new(); - for input in inputs { - canonical_preprocessing(memo, &input, &boundaries, &queries, &mut replacements)?; - } - if !replacements.is_empty() { - let reader = replace(&original.root_node, &replacements); - append_reader(memo, owner, &original, reader, original.origin.clone())?; - } - } - Ok(()) -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -fn canonical_preprocessing( - memo: &mut Memo, - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - queries: &[usize], - replacements: &mut HashMap<*const OperatorNode, Rc>, -) -> Result { - if replacements.contains_key(&Rc::as_ptr(node)) { - return Ok(true); - } - if boundaries.contains(&Rc::as_ptr(node)) { - // Logical TimeRange groups can remain distinct when their inputs - // differ structurally. The existing state-sharing rule nevertheless - // permits identical range descriptors over the same canonical source. - if let Some(group) = memo.groups.iter().find(|group| { - crate::pass2::memo::state_groups::same_range_input(&group.logical_node, node) - }) { - replacements.insert(Rc::as_ptr(node), Rc::clone(&group.logical_node)); - } - return Ok(true); - } - if !matches!( - node.non_asap(), - Some(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) - ) { - return Ok(false); - } - // Predicates/projections can reference subqueries. Every dependency must - // be canonical, not only the row-stream child. - for child in node.children() { - if !canonical_preprocessing(memo, child, boundaries, queries, replacements)? { - return Ok(false); - } - } - let canonical = replace(node, replacements); - let group = append_producer( - memo, - canonical, - queries, - OriginOfCandidateForMemoGroup::SummaryInput, - // These operators introduce no approximate state. Their identity does - // not depend on how a downstream consumer divides its error budget. - 1, - )?; - replacements.insert( - Rc::as_ptr(node), - Rc::clone(&memo.groups[group.0 as usize].logical_node), - ); - Ok(true) +/// Rule 3, summary inputs: a value projection or filter below a summary build as its own input group. +pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 3 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs deleted file mode 100644 index 49d84f991..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_results.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Factor composite implementations at summary readouts. Each readout can then -//! choose a state configuration independently, without enumerating combinations -//! of all private states in the enclosing query. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use crate::pass2::memo::state_groups::{ - boundary_ptrs, direct_input_builds, nodes_within, Pass2RuleError, -}; -use asap_types::ir::{ASAPOp, OperatorNode}; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -pub(crate) fn factor_summary_results(memo: &mut Memo) -> Result { - let originals: Vec<_> = memo - .candidates - .iter() - .enumerate() - .flat_map(|(group, candidates)| { - candidates - .iter() - .filter(|candidate| eligible(candidate)) - .map(move |candidate| (group, candidate.clone())) - }) - .collect(); - let mut added = 0; - for (owner, original) in originals { - let builds = direct_input_builds(memo, &original); - if builds.len() < 2 { - continue; - } - let build_pointers: HashSet<_> = - builds.iter().map(|(build, _)| Rc::as_ptr(build)).collect(); - let boundaries = boundary_ptrs(memo, &original); - let mut readouts = Vec::new(); - for node in nodes_within(&original.root_node, &boundaries) { - if !Rc::ptr_eq(&node, &original.root_node) - && matches!( - node.asap(), - Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }) - ) - { - let private_builds: HashSet<_> = OperatorNode::reachable(&node) - .iter() - .map(Rc::as_ptr) - .filter(|pointer| build_pointers.contains(pointer)) - .collect(); - if private_builds.len() == 1 { - readouts.push(Rc::clone(&node)); - } - } - } - if readouts.is_empty() { - continue; - } - let queries = memo.groups[owner].queries.clone(); - let mut replacements = HashMap::new(); - for readout in readouts { - let group = append_producer( - memo, - Rc::clone(&readout), - &queries, - OriginOfCandidateForMemoGroup::SummaryResult, - original.accuracy_budget_parts, - )?; - // The original reader's allocation was used to generate these - // parameters; lifting the boundary must not reset that provenance. - replacements.insert( - Rc::as_ptr(&readout), - Rc::clone(&memo.groups[group.0 as usize].logical_node), - ); - } - let reader = replace(&original.root_node, &replacements); - added += usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::SummaryResult, - )?); - } - Ok(added) -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs index 9593a317e..e3b1b22dc 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs @@ -1,267 +1,11 @@ //! Exact ranking reuse with explicit exact-input groups. No global selection //! flags are required: only exact implementations can supply these inputs. -use super::reuse::{append_producer, append_reader, eligible, replace}; -use super::shared_producer::{add_shared_producer_candidates, Reader, SharedProducer}; -use crate::pass2::asap_aware_cse::topk_reuse_candidates; -use crate::pass2::memo::state_groups::{boundary_ptrs, nodes_within, Pass2RuleError}; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::stages::MemoGroupId; -use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; -use std::collections::HashMap; -use std::rc::Rc; -pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result { - add_shared_producer_candidates( - memo, - &mut TopKRankings { - exact_groups: HashMap::new(), - }, - ) -} - -/// Exact top-k readers of one input and partition with different `k`: the -/// covering need is the ranking with the largest `k`, `Limit(k_max)` over -/// `Sort` of an exact input ([`topk_reuse_candidates`]); each reader derives -/// `Limit(k) ▸ Project` from it. -struct TopKRankings { - /// Each input group's exact-only group ([`exact_input_group`]). - exact_groups: HashMap>, -} - -/// One input's top-k readers' shared ranking: `Limit(k_max)` over `Sort`, -/// and the readers' groups, whose queries the producer serves. -struct SharedRanking { - ranking: Rc, - reader_groups: Vec, -} - -impl SharedProducer for TopKRankings { - /// The ranking of the reader's input and partition, at the largest `k` - /// of its set. - type Need = Rc; - type Covering = SharedRanking; - - fn sets(&mut self, memo: &Memo) -> Result>>>, Pass2RuleError> { - let mut owners = HashMap::new(); - let mut roots = Vec::new(); - for (group, entry) in memo.groups.iter().enumerate() { - let [input] = entry.logical_node.children()[..] else { - continue; - }; - let exact = Rc::new( - (**input) - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact( - "exact input candidate group", - ))), - ); - let Ok(copy) = entry.logical_node.with_new_children(|_| Rc::clone(&exact)) else { - continue; - }; - let copy = Rc::new(copy); - owners.insert(Rc::as_ptr(©), group); - roots.push(QueryRoot::Operator(copy)); - } - Ok(topk_reuse_candidates(&roots) - .into_iter() - .map(|reuse| { - reuse - .members - .iter() - .filter_map(|node| { - Some(Reader { - group: *owners.get(&Rc::as_ptr(node))?, - need: Rc::clone(&reuse.ranking), - }) - }) - .collect() - }) - .collect()) - } - - fn coverings( - &self, - set: &[Reader>], - ) -> Result, Pass2RuleError> { - // The largest `k` holds every smaller top-k, so it serves the set. - Ok(vec![SharedRanking { - ranking: Rc::clone(&set[0].need), - reader_groups: set.iter().map(|reader| reader.group).collect(), - }]) - } - - fn derive( - &mut self, - memo: &mut Memo, - reader: &Reader>, - covering: &SharedRanking, - ) -> Result { - // Interning the ranking is idempotent, so every reader of the set - // reads the producer the first one interned. - let Some(input_group) = memo.groups[covering.reader_groups[0]] - .child_groups - .first() - .copied() - else { - return Ok(0); - }; - let Some(exact_input) = exact_input_group(memo, input_group, &mut self.exact_groups)? - else { - return Ok(0); - }; - let input = Rc::clone(&memo.groups[exact_input.0 as usize].logical_node); - let queries: Vec<_> = covering - .reader_groups - .iter() - .flat_map(|&group| memo.groups[group].queries.iter().copied()) - .collect::>() - .into_iter() - .collect(); - let sort = covering.ranking.children()[0] - .with_new_children(|_| Rc::clone(&input)) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let sort_group = append_producer( - memo, - Rc::new(sort), - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - let sort = Rc::clone(&memo.groups[sort_group.0 as usize].logical_node); - let ranking = covering - .ranking - .with_new_children(|_| Rc::clone(&sort)) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let ranking_group = append_producer( - memo, - Rc::new(ranking), - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - let ranking = Rc::clone(&memo.groups[ranking_group.0 as usize].logical_node); - let owner = reader.group; - let logical = Rc::clone(&memo.groups[owner].logical_node); - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(keys), - measures, - .. - }) = logical.non_asap() - else { - return Ok(0); - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Ok(0); - }; - let Some(original) = memo.candidates[owner].iter().find(|candidate| eligible(candidate) && matches!(candidate.root_node.non_asap(), Some(NonASAPOp::Aggregate {measures, ..}) if matches!(measures.as_slice(), [AggIntent::TopK {..}]))).cloned() else { return Ok(0) }; - let limited = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: Rc::clone(&ranking), - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let columns = asap_types::ir::schema::topk::ranked_row_projection( - &ranking, - &Reduction::Reduce(keys.clone()), - &logical.schema.fields[keys.len()].name, - ) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - let reader = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: columns, - qualifier: None, - child: limited, - })) - .map_err(|error| Pass2RuleError::ReuseRule(error.to_string()))?; - Ok(usize::from(append_reader( - memo, - owner, - &original, - reader, - OriginOfCandidateForMemoGroup::TopKReuse, - )?)) - } -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; -fn exact_input_group( - memo: &mut Memo, - group: MemoGroupId, - known: &mut HashMap>, -) -> Result, Pass2RuleError> { - if let Some(result) = known.get(&group) { - return Ok(*result); - } - // An in-progress marker also prevents malformed cyclic inventories recursing. - known.insert(group, None); - let originals = memo.candidates[group.0 as usize].clone(); - let original_children: Vec<_> = originals - .iter() - .map(|candidate| candidate.input_groups.clone()) - .collect(); - let mut candidates = Vec::new(); - for mut candidate in originals { - if !eligible(&candidate) { - continue; - } - let boundaries = boundary_ptrs(memo, &candidate); - let mut exact = !nodes_within(&candidate.root_node, &boundaries).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg {family, ..} | ASAPOp::SummaryTumblingWindowChain {family, ..}) if !matches!(family, FieldDataType::ExactAggregate(..))) - }); - if !exact { - continue; - } - let mut replacements = HashMap::new(); - let mut children = Vec::new(); - for child in &candidate.input_groups { - let Some(exact_child) = exact_input_group(memo, *child, known)? else { - exact = false; - break; - }; - replacements.insert( - Rc::as_ptr(&memo.groups[child.0 as usize].logical_node), - Rc::clone(&memo.groups[exact_child.0 as usize].logical_node), - ); - children.push(exact_child); - } - if !exact { - continue; - } - candidate.root_node = replace(&candidate.root_node, &replacements); - candidate.input_groups = children; - candidates.push(candidate); - } - if candidates.is_empty() { - return Ok(None); - } - if candidates.len() == original_children.len() - && candidates - .iter() - .zip(&original_children) - .all(|(candidate, children)| candidate.input_groups == *children) - { - // An already exact-only group remains the shared source for ordinary - // consumers too; cloning it would unnecessarily split scan/state reuse. - known.insert(group, Some(group)); - return Ok(Some(group)); - } - let node = Rc::new((*candidates[0].root_node).clone().with_guarantee(Some( - AccuracyGuarantee::exact("only exact implementations and exact inputs"), - ))); - let queries = memo.groups[group.0 as usize].queries.clone(); - let id = append_producer( - memo, - node, - &queries, - OriginOfCandidateForMemoGroup::TopKReuse, - 1, - )?; - for candidate in &mut candidates { - candidate.fused_groups = Vec::new(); - } - memo.candidates[id.0 as usize] = candidates; - known.insert(group, Some(id)); - Ok(Some(id)) +/// Rule 4, top-k reuse: a candidate of each top-k group reading a larger-`k` group, with an exact-input requirement. +pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let _ = memo; + todo!("#1207 PR 4: rule 4 over groups") } diff --git a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs b/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs deleted file mode 100644 index 9eb61a2db..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Finite state configurations shared by memo rules and candidate selection. -use crate::stage0::cse::same_value; -use asap_accuracy::estimators::size_for_all_readers; -use asap_types::ir::schema::{ - sketch::{SketchKind, SketchParams}, - FieldDataType, -}; -use asap_types::workload::greatest_common_divisor; -use std::collections::BTreeSet; - -/// Every tumbling-window width that divides one of `grids` (zero grids -/// ignored), in ascending order. -pub(crate) fn tumbling_window_width_domain(grids: impl IntoIterator) -> Vec { - let grids: BTreeSet = grids.into_iter().filter(|&grid| grid > 0).collect(); - let mut widths = BTreeSet::new(); - for grid in grids { - widths.extend(tumbling_window_width_divisors(grid)); - } - widths.into_iter().collect() -} - -#[derive(Debug, Clone)] -pub(crate) struct TumblingWindowRequirement { - pub tumbling_window_width_ms: u64, - pub window_ms: u64, - pub family: FieldDataType, -} -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct TumblingWindowConfiguration { - pub tumbling_window_width_ms: u64, - pub longest_window_ms: u64, - pub family: FieldDataType, -} -impl TumblingWindowConfiguration { - pub fn serves(&self, reader: &TumblingWindowRequirement) -> bool { - self.tumbling_window_width_ms > 0 - && reader - .tumbling_window_width_ms - .is_multiple_of(self.tumbling_window_width_ms) - && reader - .window_ms - .is_multiple_of(self.tumbling_window_width_ms) - && self - .longest_window_ms - .is_multiple_of(self.tumbling_window_width_ms) - && reader.window_ms <= self.longest_window_ms - && shared_size(&self.family, &reader.family) - .is_some_and(|joined| same_value(&joined, &self.family)) - } -} -/// Every positive integer-millisecond divisor of a grid, in ascending order. -/// The time representation fixes the search resolution; no representative-width -/// sampling or arbitrary candidate cap changes this domain. -pub(crate) fn tumbling_window_width_divisors(grid_ms: u64) -> Vec { - let mut widths = BTreeSet::new(); - let mut divisor = 1; - while divisor <= grid_ms / divisor { - if grid_ms.is_multiple_of(divisor) { - widths.insert(divisor); - widths.insert(grid_ms / divisor); - } - divisor += 1; - } - widths.into_iter().collect() -} - -/// Complete divisor grids, distinct required horizons, and joined state sizes. -/// This enumerates configurations of one compatible producer, not workload plans. -pub(crate) fn tumbling_window_configurations( - readers: &[TumblingWindowRequirement], -) -> Vec { - if readers - .iter() - .any(|r| r.tumbling_window_width_ms == 0 || r.window_ms == 0) - { - return Vec::new(); - } - let grids = - tumbling_window_width_domain(readers.iter().map(|reader| { - greatest_common_divisor(reader.tumbling_window_width_ms, reader.window_ms) - })); - let mut configurations = Vec::new(); - for &tumbling_window_width_ms in &grids { - let eligible: Vec<&TumblingWindowRequirement> = readers - .iter() - .filter(|r| { - r.tumbling_window_width_ms - .is_multiple_of(tumbling_window_width_ms) - && r.window_ms.is_multiple_of(tumbling_window_width_ms) - }) - .collect(); - let windows: BTreeSet = eligible.iter().map(|r| r.window_ms).collect(); - // The sizes: the readers' own, closed under the shared size. - let mut sizes: Vec = Vec::new(); - for reader in &eligible { - if !sizes.iter().any(|known| same_value(known, &reader.family)) { - sizes.push(reader.family.clone()); - } - } - let mut i = 0; - while i < sizes.len() { - for j in 0..i { - if let Some(joined) = shared_size(&sizes[i], &sizes[j]) { - if !sizes.iter().any(|known| same_value(known, &joined)) { - sizes.push(joined); - } - } - } - i += 1; - } - for &longest_window_ms in &windows { - for family in &sizes { - let configuration = TumblingWindowConfiguration { - tumbling_window_width_ms, - longest_window_ms, - family: family.clone(), - }; - let admitted: Vec<&&TumblingWindowRequirement> = eligible - .iter() - .filter(|r| configuration.serves(r)) - .collect(); - let window = admitted.iter().map(|r| r.window_ms).max(); - let size = admitted.iter().try_fold(None::, |acc, r| { - Some(Some(match acc { - None => r.family.clone(), - Some(acc) => shared_size(&acc, &r.family)?, - })) - }); - if window == Some(longest_window_ms) - && size.flatten().is_some_and(|size| same_value(&size, family)) - { - configurations.push(configuration); - } - } - } - } - configurations -} - -/// All distinct joined reader requirements, retaining recorded error contracts. -/// This is configuration generation only; sharing is represented by state groups. -pub(crate) fn state_size_configurations( - requirements: Vec>, -) -> Vec> { - if requirements.iter().any(Option::is_none) { - return vec![None]; - } - let requirements: Vec = requirements.into_iter().flatten().collect(); - let mut joins: Vec = Vec::new(); - for requirement in &requirements { - let mut grown = vec![requirement.clone()]; - for join in &joins { - if let Some(joined) = size_for_all_readers(&[join, requirement]) { - grown.push(joined); - } - } - for join in grown { - if !joins.iter().any(|known| same_size(known, &join)) { - joins.push(join); - } - } - } - let mut configurations = Vec::new(); - for configuration in joins { - let admitted: Vec<_> = requirements - .iter() - .filter(|required| { - size_for_all_readers(&[&configuration, required]) - .is_some_and(|joined| same_size(&joined, &configuration)) - }) - .collect(); - if size_for_all_readers(&admitted).is_some_and(|joined| same_size(&joined, &configuration)) - { - configurations.push(Some(configuration)); - } - } - configurations -} - -fn same_size(a: &SketchParams, b: &SketchParams) -> bool { - a == b && format!("{a:?}") == format!("{b:?}") -} - -pub(crate) fn shared_size(a: &FieldDataType, b: &FieldDataType) -> Option { - match (a, b) { - (FieldDataType::Sketch(a, grouping), FieldDataType::Sketch(b, grouping_b)) - if a.algorithm() == b.algorithm() && grouping == grouping_b => - { - let params = size_for_all_readers(&[a.params(), b.params()])?; - Some(FieldDataType::Sketch( - SketchKind::new(*a.algorithm(), params), - grouping.clone(), - )) - } - (FieldDataType::Sketch(..), _) | (_, FieldDataType::Sketch(..)) => None, - _ => (a == b).then(|| a.clone()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::schema::sketch::{GroupingStrategy, SketchAlgorithm, SketchParams}; - - fn kll(probability: Option) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 331, - failure_probability: probability, - }, - ), - GroupingStrategy::default(), - ) - } - - // State-size generation retains both local and shared capacities without - // depending on the removed implicit coupling-site representation. - #[test] - fn state_sizes_keep_each_distinct_reader_configuration() { - let small = SketchParams::Kll { - k: 100, - failure_probability: None, - }; - let large = SketchParams::Kll { - k: 200, - failure_probability: None, - }; - let configurations = state_size_configurations(vec![ - Some(small.clone()), - Some(large.clone()), - Some(small.clone()), - ]); - assert_eq!(configurations.len(), 2); - assert!(configurations.contains(&Some(small))); - assert!(configurations.contains(&Some(large))); - assert_eq!(state_size_configurations(vec![None, None]), vec![None]); - } - - // Equal capacity does not erase the stricter recorded failure probability. - #[test] - fn state_size_join_preserves_probability_metadata() { - let weak = SketchParams::Kll { - k: 331, - failure_probability: Some(0.01), - }; - let strict = SketchParams::Kll { - k: 331, - failure_probability: Some(0.001), - }; - let configurations = state_size_configurations(vec![Some(weak), Some(strict.clone())]); - assert!(configurations - .iter() - .flatten() - .any(|value| same_size(value, &strict))); - } - - // Equal physical capacities still carry distinct recorded error contracts; - // only the joined contract may serve the stricter probability requirement. - #[test] - fn tumbling_window_configuration_keeps_failure_probability_contract() { - let weaker = TumblingWindowConfiguration { - tumbling_window_width_ms: 1000, - longest_window_ms: 5000, - family: kll(None), - }; - let strict = TumblingWindowRequirement { - tumbling_window_width_ms: 1000, - window_ms: 5000, - family: kll(Some(0.001)), - }; - assert!(!weaker.serves(&strict)); - let configurations = tumbling_window_configurations(&[ - TumblingWindowRequirement { - family: kll(None), - ..strict.clone() - }, - strict.clone(), - ]); - assert!(configurations - .iter() - .any(|configuration| configuration.serves(&strict))); - assert!(configurations - .iter() - .all(|configuration| 1000_u64.is_multiple_of(configuration.tumbling_window_width_ms))); - } - - // Every integer-millisecond width that divides the grid is offered, not - // just the coarsest width or GCDs of the readers' original configurations. - #[test] - fn tumbling_window_width_domain_contains_every_legal_divisor() { - let reader = TumblingWindowRequirement { - tumbling_window_width_ms: 12, - window_ms: 60, - family: kll(None), - }; - let widths: BTreeSet<_> = tumbling_window_configurations(&[reader]) - .into_iter() - .map(|configuration| configuration.tumbling_window_width_ms) - .collect(); - assert_eq!(widths, BTreeSet::from([1, 2, 3, 4, 6, 12])); - } - - // Compare the divisor generator to direct enumeration, including zero. - #[test] - fn width_divisors_match_complete_integer_domain() { - for grid in 0..=256_u64 { - let expected: Vec<_> = (1..=grid) - .filter(|width| grid.is_multiple_of(*width)) - .collect(); - assert_eq!(tumbling_window_width_divisors(grid), expected); - } - } - - // Public finite enumeration rejects invalid grids instead of dividing by zero. - #[test] - fn invalid_tumbling_window_grid_has_no_configurations() { - assert!(tumbling_window_configurations(&[TumblingWindowRequirement { - tumbling_window_width_ms: 0, - window_ms: 5000, - family: kll(None) - }]) - .is_empty()); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs b/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs deleted file mode 100644 index e0c5bf94c..000000000 --- a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs +++ /dev/null @@ -1,1615 +0,0 @@ -//! Opt-in concrete summary-state groups for the candidate-connection planner. -//! -//! Simple summary builds and tumbling-window chains directly over canonical input groups -//! gain finite compatible sizes and tumbling-window horizons before exact state interning. -//! Concrete state identity includes capacity, guarantee, update/filter semantics, -//! and materialization. Supported private projections/filters gain explicit input groups. - -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; - -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId}; - -/// Why Pass 2's rules failed on a memo: its precondition check, a rule -/// (2–9), or the shared summary-state groups (10). -#[derive(Debug, thiserror::Error)] -pub enum Pass2RuleError { - #[error("memo group and candidate vectors have different lengths")] - MismatchedGroups, - #[error("memo contains an out-of-range group reference")] - InvalidGroupReference, - #[error("memo group capacity exceeds the group identifier range")] - TooManyGroups, - #[error("memo dependencies contain a cycle")] - CyclicDependencies, - #[error("explicit summary-state groups have already been introduced")] - AlreadyExpanded, - #[error("scan-label candidate expansion: {0}")] - ScanLabelWidening(String), - #[error("state configuration expansion: {0}")] - StateConfiguration(String), - #[error("producer reuse rule: {0}")] - ReuseRule(String), -} - -struct BuildReaders { - build: Rc, - children: Vec, - readers: Vec<(usize, usize, Rc)>, -} - -/// Rejects a memo the Pass 2 rules cannot extend: mismatched or -/// out-of-range group references, shared summary-state groups already -/// added, or cyclic dependencies. Pass 2 checks this before its rules. -pub(crate) fn check_unexpanded_memo(memo: &Memo) -> Result<(), Pass2RuleError> { - validate_references(memo)?; - if memo.groups.iter().any(|group| { - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) { - return Err(Pass2RuleError::AlreadyExpanded); - } - if memo.topological_order().is_none() { - return Err(Pass2RuleError::CyclicDependencies); - } - Ok(()) -} - -/// Pass 2's last step, after its rules: give each summary build that two or -/// more candidates make identically its own group, and add to each reader -/// group a candidate reading that group. Every existing candidate stays -/// (unsupported candidate shapes unchanged). Then rebuilds every group's -/// parents and the queries it may serve. On error, discard `memo`: it may -/// hold partial work. -pub(crate) fn add_shared_summary_state_groups(memo: &mut Memo) -> Result<(), Pass2RuleError> { - add_shared_summary_state_groups_with_index(memo, true) -} - -/// `indexed` looks builds up by fingerprint; otherwise by a linear scan -/// (the reference the index is tested against). -fn add_shared_summary_state_groups_with_index( - memo: &mut Memo, - indexed: bool, -) -> Result<(), Pass2RuleError> { - let mut builds: Vec = Vec::new(); - let mut build_index: HashMap, Vec> = HashMap::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - for (build, children) in direct_input_builds(memo, candidate) { - let fingerprint = build_fingerprint(&build); - let bucket = build_index.entry(fingerprint).or_default(); - let matching = if indexed { - bucket - .iter() - .copied() - .find(|&index| same_concrete_build(&builds[index].build, &build)) - } else { - builds - .iter() - .position(|entry| same_concrete_build(&entry.build, &build)) - }; - if let Some(build_index) = matching { - builds[build_index].readers.push((group, index, build)); - } else { - bucket.push(builds.len()); - builds.push(BuildReaders { - build: Rc::clone(&build), - children, - readers: vec![(group, index, build)], - }); - } - } - } - } - let mut state_groups = Vec::new(); - let mut replacements: BTreeMap<(usize, usize), HashMap<*const OperatorNode, Rc>> = - BTreeMap::new(); - for build in builds { - // Repeated private builds within one result are reusable too. The - // reader list records concrete build occurrences, not just owners. - if build.readers.len() < 2 { - continue; - } - let state_group = MemoGroupId( - u32::try_from(memo.groups.len()).map_err(|_| Pass2RuleError::TooManyGroups)?, - ); - let mut queries: Vec<_> = build - .readers - .iter() - .flat_map(|(group, _, _)| memo.groups[*group].queries.iter().copied()) - .collect(); - queries.sort_unstable(); - queries.dedup(); - memo.groups.push(MemoGroup { - logical_node: Rc::clone(&build.build), - child_groups: build.children.clone(), - parents: Vec::new(), - queries, - }); - // The build is its readers' own build, so it is sized for their - // shares of the budget (equal builds have equal sizes). - let accuracy_budget_parts = build - .readers - .iter() - .map(|(group, index, _)| memo.candidates[*group][*index].accuracy_budget_parts) - .max() - .unwrap_or(1); - memo.candidates.push(vec![CandidateForMemoGroup { - fused_groups: Vec::new(), - root_node: Rc::clone(&build.build), - input_groups: build.children, - origin: OriginOfCandidateForMemoGroup::SharedBuild, - accuracy_budget_parts, - }]); - for (group, index, own_build) in build.readers { - replacements - .entry((group, index)) - .or_default() - .insert(Rc::as_ptr(&own_build), Rc::clone(&build.build)); - } - state_groups.push(state_group); - } - for ((group, index), replacements) in replacements { - // Factor every available producer boundary together. A producer used by - // only one selected reader is still legal; no state-subset enumeration - // is required. Retain the original entirely local implementation too. - let mut reader = memo.candidates[group][index].clone(); - // Only declared inputs and newly lifted states are boundaries. A - // factored result group's representative can equal its candidate root; - // treating every memo node as an input would create a self-alias here. - let boundaries = reader - .input_groups - .iter() - .chain(&state_groups) - .map(|id| (Rc::as_ptr(&memo.groups[id.0 as usize].logical_node), *id)) - .collect(); - reader.root_node = super::rules::reuse::replace(&reader.root_node, &replacements); - reader.input_groups = boundary_groups(&reader.root_node, &boundaries); - reader.origin = OriginOfCandidateForMemoGroup::SharedBuild; - memo.candidates[group].push(reader); - } - // Parents describe every possible group/candidate dependency, not a selected plan. - for group in &mut memo.groups { - group.parents.clear(); - } - for group in 0..memo.groups.len() { - let dependencies: HashSet<_> = memo.groups[group] - .child_groups - .iter() - .chain( - memo.candidates[group] - .iter() - .flat_map(|candidate| &candidate.input_groups), - ) - .copied() - .collect(); - for child in dependencies { - memo.groups[child.0 as usize] - .parents - .push(MemoGroupId(group as u32)); - } - } - if memo.topological_order().is_none() { - return Err(Pass2RuleError::CyclicDependencies); - } - // The chosen representative input may now serve additional queries through - // state reuse; propagate potential reach along the expanded dependency DAG. - for group in memo - .topological_order() - .ok_or(Pass2RuleError::CyclicDependencies)? - .into_iter() - .rev() - { - let queries = memo.groups[group.0 as usize].queries.clone(); - let children: HashSet<_> = memo.groups[group.0 as usize] - .child_groups - .iter() - .chain( - memo.candidates[group.0 as usize] - .iter() - .flat_map(|candidate| &candidate.input_groups), - ) - .copied() - .collect(); - for child in children { - memo.groups[child.0 as usize].queries.extend(&queries); - memo.groups[child.0 as usize].queries.sort_unstable(); - memo.groups[child.0 as usize].queries.dedup(); - } - } - Ok(()) -} - -fn validate_references(memo: &Memo) -> Result<(), Pass2RuleError> { - if memo.groups.len() != memo.candidates.len() { - return Err(Pass2RuleError::MismatchedGroups); - } - if memo.groups.len() > u32::MAX as usize { - return Err(Pass2RuleError::TooManyGroups); - } - for group in &memo.groups { - if group - .child_groups - .iter() - .chain(&group.parents) - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(Pass2RuleError::InvalidGroupReference); - } - } - for candidate in memo.candidates.iter().flatten() { - if candidate - .input_groups - .iter() - .chain(&candidate.fused_groups) - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(Pass2RuleError::InvalidGroupReference); - } - } - Ok(()) -} - -pub(super) fn simple_build( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> Option<(Rc, Vec)> { - let mut builds = direct_input_builds(memo, candidate); - (builds.len() == 1).then(|| builds.remove(0)) -} - -pub(super) fn direct_input_builds( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> Vec<(Rc, Vec)> { - if !matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::TumblingWindows - | OriginOfCandidateForMemoGroup::WidenedScans - | OriginOfCandidateForMemoGroup::Rollup - | OriginOfCandidateForMemoGroup::SummaryResult - ) || candidate.accuracy_budget_parts == 0 - { - return Vec::new(); - } - let boundaries: HashMap<_, _> = candidate - .input_groups - .iter() - .map(|&group| { - ( - Rc::as_ptr(&memo.groups[group.0 as usize].logical_node), - group, - ) - }) - .collect(); - let mut stack = vec![Rc::clone(&candidate.root_node)]; - let mut seen = HashSet::new(); - let mut builds = Vec::new(); - while let Some(node) = stack.pop() { - if boundaries.contains_key(&Rc::as_ptr(&node)) || !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if !Rc::ptr_eq(&node, &candidate.root_node) - && matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) - { - let children: Option> = node - .children() - .iter() - .map(|child| boundaries.get(&Rc::as_ptr(child)).copied()) - .collect(); - if let Some(mut children) = children { - children.dedup(); - builds.push((Rc::clone(&node), children)); - } - } - stack.extend(node.children().into_iter().cloned()); - } - builds -} - -pub(super) fn boundary_groups( - node: &Rc, - boundaries: &HashMap<*const OperatorNode, MemoGroupId>, -) -> Vec { - let mut found = Vec::new(); - let mut stack = vec![Rc::clone(node)]; - let mut seen = HashSet::new(); - while let Some(node) = stack.pop() { - if !seen.insert(Rc::as_ptr(&node)) { - continue; - } - if let Some(&group) = boundaries.get(&Rc::as_ptr(&node)) { - if !found.contains(&group) { - found.push(group); - } - } else { - stack.extend(node.children().into_iter().rev().cloned()); - } - } - found -} - -/// The nodes of `candidate`'s input groups, where its own nodes end. -pub(super) fn boundary_ptrs( - memo: &Memo, - candidate: &CandidateForMemoGroup, -) -> HashSet<*const OperatorNode> { - candidate - .input_groups - .iter() - .map(|group| Rc::as_ptr(&memo.groups[group.0 as usize].logical_node)) - .collect() -} - -/// `root` and the nodes beneath it down to `boundaries` (excluded), each -/// once, in depth-first order (last child first). -pub(super) fn nodes_within( - root: &Rc, - boundaries: &HashSet<*const OperatorNode>, -) -> Vec> { - let mut pending = vec![Rc::clone(root)]; - let mut seen = HashSet::new(); - let mut nodes = Vec::new(); - while let Some(node) = pending.pop() { - if boundaries.contains(&Rc::as_ptr(&node)) || !seen.insert(Rc::as_ptr(&node)) { - continue; - } - pending.extend(node.children().into_iter().cloned()); - nodes.push(node); - } - nodes -} - -// Hashing narrows comparisons only; full strict equality below remains the -// authority, including probability metadata and floating-point corner cases. -#[derive(Hash, PartialEq, Eq)] -struct BuildFingerprint { - local: String, - inputs: Vec<(usize, Option)>, -} - -fn local_fingerprint(node: &OperatorNode) -> Option { - serde_json::to_string(&( - node.operator.map_children(|_| ()), - &node.schema, - node.result_kind, - &node.guarantee, - node.timing, - node.materialization, - )) - .ok() -} - -fn build_fingerprint(node: &OperatorNode) -> Option { - let inputs = node - .children() - .into_iter() - .map(|input| { - match input.non_asap() { - Some(NonASAPOp::TimeRange { child, .. }) => { - // Exactly the existing same_range_input contract: identical - // local range semantics over the same canonical source. - Some((Rc::as_ptr(child) as usize, Some(local_fingerprint(input)?))) - } - _ => Some((Rc::as_ptr(input) as usize, None)), - } - }) - .collect::>>()?; - Some(BuildFingerprint { - local: local_fingerprint(node)?, - inputs, - }) -} - -// Immediate inputs are canonical memo nodes. Compare local fields without -// recursively walking their potentially shared DAGs; serialization additionally -// distinguishes signed-zero parameters, matching conservative CSE semantics. -fn same_concrete_build(left: &OperatorNode, right: &OperatorNode) -> bool { - let left_inputs = left.children(); - let right_inputs = right.children(); - left_inputs.len() == right_inputs.len() - && left_inputs - .iter() - .zip(right_inputs) - .all(|(left, right)| Rc::ptr_eq(left, right) || same_range_input(left, right)) - && same_local_fields(left, right) -} - -// TimeRange lacks a unique row key and therefore is not generally memo-interned. -// Equal ranges over the same canonical input nevertheless feed identical data -// into a summary build, as the existing ASAP-aware summary reuse rule allows. -pub(super) fn same_range_input(left: &OperatorNode, right: &OperatorNode) -> bool { - let ( - Some(NonASAPOp::TimeRange { - child: left_child, .. - }), - Some(NonASAPOp::TimeRange { - child: right_child, .. - }), - ) = (left.non_asap(), right.non_asap()) - else { - return false; - }; - Rc::ptr_eq(left_child, right_child) && same_local_fields(left, right) -} - -fn same_local_fields(left: &OperatorNode, right: &OperatorNode) -> bool { - let signature = |node: &OperatorNode| { - ( - node.operator.map_children(|_| ()), - node.result_kind, - node.schema.clone(), - node.guarantee.clone(), - node.timing, - node.materialization, - ) - }; - let left = signature(left); - let right = signature(right); - left == right - && matches!((serde_json::to_string(&left), serde_json::to_string(&right)), (Ok(left), Ok(right)) if left == right) -} - -#[cfg(test)] -fn replace_node( - node: &Rc, - old: &Rc, - new: &Rc, -) -> Rc { - super::rules::reuse::replace(node, &HashMap::from([(Rc::as_ptr(old), Rc::clone(new))])) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::QueryRoot; - - /// An expanded memo with what its shared summary-state step added. - struct Expanded { - memo: Memo, - /// The groups the shared summary-state step added. - state_groups: Vec, - /// The candidates it added to reader groups. - added_reader_candidates: usize, - } - - /// Pass 2 after local window composition, over the complete tumbling-window-width - /// domain, as the Pass 2 entry runs it. - fn expand(memo: Memo) -> Result { - let memo = crate::test_support::sharing_rules_and_shared_state_groups(memo)?; - Ok(expanded(memo)) - } - - fn expanded(memo: Memo) -> Expanded { - let state_groups = crate::test_support::shared_summary_state_groups(&memo); - // Before this step no candidate is a shared build; after it, each - // state group has one and each added reader candidate is one. - let shared_builds = memo - .candidates - .iter() - .flatten() - .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild) - .count(); - Expanded { - added_reader_candidates: shared_builds - state_groups.len(), - state_groups, - memo, - } - } - - // Hash indexing is only an acceleration: independently run the original - // linear state lookup and compare every generated candidate and state. - #[test] - fn indexed_state_interning_preserves_linear_catalog() { - let memo = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[15m])", 0.02), - ("quantile_over_time(0.9, latency{job=\"other\"}[5m])", 0.01), - ]), - &[6, 6, 6], - ); - let mut memo = memo; - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo).unwrap(); - let mut linear = memo.clone(); - let mut indexed = memo; - add_shared_summary_state_groups_with_index(&mut indexed, true).unwrap(); - add_shared_summary_state_groups_with_index(&mut linear, false).unwrap(); - let (indexed, linear) = (expanded(indexed), expanded(linear)); - assert_eq!(indexed.state_groups, linear.state_groups); - assert_eq!(indexed.memo.groups.len(), linear.memo.groups.len()); - for (left, right) in indexed.memo.candidates.iter().zip(&linear.memo.candidates) { - let signatures = |candidates: &[CandidateForMemoGroup]| { - candidates - .iter() - .map(|candidate| { - ( - candidate.input_groups.clone(), - candidate.accuracy_budget_parts, - serde_json::to_string( - &asap_types::ir::flat::flatten(&[QueryRoot::Operator(Rc::clone( - &candidate.root_node, - ))]) - .0, - ) - .unwrap(), - ) - }) - .collect::>() - }; - assert_eq!(signatures(left), signatures(right)); - } - for (left, right) in indexed.memo.groups.iter().zip(&linear.memo.groups) { - assert_eq!(left.child_groups, right.child_groups); - assert_eq!(left.parents, right.parents); - assert_eq!(left.queries, right.queries); - } - } - - fn workload(queries: &[(&str, f64)]) -> Memo { - workload_with_identity(queries, false) - } - - fn workload_with_identity(queries: &[(&str, f64)], identity: bool) -> Memo { - let roots: Vec<_> = queries - .iter() - .map(|(query, error)| { - let root = crate::test_support::lower_promql( - query, - AccuracyTarget::max_error(*error).failure_probability(0.001), - ); - let root = if identity { - asap_types::ir::schema::schema_support::with_promql_series_identity(&root) - .unwrap() - } else { - root - }; - QueryRoot::Operator(root) - }) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let inventory = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - try_populate_pass1(initial, &inventory).unwrap() - } - - // Equal-sized quantiles share one canonical state while local choices remain. - #[test] - fn quantile_readers_share_concrete_state_and_retain_local_candidates() { - let original = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]); - let original_count = original.groups.len(); - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - assert!(expanded.added_reader_candidates >= 2); - let order = expanded.memo.topological_order().unwrap(); - for state in &expanded.state_groups { - let group = &expanded.memo.groups[state.0 as usize]; - assert_eq!(group.queries, vec![0, 1]); - for input in &group.child_groups { - assert_eq!(expanded.memo.groups[input.0 as usize].queries, vec![0, 1]); - assert!(group.logical_node.children().iter().any(|node| Rc::ptr_eq( - node, - &expanded.memo.groups[input.0 as usize].logical_node - ))); - } - assert_eq!(group.parents.len(), 2); - assert!(!asap_types::stages::root_groups(&expanded.memo.query_roots).contains(state)); - for &parent in &group.parents { - assert!( - order.iter().position(|id| id == state).unwrap() - < order.iter().position(|id| id == &parent).unwrap() - ); - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| candidate.input_groups.contains(state)) - .unwrap(); - assert!(OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| Rc::ptr_eq(node, &group.logical_node))); - assert_eq!(reader.input_groups, vec![*state]); - } - } - for group in 0..original_count { - for (before, after) in original.candidates[group] - .iter() - .zip(&expanded.memo.candidates[group]) - { - assert!(Rc::ptr_eq(&before.root_node, &after.root_node)); - assert_eq!(before.input_groups, after.input_groups); - } - } - assert!(matches!( - expand(expanded.memo), - Err(Pass2RuleError::AlreadyExpanded) - )); - } - - // A stricter concrete configuration can serve both readers; local sizes remain. - #[test] - fn distinct_sizes_gain_compatible_shared_configurations() { - let original = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.02), - ]); - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - for (group, candidates) in original.candidates.iter().enumerate() { - for candidate in candidates { - assert!(expanded.memo.candidates[group] - .iter() - .any(|after| Rc::ptr_eq(&candidate.root_node, &after.root_node))); - } - } - assert!(expanded - .state_groups - .iter() - .any(|id| expanded.memo.groups[id.0 as usize].queries == vec![0, 1])); - } - - // Resizing never makes different source predicates compatible. - #[test] - fn distinct_filters_do_not_share_concrete_builds() { - let original = workload(&[ - ("quantile_over_time(0.95, latency{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, latency{job=\"b\"}[5m])", 0.02), - ]); - let count = original.groups.len(); - let expanded = expand(original).unwrap(); - assert!(expanded.state_groups.is_empty()); - assert_eq!(expanded.memo.groups.len(), count); - } - - fn with_local_tumbling_windows(mut memo: Memo, grids: &[u64]) -> Memo { - use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_pass2_candidates, - }; - - let boundaries: HashMap<_, _> = memo - .groups - .iter() - .enumerate() - .map(|(id, group)| (Rc::as_ptr(&group.logical_node), MemoGroupId(id as u32))) - .collect(); - for group in 0..memo.groups.len() { - let grid = grids[memo.groups[group].queries[0]]; - for candidate in memo.candidates[group].clone() { - let roots = vec![QueryRoot::Operator(Rc::clone(&candidate.root_node))]; - let tumbling_windows = tumbling_window_candidates(&roots, &[Some(grid)]); - if tumbling_windows.len() != 1 { - continue; - } - let rewritten = window_composition_pass2_candidates(&roots, &tumbling_windows) - .unwrap() - .remove(0) - .roots - .unwrap(); - let QueryRoot::Operator(node) = &rewritten[0] else { - unreachable!() - }; - // Tumbling window rewriting recomposes guarantees; restore the original - // canonical scan boundary rather than making it a private input. - let input = tumbling_windows[0].build.children()[0].children()[0].clone(); - let chain = OperatorNode::reachable(node) - .into_iter() - .find(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - }) - .unwrap(); - let node = replace_node(node, chain.children()[0], &input); - let mut tumbling_window = candidate; - tumbling_window.root_node = Rc::clone(&node); - tumbling_window.input_groups = boundary_groups(&node, &boundaries); - tumbling_window.origin = OriginOfCandidateForMemoGroup::TumblingWindows; - memo.candidates[group].push(tumbling_window); - } - } - memo - } - - // Different label projections can consume one wider tumbling-window state through - // typed reader projections; no original scan needs to contain the union. - #[test] - fn disjoint_scan_labels_share_a_union_tumbling_window_state_with_typed_readers() { - let original = with_local_tumbling_windows( - workload(&[ - ("avg by (job) (quantile_over_time(0.95, latency[5m]))", 0.01), - ( - "avg by (region) (quantile_over_time(0.99, latency[5m]))", - 0.01, - ), - ]), - &[60_000, 60_000], - ); - assert!(!original.groups.iter().any(|group| matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Scan { .. }) - ) && group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == "job") - && group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == "region"))); - let local_nodes: Vec<_> = original - .candidates - .iter() - .map(|candidates| { - candidates - .iter() - .map(|candidate| Rc::clone(&candidate.root_node)) - .collect::>() - }) - .collect(); - let expanded = expand(original).unwrap(); - let shared = expanded - .state_groups - .iter() - .find(|id| { - let group = &expanded.memo.groups[id.0 as usize]; - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && group.queries == vec![0, 1] - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .expect("one state over union labels"); - let state = &expanded.memo.groups[shared.0 as usize]; - assert!(state.parents.len() >= 2); - for parent in &state.parents { - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| candidate.input_groups.contains(shared)) - .unwrap(); - assert_eq!( - reader.root_node.schema.fields, - expanded.memo.groups[parent.0 as usize] - .logical_node - .schema - .fields - ); - assert!(OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Project { .. })))); - reader.root_node.validate_structure().unwrap(); - } - for (group, locals) in local_nodes.into_iter().enumerate() { - assert!(locals - .iter() - .zip(&expanded.memo.candidates[group]) - .all(|(before, after)| Rc::ptr_eq(before, &after.root_node))); - } - assert!(expanded.memo.topological_order().is_some()); - assert!(state - .child_groups - .iter() - .all(|child| expanded.memo.groups[child.0 as usize].queries == vec![0, 1])); - } - - // Widening shifts identity's position but must retain its full typed field - // and remap the source's unique key rather than treating it as a plain label. - #[test] - fn widened_tumbling_window_inputs_preserve_series_identity_and_keys() { - let memo = with_local_tumbling_windows( - workload_with_identity( - &[ - ("avg by (job) (quantile_over_time(0.95, latency[5m]))", 0.01), - ( - "avg by (region) (quantile_over_time(0.99, latency[5m]))", - 0.01, - ), - ], - true, - ), - &[60_000, 60_000], - ); - let expanded = expand(memo).unwrap(); - let state = expanded - .state_groups - .iter() - .map(|id| &expanded.memo.groups[id.0 as usize]) - .find(|group| { - matches!( - group.logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && group.queries == vec![0, 1] - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .expect("identity-carrying union tumbling-window state"); - let input = state.logical_node.children()[0]; - assert!(input.schema.has_promql_series_identity()); - let identity = input - .schema - .column_id(asap_types::ir::schema::PROMQL_SERIES_IDENTITY) - .unwrap(); - let time = input.schema.time_index.unwrap(); - assert!(input - .schema - .unique_keys - .iter() - .any(|key| key.contains(&identity) && key.contains(&time))); - for parent in &state.parents { - let reader = expanded.memo.candidates[parent.0 as usize] - .iter() - .find(|candidate| { - candidate.input_groups.contains(&MemoGroupId( - expanded - .memo - .groups - .iter() - .position(|group| Rc::ptr_eq(&group.logical_node, &state.logical_node)) - .unwrap() as u32, - )) - }) - .unwrap(); - assert_eq!( - reader.root_node.schema, - expanded.memo.groups[parent.0 as usize].logical_node.schema - ); - } - } - - /// The groups `group`'s candidates read, directly or through the - /// candidates of the groups they read. - fn groups_read_by(memo: &Memo, group: usize) -> HashSet { - let mut read = HashSet::new(); - let mut pending = vec![group]; - while let Some(group) = pending.pop() { - for input in memo.candidates[group] - .iter() - .flat_map(|candidate| &candidate.input_groups) - { - if read.insert(input.0 as usize) { - pending.push(input.0 as usize); - } - } - } - read - } - - // A scan (or time range) that already has every label of its metric is - // not widened: widening recomposes guarantees, and an unwidened root - // that differs from the original only in its guarantee annotation must - // not become an identity `Project` over its own group. - #[test] - fn scan_label_widening_never_reads_its_own_group() { - for queries in [ - [ - ("sum by (job, region) (m)", 0.01), - ("sum by (job) (m)", 0.01), - ], - [ - ("sum by (job, region) (rate(m[5m]))", 0.01), - ("sum by (job) (rate(m[5m]))", 0.01), - ], - ] { - let mut memo = workload_with_identity(&queries, true); - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo).unwrap(); - assert!( - memo.candidates.iter().flatten().any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::WidenedScans - ), - "{queries:?}: the narrower scan is widened" - ); - for group in 0..memo.groups.len() { - assert!( - !groups_read_by(&memo, group).contains(&group), - "{queries:?}: group {group} reads itself: {:?}", - memo.candidates[group] - .iter() - .map(|candidate| (&candidate.origin, &candidate.input_groups)) - .collect::>() - ); - } - let expanded = expand(workload_with_identity(&queries, true)).unwrap(); - for group in 0..expanded.memo.groups.len() { - assert!(!groups_read_by(&expanded.memo, group).contains(&group)); - } - } - } - - // A candidate reading its own group is a cycle the memo rejects. - #[test] - fn a_group_reading_itself_is_a_cyclic_memo() { - let mut memo = workload(&[("sum by (job) (m)", 0.01)]); - assert!(check_unexpanded_memo(&memo).is_ok()); - let mut reader = memo.candidates[0][0].clone(); - reader.input_groups = vec![MemoGroupId(0)]; - memo.candidates[0].push(reader); - assert_eq!(memo.topological_order(), None); - assert!(matches!( - check_unexpanded_memo(&memo), - Err(Pass2RuleError::CyclicDependencies) - )); - } - - // Label widening must not erase source predicates in order to create reuse. - #[test] - fn scan_label_widening_keeps_different_filters_separate() { - let original = with_local_tumbling_windows( - workload(&[ - ( - "avg by (job) (quantile_over_time(0.95, latency{env=\"a\"}[5m]))", - 0.01, - ), - ( - "avg by (region) (quantile_over_time(0.99, latency{env=\"b\"}[5m]))", - 0.01, - ), - ]), - &[60_000, 60_000], - ); - let expanded = expand(original).unwrap(); - assert!(expanded - .state_groups - .iter() - .all(|id| expanded.memo.groups[id.0 as usize].queries.len() == 1)); - } - - // Real tumbling window rewrites expose one identical chain to distinct quantile readers. - #[test] - fn identical_local_tumbling_window_chains_share_state_without_removing_local_choices() { - let original = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]), - &[60_000, 60_000], - ); - let local_tumbling_windows = original - .candidates - .iter() - .flatten() - .filter(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows) - .count(); - assert!(local_tumbling_windows >= 2); - let expanded = expand(original.clone()).unwrap(); - let chains: Vec<_> = expanded - .state_groups - .iter() - .filter(|id| { - matches!( - expanded.memo.groups[id.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .collect(); - assert!(!chains.is_empty()); - for id in chains { - let group = &expanded.memo.groups[id.0 as usize]; - assert_eq!(group.queries, vec![0, 1]); - assert_eq!(group.parents.len(), 2); - assert!(group.child_groups.iter().all(|input| group - .logical_node - .children() - .iter() - .any(|node| Rc::ptr_eq( - node, - &expanded.memo.groups[input.0 as usize].logical_node - )))); - } - for (group, candidates) in original.candidates.iter().enumerate() { - for original in candidates { - assert!(expanded.memo.candidates[group] - .iter() - .any(|candidate| Rc::ptr_eq(&original.root_node, &candidate.root_node))); - } - } - assert!(matches!( - expand(expanded.memo), - Err(Pass2RuleError::AlreadyExpanded) - )); - } - - // A finer grid and longer horizon serves a shorter reader through newest_tumbling_windows. - #[test] - fn different_tumbling_window_grids_and_horizons_gain_explicit_newest_tumbling_window_readers() { - let original = with_local_tumbling_windows( - workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[15m])", 0.02), - ]), - &[60_000, 30_000], - ); - let expanded = expand(original.clone()).unwrap(); - let state = expanded - .state_groups - .iter() - .find(|id| { - matches!( - expanded.memo.groups[id.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 30_000, - tumbling_window_count: 30, - .. - }) - ) - }) - .expect("shared fine grid covering fifteen minutes"); - assert_eq!(expanded.memo.groups[state.0 as usize].queries, vec![0, 1]); - assert!(expanded.memo.candidates.iter().flatten().any(|reader| { - reader.input_groups.contains(state) - && OperatorNode::reachable(&reader.root_node) - .iter() - .any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(10), - .. - }) - ) - }) - })); - for (group, candidates) in original.candidates.iter().enumerate() { - assert!(candidates - .iter() - .all(|before| expanded.memo.candidates[group] - .iter() - .any(|after| Rc::ptr_eq(&before.root_node, &after.root_node)))); - } - assert!(expanded.memo.topological_order().is_some()); - } - - // Rollup readers preserve original outputs while referencing the finer - // query's finalized values through an ordinary child-group dependency: - // the finer query's own result group, whose result is those values. - #[test] - fn exact_rollup_exposes_shared_finalized_values() { - let original = workload(&[ - ("sum by (job) (m)", 0.01), - ("sum by (job, instance) (m)", 0.01), - ]); - let expanded = expand(original).unwrap(); - let memo = &expanded.memo; - let fine = memo.query_roots[1].as_operator().unwrap().0 as usize; - assert!(memo.candidates[fine].iter().all(|candidate| matches!( - candidate.root_node.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - ))); - assert!(memo - .candidates - .iter() - .enumerate() - .filter(|(group, _)| *group != fine) - .flat_map(|(_, candidates)| candidates) - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::Rollup - && candidate.input_groups.contains(&MemoGroupId(fine as u32)) - )); - assert!(memo.topological_order().is_some()); - } - - // Sketch rollup exposes the resized finer state through an ordinary group - // dependency, while each output retains its own grouping and quantile. - #[test] - fn sketch_rollup_exposes_shared_finer_state() { - let original = workload(&[ - ("quantile(0.9, m)", 0.01), - ("quantile by (job) (0.9, m)", 0.02), - ]); - let expanded = expand(original).unwrap(); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - )) - )); - assert!(expanded.memo.topological_order().is_some()); - } - - // Different k outputs share one sorted ranking, backed by a recursively - // exact candidate group rather than an unenforced exact-input annotation. - #[test] - fn topk_outputs_share_explicit_exact_ranking() { - let original = workload(&[ - ("topk by (job) (3, sum_over_time(m[1m]))", 0.01), - ("topk by (job) (5, sum_over_time(m[1m]))", 0.01), - ]); - let expanded = expand(original).unwrap(); - assert!(expanded.memo.groups.iter().any(|group| matches!( - group.logical_node.non_asap(), - Some(NonASAPOp::Limit { n: Some(5), .. }) - ) && group.parents.len() >= 2)); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any(|candidate| candidate.origin == OriginOfCandidateForMemoGroup::TopKReuse)); - assert!(expanded.memo.topological_order().is_some()); - } - - // Widened source-group candidates make shared scans available even to - // ordinary operators that do not build an ASAP summary. - #[test] - fn general_source_groups_offer_typed_union_scan_readers() { - let original = workload(&[ - ("topk by (job) (3, m)", 0.01), - ("topk by (region) (5, m)", 0.01), - ]); - let expanded = expand(original.clone()).unwrap(); - let union = expanded - .memo - .groups - .iter() - .enumerate() - .find(|(_, group)| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::Scan { .. })) - && ["job", "region"].iter().all(|name| { - group - .logical_node - .schema - .fields - .iter() - .any(|field| field.name == *name) - }) - }) - .map(|(id, _)| MemoGroupId(id as u32)) - .expect("union scan"); - let readers = original - .groups - .iter() - .enumerate() - .filter(|(_, group)| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::Scan { .. })) - }) - .filter(|(group, _)| { - expanded.memo.candidates[*group].iter().any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::WidenedScans - && candidate.input_groups.contains(&union) - }) - }) - .count(); - assert!(readers >= 2); - } - - // A composite average can share only its sum state with another query while - // retaining the count state locally; reader budget allocation is preserved. - #[test] - fn multiple_private_states_can_share_only_the_common_producer() { - let mut original = workload(&[("avg(m)", 0.01), ("sum(m)", 0.01)]); - let average = original.query_roots[0].as_operator().unwrap().0 as usize; - assert!(original.candidates[average] - .iter() - .any(|candidate| direct_input_builds(&original, candidate).len() >= 2)); - for candidates in &mut original.candidates { - for candidate in candidates { - candidate.accuracy_budget_parts = 2; - } - } - let expanded = expand(original.clone()).unwrap(); - assert!(!expanded.state_groups.is_empty()); - let reader = expanded.memo.candidates[average] - .iter() - .find(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && candidate - .input_groups - .iter() - .any(|group| expanded.state_groups.contains(group)) - }) - .expect("average has shared state input"); - assert_eq!(reader.accuracy_budget_parts, 2); - // The shared build is the readers' own, sized for two shares too. - for state in &expanded.state_groups { - assert_eq!( - expanded.memo.candidates[state.0 as usize][0].accuracy_budget_parts, - 2 - ); - } - assert!( - reader - .input_groups - .iter() - .any(|id| expanded.state_groups.contains(id) - && expanded.memo.groups[id.0 as usize].queries == vec![0, 1]), - "sum state is reusable by both queries" - ); - assert!( - reader - .input_groups - .iter() - .any(|id| expanded.state_groups.contains(id) - && expanded.memo.groups[id.0 as usize].queries == vec![0]), - "count state belongs only to the average query" - ); - assert!(original.candidates[average] - .iter() - .all(|before| expanded.memo.candidates[average] - .iter() - .any(|after| Rc::ptr_eq(&before.root_node, &after.root_node)))); - } - - // A global SQL-window quantile can consume newest tumbling windows of a longer, - // finer-grouped chain through the existing regrouping rule. - #[test] - fn grouped_window_rollup_exposes_one_finer_chain() { - use crate::test_support::{agg, scan_from, time_range}; - use asap_types::ir::operator::{AggIntent, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - let scan = || { - scan_from( - Source::Table { - table_ref: "metrics".into(), - }, - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - ) - }; - let quantile = |q| AggIntent::Quantile { - col: Some(2), - q, - accuracy: AccuracyTarget::max_error(0.01), - }; - let roots = vec![ - QueryRoot::Operator(agg( - vec![], - quantile(0.5), - time_range(std::time::Duration::from_secs(300), scan()), - )), - QueryRoot::Operator(agg( - vec![1], - quantile(0.9), - time_range(std::time::Duration::from_secs(1200), scan()), - )), - ]; - let initial = generate_dag_of_logical_memo_groups(&roots); - let generated = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = with_local_tumbling_windows( - try_populate_pass1(initial, &generated).unwrap(), - &[10_000, 10_000], - ); - let expanded = expand(memo).unwrap(); - assert!(expanded - .memo - .candidates - .iter() - .flatten() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && OperatorNode::reachable(&candidate.root_node) - .iter() - .any(|node| matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { - group_by: Some(_), - .. - }) - )) - && candidate.input_groups.iter().any(|group| matches!( - expanded.memo.groups[group.0 as usize].logical_node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count: 120, - .. - }) - )) - )); - } - - // Private identity projections retain their full typed contract, but no - // longer hide compatible builds from canonical state sharing. - #[test] - fn private_projection_inputs_gain_canonical_shared_states() { - check_private_inputs(false); - } - - // Key-only HAVING uses private Filter nodes below summary builds; equal - // filters can share without dropping their predicates. - #[test] - fn private_filter_inputs_gain_canonical_shared_states() { - check_private_inputs(true); - } - - fn check_private_inputs(filter: bool) { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::{Operator, Predicate, ProjectItem, ScalarExpr}; - let mut memo = workload(&[ - ("quantile_over_time(0.95, latency[5m])", 0.01), - ("quantile_over_time(0.99, latency[5m])", 0.01), - ]); - let mut injected = 0; - for owner in 0..memo.candidates.len() { - let originals = memo.candidates[owner].clone(); - for mut candidate in originals { - let builds = direct_input_builds(&memo, &candidate); - if builds.is_empty() { - continue; - } - let mut replacements = HashMap::new(); - for (build, _) in builds { - let input = build.children()[0]; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: input - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: Rc::clone(input), - })) - .unwrap(); - let project = if filter { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - child: project, - })) - .unwrap() - } else { - project - }; - replacements.insert(Rc::as_ptr(input), project); - } - candidate.root_node = - crate::pass2::memo::rules::reuse::replace(&candidate.root_node, &replacements); - assert!(direct_input_builds(&memo, &candidate).is_empty()); - memo.candidates[owner].push(candidate); - injected += 1; - } - } - assert!(injected >= 2); - let expanded = expand(memo).unwrap(); - assert!(expanded.state_groups.iter().any(|state| { - let group = &expanded.memo.groups[state.0 as usize]; - group.queries == vec![0, 1] - && group.child_groups.iter().any(|input| { - expanded.memo.candidates[input.0 as usize] - .iter() - .any(|candidate| { - candidate.origin == OriginOfCandidateForMemoGroup::SummaryInput - }) - }) - })); - assert!(expanded.memo.topological_order().is_some()); - } - - // Real identity-aware frequency candidates retain UnivMon inputs and their - // error metadata when private preprocessing is made an explicit boundary. - #[test] - fn univmon_identity_inputs_remain_admissible_after_factoring() { - use asap_types::ir::schema::{sketch::SketchAlgorithm, FieldDataType}; - let mut memo = workload_with_identity(&[("count_over_time(latency[5m])", 0.01)], true); - let is_univmon = |node: &OperatorNode| matches!(node.asap(), Some(ASAPOp::SummaryAgg {family: FieldDataType::Sketch(kind, _), ..}) if kind.algorithm() == &SketchAlgorithm::UnivMon); - let before: Vec<_> = memo - .candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .filter(|node| is_univmon(node)) - .collect(); - assert!(!before.is_empty(), "real generation must offer UnivMon"); - crate::pass2::memo::rules::summary_inputs::factor_summary_inputs(&mut memo).unwrap(); - assert!(memo - .candidates - .iter() - .flatten() - .any(|candidate| direct_input_builds(&memo, candidate) - .iter() - .any(|(node, _)| is_univmon(node)))); - for original in before { - assert!( - memo.candidates - .iter() - .flatten() - .flat_map(|candidate| OperatorNode::reachable(&candidate.root_node)) - .any(|node| Rc::ptr_eq(&original, &node)), - "local generated candidates remain intact" - ); - } - } - - // Reader deduplication must retain distinct recorded probability contracts - // even though SketchParams structural equality intentionally ignores them. - #[test] - fn same_capacity_readers_retain_distinct_failure_probability_contracts() { - use asap_types::ir::schema::{ - sketch::{SketchAlgorithm, SketchKind, SketchParams}, - FieldDataType, - }; - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - let (owner, mut base, build) = memo.candidates.iter().enumerate().find_map(|(owner, candidates)| candidates.iter().find_map(|candidate| { - let (build, _) = simple_build(&memo, candidate)?; - matches!(build.asap(), Some(ASAPOp::SummaryAgg {family: FieldDataType::Sketch(kind, _), ..}) if kind.algorithm() == &SketchAlgorithm::Kll).then(|| (owner, candidate.clone(), build)) - })).unwrap(); - let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - }) = build.asap() - else { - unreachable!() - }; - let SketchParams::Kll { k, .. } = kind.params() else { - unreachable!() - }; - let boundaries: HashSet<_> = base - .input_groups - .iter() - .map(|id| Rc::as_ptr(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn without_private_guarantees( - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - ) -> Rc { - if boundaries.contains(&Rc::as_ptr(node)) { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| without_private_guarantees(child, boundaries)), - guarantee: None, - ..(**node).clone() - }) - } - let rewritten = |probability| { - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: *k, - failure_probability: Some(probability), - }, - ), - grouping.clone(), - ); - let roots = crate::pass2::asap_aware_cse::with_resized_build( - &[QueryRoot::Operator(Rc::clone(&base.root_node))], - &build, - family, - ) - .unwrap(); - let QueryRoot::Operator(root) = &roots[0] else { - unreachable!() - }; - let root = - crate::pass2::memo::rules::reuse::restore_inputs(root, &memo, &base.input_groups); - without_private_guarantees(&root, &boundaries) - }; - let weak = rewritten(0.001); - let strict = rewritten(0.0001); - assert_eq!( - *weak, *strict, - "ordinary equality omits the probability metadata" - ); - base.root_node = weak; - memo.candidates[owner] = vec![base.clone()]; - assert!(crate::pass2::memo::rules::reuse::append_reader( - &mut memo, - owner, - &base, - strict, - OriginOfCandidateForMemoGroup::Rollup - ) - .unwrap()); - assert_eq!(memo.candidates[owner].len(), 2); - } - - // Equal private producers can be shared within one composite result; they - // do not need to originate from different memo owner groups. - #[test] - fn repeated_states_within_one_owner_share_a_producer() { - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - let (owner, mut candidate) = memo - .candidates - .iter() - .enumerate() - .find_map(|(group, candidates)| { - candidates - .iter() - .find(|candidate| simple_build(&memo, candidate).is_some()) - .map(|candidate| (group, candidate.clone())) - }) - .unwrap(); - let boundaries: HashSet<_> = candidate - .input_groups - .iter() - .map(|id| Rc::as_ptr(&memo.groups[id.0 as usize].logical_node)) - .collect(); - fn duplicate( - node: &Rc, - boundaries: &HashSet<*const OperatorNode>, - ) -> Rc { - if boundaries.contains(&Rc::as_ptr(node)) { - return Rc::clone(node); - } - Rc::new(OperatorNode { - operator: node - .operator - .map_children(|child| duplicate(child, boundaries)), - ..(**node).clone() - }) - } - let second = duplicate(&candidate.root_node, &boundaries); - let combined = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![Rc::clone(&candidate.root_node), second], - })) - .unwrap(); - candidate.root_node = Rc::clone(&combined); - memo.groups[owner].logical_node = combined; - memo.candidates[owner] = vec![candidate]; - let expanded = expand(memo).unwrap(); - assert_eq!(expanded.state_groups.len(), 1); - let state = expanded.state_groups[0]; - assert!(expanded.memo.groups[state.0 as usize] - .parents - .contains(&MemoGroupId(owner as u32))); - assert!(expanded.memo.candidates[owner] - .iter() - .any( - |candidate| candidate.origin == OriginOfCandidateForMemoGroup::SharedBuild - && candidate.input_groups.contains(&state) - )); - } - - // Public expansion rejects malformed IDs before indexing dependency arrays. - #[test] - fn malformed_references_fail_before_dependency_traversal() { - let mut memo = workload(&[("quantile_over_time(0.95, latency[5m])", 0.01)]); - memo.groups[0].child_groups.push(MemoGroupId(u32::MAX)); - assert!(matches!( - expand(memo), - Err(Pass2RuleError::InvalidGroupReference) - )); - } -} diff --git a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs index 468de54d5..62594119f 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs @@ -1,315 +1,18 @@ -//! Local tumbling-window candidates generated from compatible reader grids. Selected -//! connections, not the union of potential consumers, determine eligibility. -#[cfg(test)] -use super::shared_configurations::tumbling_window_width_divisors; -use super::shared_configurations::tumbling_window_width_domain; -use crate::pass2::asap_aware_cse::{ - tumbling_window_candidates, window_composition_pass2_candidates, -}; -use asap_types::ir::flat::flatten; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -use asap_types::stages::{MemoGroup, MemoGroupId as GroupId}; -use asap_types::workload::QueryWorkloadEntry; -#[cfg(test)] -use std::collections::BTreeSet; -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; +//! Rule 1, local window composition: tumbling-window layouts $\omega_v$ as +//! candidates of a summary state group (#1160). -/// Offer each supported reader-grid configuration, for every legal -/// tumbling-window width. Stage 2 checks the actual chosen consumers, -/// including ingestion alignment and range evaluation steps. -pub fn add_local_window_composition(memo: &mut Memo, entries: &[QueryWorkloadEntry]) { - // Flatten each existing/generated graph once. Repeatedly flattening the - // growing candidate list makes a complete tumbling-window-width domain quadratic. - let mut known: HashMap<(usize, String), Vec> = HashMap::new(); - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - if let Ok(signature) = candidate_signature(candidate) { - known.entry((group, signature)).or_default().push(index); - } - } - } - for (group, candidate) in local_tumbling_window_candidates(memo, entries) { - let Ok(signature) = candidate_signature(&candidate) else { - memo.candidates[group].push(candidate); - continue; - }; - let bucket = known.entry((group, signature)).or_default(); - if !bucket.iter().any(|&index| { - let previous = &memo.candidates[group][index]; - previous.origin == candidate.origin - && previous.input_groups == candidate.input_groups - && previous.fused_groups == candidate.fused_groups - && previous.accuracy_budget_parts == candidate.accuracy_budget_parts - && previous.root_node == candidate.root_node - }) { - bucket.push(memo.candidates[group].len()); - memo.candidates[group].push(candidate); - } - } -} +use crate::pass2::memo::rules::Pass2RuleError; +use asap_types::stages::Memo; +use asap_types::workload::QueryWorkloadEntry; -/// Expand legal tumbling-window widths while preserving canonical memo inputs. -fn local_tumbling_window_candidates( - memo: &Memo, +/// Rule 1: for each summary state group, a candidate per tumbling width $w$ +/// from the gcd closure of the ranges and steps of the queries that reach it, +/// offered only when $w$ divides the group's query range. `entries[i]` is +/// query `i`'s workload entry. +pub(crate) fn add_local_window_composition( + memo: &mut Memo, entries: &[QueryWorkloadEntry], -) -> Vec<(usize, CandidateForMemoGroup)> { - let group_of = group_of_node(&memo.groups); - let mut own: Vec<(usize, CandidateForMemoGroup)> = Vec::new(); - for (g, group) in memo.groups.iter().enumerate() { - let grids = tumbling_window_width_domain( - group - .queries - .iter() - .filter_map(|&query| entries.get(query)?.evaluation_grid_ms()), - ); - for grid in grids { - for candidate in &memo.candidates[g] { - if !matches!( - candidate.origin, - OriginOfCandidateForMemoGroup::Pass1 - | OriginOfCandidateForMemoGroup::SharedBuild - ) { - continue; - } - let canonical_scans = OperatorNode::reachable(&candidate.root_node) - .into_iter() - .filter(|node| { - matches!( - node.non_asap(), - Some(asap_types::ir::NonASAPOp::Scan { .. }) - ) && group_of.contains_key(&Rc::as_ptr(node)) - }) - .collect::>(); - let roots = [QueryRoot::Operator(Rc::clone(&candidate.root_node))]; - let tumbling_windows = tumbling_window_candidates(&roots, &[Some(grid)]); - if tumbling_windows.is_empty() { - continue; - } - let Ok(pass2_candidates) = - window_composition_pass2_candidates(&roots, &tumbling_windows) - else { - continue; - }; - for pass2_candidate in pass2_candidates { - let Ok(rewritten) = pass2_candidate.roots else { - continue; - }; - let [QueryRoot::Operator(node)] = &rewritten[..] else { - continue; - }; - // Guarantee recomposition can clone unchanged source scans. - // Restore only an unambiguous original scan with the same full - // source/schema/execution contract; scans have intrinsic exact - // semantics. Otherwise the tumbling window's canonical input disappears. - let mut restored = HashMap::new(); - for rewritten in OperatorNode::reachable(node) { - let mut matches = canonical_scans.iter().filter(|original| { - original.operator == rewritten.operator - && original.schema == rewritten.schema - && original.result_kind == rewritten.result_kind - && original.timing == rewritten.timing - && original.materialization == rewritten.materialization - }); - if let Some(original) = matches.next() { - if matches.next().is_none() { - restored.insert(Rc::as_ptr(&rewritten), original); - } - } - } - let node = restore_scan_boundaries(node, &restored, &mut HashMap::new()); - own.push(( - g, - rewritten_candidate( - &group_of, - GroupId(g as u32), - candidate, - node, - OriginOfCandidateForMemoGroup::TumblingWindows, - ), - )); - } - } - } - } - own -} - -fn restore_scan_boundaries( - node: &Rc, - restored: &HashMap<*const OperatorNode, &Rc>, - done: &mut HashMap<*const OperatorNode, Rc>, -) -> Rc { - if let Some(scan) = restored.get(&Rc::as_ptr(node)) { - return Rc::clone(scan); - } - if let Some(previous) = done.get(&Rc::as_ptr(node)) { - return Rc::clone(previous); - } - let mut changed = false; - let operator = node.operator.map_children(|child| { - let rebuilt = restore_scan_boundaries(child, restored, done); - changed |= !Rc::ptr_eq(child, &rebuilt); - rebuilt - }); - let rebuilt = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - done.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt -} - -// Union of complete divisor domains covers individual readers and every -// compatible subset: common widths belong to each reader's divisor domain. -// Nongridded potential consumers do not veto a repeated reader's tumbling windows. -#[cfg(test)] -fn compatible_reader_grids(group: &MemoGroup, entries: &[QueryWorkloadEntry]) -> Vec { - super::shared_configurations::tumbling_window_width_domain( - group - .queries - .iter() - .filter_map(|&query| entries.get(query)?.evaluation_grid_ms()), - ) -} - -fn group_of_node(groups: &[MemoGroup]) -> HashMap<*const OperatorNode, usize> { - groups - .iter() - .enumerate() - .map(|(g, group)| (Rc::as_ptr(&group.logical_node), g)) - .collect() -} - -/// `from`, a candidate of group `owner`, with root `node` and `origin`. -fn rewritten_candidate( - group_of: &HashMap<*const OperatorNode, usize>, - owner: GroupId, - from: &CandidateForMemoGroup, - node: Rc, - origin: OriginOfCandidateForMemoGroup, -) -> CandidateForMemoGroup { - let implements: Vec = std::iter::once(owner) - .chain(from.fused_groups.iter().copied()) - .collect(); - let reached = |node: &Rc| child_groups(group_of, node, &implements); - let children = if reached(&from.root_node) == from.input_groups { - reached(&node) - } else { - from.input_groups.clone() - }; - CandidateForMemoGroup { - fused_groups: from.fused_groups.clone(), - root_node: node, - input_groups: children, - origin, - accuracy_budget_parts: from.accuracy_budget_parts, - } -} - -fn child_groups( - group_of: &HashMap<*const OperatorNode, usize>, - node: &Rc, - implements: &[GroupId], -) -> Vec { - fn walk( - node: &Rc, - group_of: &HashMap<*const OperatorNode, usize>, - implements: &[GroupId], - seen: &mut HashSet<*const OperatorNode>, - out: &mut Vec, - ) { - for child in node.children() { - if !seen.insert(Rc::as_ptr(child)) { - continue; - } - match group_of.get(&Rc::as_ptr(child)) { - Some(&g) if !implements.contains(&GroupId(g as u32)) => { - if !out.contains(&GroupId(g as u32)) { - out.push(GroupId(g as u32)); - } - } - _ => walk(child, group_of, implements, seen, out), - } - } - } - let mut out = Vec::new(); - walk(node, group_of, implements, &mut HashSet::new(), &mut out); - out -} - -fn candidate_signature(candidate: &CandidateForMemoGroup) -> Result { - serde_json::to_string(&flatten(&[QueryRoot::Operator(Rc::clone(&candidate.root_node))]).0) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::memo_population::try_populate_pass1; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use crate::stage0::generate_dag_of_logical_memo_groups; - use crate::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - fn repeated(interval: u32) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(interval))) - } - - fn fixture(recurrences: &[QueryRecurrence]) -> (Memo, Vec) { - let target = AccuracyTarget::max_error(0.05).failure_probability(0.1); - let root = lower_promql("quantile_over_time(0.95, latency[15m])", target.clone()); - let roots: Vec<_> = recurrences - .iter() - .map(|_| QueryRoot::Operator(Rc::clone(&root))) - .collect(); - let initial = generate_dag_of_logical_memo_groups(&roots); - let generated = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&roots)) - .unwrap(); - let memo = try_populate_pass1(initial, &generated).unwrap(); - let entries = recurrences - .iter() - .map(|recurrence| entry_with(target.clone(), recurrence.clone())) - .collect(); - (memo, entries) - } - - // A nongridded potential reader cannot eliminate another reader's tumbling windows. - #[test] - fn nongridded_potential_reader_preserves_local_tumbling_window_candidates() { - let (memo, entries) = fixture(&[repeated(60_000), QueryRecurrence::Unknown]); - let candidates = local_tumbling_window_candidates(&memo, &entries); - assert!(!candidates.is_empty()); - assert!(candidates.iter().all( - |(_, candidate)| candidate.origin == OriginOfCandidateForMemoGroup::TumblingWindows - )); - } - - // Individual grids and subset GCDs allow separate or shared physical choices. - #[test] - fn reader_grids_include_subsets_instead_of_only_full_union() { - let (memo, entries) = fixture(&[ - repeated(120_000), - repeated(180_000), - QueryRecurrence::Unknown, - ]); - let group = &memo.groups[memo.query_roots[0].as_operator().unwrap().0 as usize]; - assert_eq!( - compatible_reader_grids(group, &entries), - tumbling_window_width_divisors(120_000) - .into_iter() - .chain(tumbling_window_width_divisors(180_000)) - .collect::>() - .into_iter() - .collect::>() - ); - } +) -> Result<(), Pass2RuleError> { + let _ = (memo, entries); + todo!("#1207 PR 4: rule 1, tumbling layouts as candidates of the state group") } diff --git a/crates/logical-asap-optimization/src/stage0.rs b/crates/logical-asap-optimization/src/stage0.rs index c26a9dc2e..d2749a7dd 100644 --- a/crates/logical-asap-optimization/src/stage0.rs +++ b/crates/logical-asap-optimization/src/stage0.rs @@ -1,5 +1,5 @@ //! Stage 0: the memo groups of the workload's query roots -//! ([`DagOfLogicalMemoGroups`]), by hash-consing identical logical +//! (a [`Memo`] without candidates), by hash-consing identical logical //! sub-expressions (the `cse` module). Stage 1 starts from these groups. /// Structural common-subexpression elimination (hash-consing) over the @@ -11,12 +11,12 @@ use std::rc::Rc; use crate::stage0::cse::share_common_sub_dags; use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{DagOfLogicalMemoGroups, MemoGroup, MemoGroupId}; +use asap_types::stages::{Memo, MemoGroup, MemoGroupId}; /// Stage 0: the memo groups of the workload's query roots, query `i` /// being `roots[i]`, by hash-consing identical logical sub-expressions /// (`share_common_sub_dags`). -pub fn generate_dag_of_logical_memo_groups(roots: &[QueryRoot]) -> DagOfLogicalMemoGroups { +pub fn memo_of_queries(roots: &[QueryRoot]) -> Memo { let (groups, group_of) = hash_consed(roots); let group = |node: &Rc| group_of[&Rc::as_ptr(node)]; let query_roots = roots @@ -28,7 +28,10 @@ pub fn generate_dag_of_logical_memo_groups(roots: &[QueryRoot]) -> DagOfLogicalM } }) .collect(); - DagOfLogicalMemoGroups::new(groups, query_roots, group_of) + Memo { + groups, + query_roots, + } } /// Each original node (of `roots`), by address, to its group. @@ -108,6 +111,7 @@ fn number( child_groups: children, parents: Vec::new(), queries: Vec::new(), + candidates: Vec::new(), }); id_of.insert(Rc::as_ptr(node), id); id @@ -164,7 +168,7 @@ mod tests { query("quantile_over_time(0.95, m[5m])"), query("quantile_over_time(0.99, m[5m])"), ]; - let memo = generate_dag_of_logical_memo_groups(&roots); + let memo = memo_of_queries(&roots); let root_group = |q: usize| *memo.query_roots[q].as_operator().unwrap(); assert_eq!(root_group(0), root_group(1)); assert_ne!(root_group(2), root_group(3)); @@ -190,7 +194,7 @@ mod tests { "avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))", AccuracyTarget::max_error(0.01).failure_probability(0.001), ))]; - let memo = generate_dag_of_logical_memo_groups(&roots); + let memo = memo_of_queries(&roots); let division = memo .groups .iter() @@ -217,12 +221,10 @@ mod tests { QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node.clone())), QueryRoot::Operator(node.clone()), ]; - let memo = generate_dag_of_logical_memo_groups(&roots); - let id = memo.group_for_node(&node).unwrap(); + let memo = memo_of_queries(&roots); + let id = *memo.query_roots[2].as_operator().unwrap(); assert_eq!(queries_rooted_at(&memo.query_roots, id), vec![1, 2]); assert_eq!(memo.groups[id.0 as usize].reached_by_queries(), [1, 2]); - assert!(generate_dag_of_logical_memo_groups(&roots[..1]) - .groups - .is_empty()); + assert!(memo_of_queries(&roots[..1]).groups.is_empty()); } } diff --git a/crates/logical-asap-optimization/src/stage0/cse.rs b/crates/logical-asap-optimization/src/stage0/cse.rs index 6f209a4e6..d90241d4a 100644 --- a/crates/logical-asap-optimization/src/stage0/cse.rs +++ b/crates/logical-asap-optimization/src/stage0/cse.rs @@ -85,7 +85,7 @@ fn own_fields(node: &OperatorNode) -> Operator<()> { /// hash is SipHash over two parts: /// /// 1. the canonical JSON of [`own_fields`] plus `result_kind`, `schema`, -/// `guarantee`, `timing` and `materialization` — every field `PartialEq` compares except the +/// `guarantee` — every field `PartialEq` compares except the /// children. A scalar expression is serialized as data with each operator /// node it reads replaced by `()`, so a reference to an /// interned sub-DAG contributes nothing of its own here; @@ -116,8 +116,6 @@ pub(crate) fn structural_hash(node: &OperatorNode, cache: &mut NodeToStructuralH node.result_kind, &node.schema, &node.guarantee, - node.timing, - node.materialization, ); serde_json::to_string(&own) .unwrap_or_default() @@ -177,8 +175,6 @@ fn same_node( children_equal && left.result_kind == right.result_kind && left.schema == right.schema - && left.timing == right.timing - && left.materialization == right.materialization && same_value(&left.guarantee, &right.guarantee) && same_value(&own_fields(left), &own_fields(right)) } @@ -242,8 +238,7 @@ impl InternTable { type NodeToInternedNodeCache = HashMap<*const OperatorNode, (Rc, Rc)>; /// Intern `node`'s children (recursively), then `node` itself. The rebuilt -/// node keeps `node`'s retained schema, result kind, guarantee, timing and -/// materialization: +/// node keeps `node`'s retained schema, result kind and guarantee: /// every child is replaced by an equal node, so each derived property stays /// valid, and the result is `PartialEq`-equal to the input. fn intern_bottom_up( @@ -258,9 +253,7 @@ fn intern_bottom_up( .operator .map_children(|child| intern_bottom_up(table, visited, child)); let rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization); + .with_guarantee(node.guarantee.clone()); let interned = table.intern(rebuilt); visited.insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&interned))); interned @@ -294,9 +287,7 @@ mod tests { }; use asap_types::ir::properties::guarantee::AccuracyGuarantee; use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; - use asap_types::ir::schema::sketch::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryUpdate, - }; + use asap_types::ir::schema::sketch::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::ScalarExpr; use asap_types::ir::{BinaryOperator, NonASAPOp}; @@ -605,11 +596,8 @@ mod tests { // ── ASAP nodes ────────────────────────────────────────────────────── - fn summary_agg(alpha: f64, guarantee: Option) -> Rc { - let family = FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), - GroupingStrategy::default(), - ); + fn summary_agg(_alpha: f64, guarantee: Option) -> Rc { + let family = FieldDataType::Sketch(SketchAlgorithm::DDSketch, GroupingStrategy::default()); let schema = Schema::lifted(vec![Field::new("state", family.clone(), false)], None); assert!(!schema.has_unique_key(), "fixture sanity"); Rc::new( @@ -636,11 +624,14 @@ mod tests { assert!(ra.guarantee.is_some()); } + /// θ is not part of a node's identity (#1207): two sizes of one family + /// over one input are one group. A different guarantee still keeps them + /// apart. #[test] - fn asap_nodes_with_distinct_parameters_or_guarantees_are_not_shared() { + fn asap_nodes_with_distinct_parameters_are_shared_but_distinct_guarantees_are_not() { let exact = || Some(AccuracyGuarantee::exact("fixture")); let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.001, exact())); - assert!(!Rc::ptr_eq(&ra, &rb), "different sketch parameters"); + assert!(Rc::ptr_eq(&ra, &rb), "θ is not part of the group identity"); let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, None)); assert!( !Rc::ptr_eq(&ra, &rb), diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index 988cb4210..37c029953 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -4,19 +4,11 @@ use std::rc::Rc; -// Stage 1 internals other crates' tests call: Pass 1's working form and its -// population step, single-candidate composition, Stage 0 hash-consing, and -// the Pass 2 rule entry points those tests build memos with. -pub use crate::pass1::memo_population::try_populate_pass1; +// Stage 1 internals other crates' tests call: Pass 1's working form, +// single-candidate composition and Stage 0 hash-consing. pub use crate::pass1::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, }; -pub use crate::pass2::asap_aware_cse::{ - rollup_candidates, rollup_pass2_candidates, tumbling_window_candidates, - window_composition_pass2_candidates, window_composition_shared_chain_candidates, - with_union_scan_labels, -}; -pub use crate::pass2::memo::window_candidates::add_local_window_composition; pub use crate::stage0::cse::share_common_sub_dags; use asap_types::ir::properties::AccuracyTarget; @@ -26,40 +18,6 @@ use asap_types::workload::{ QueryRequirements, QueryWorkload, }; -/// Pass 2 after local window composition, on a memo the test composed -/// itself (or not at all): rules 2–9, then -/// `add_shared_summary_state_groups`. -/// Production runs these through -/// [`generate_logical_asap_memo_candidates`](crate::pass2::logical_memo_candidates::generate_logical_asap_memo_candidates). -pub fn sharing_rules_and_shared_state_groups( - mut memo: asap_types::stages::Memo, -) -> Result { - crate::pass2::logical_memo_candidates::apply_sharing_rules(&mut memo)?; - crate::pass2::memo::state_groups::add_shared_summary_state_groups(&mut memo)?; - Ok(memo) -} - -/// The groups `add_shared_summary_state_groups` -/// added to an expanded memo, in order: each has one candidate, the -/// shared build (origin `SharedBuild`); no other group's candidates are -/// only that. -pub fn shared_summary_state_groups( - memo: &asap_types::stages::Memo, -) -> Vec { - memo.candidates - .iter() - .enumerate() - .filter(|(_, candidates)| { - matches!( - candidates.as_slice(), - [only] if only.origin - == asap_types::stages::OriginOfCandidateForMemoGroup::SharedBuild - ) - }) - .map(|(group, _)| asap_types::stages::MemoGroupId(group as u32)) - .collect() -} - /// The statistic classic HLL's failure probability at `precision` and /// `relative_error` over `source` is left as, for Stage 3 tests that bind /// it. The estimator keeps the format crate-private. @@ -108,12 +66,8 @@ use std::time::Duration; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::properties::timing::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, -}; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; -use asap_types::physical::{Materialization, Medium}; /// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, /// no unique key. @@ -220,27 +174,6 @@ pub fn time_range(range: Duration, child: Rc) -> Rc .unwrap() } -/// `root` timed under the default (every sub-DAG `NotMaterialized`) -/// assignment — the shape export and the post-ASAP validators consume. -pub fn timed(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default materialization timings apply") -} - -/// `root` timed with every sub-DAG `MatAtIngestion` where the DAG allows it. -pub fn at_ingestion(root: &Rc) -> Rc { - apply_materialization_timings( - root, - &MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)), - &mut NodeToTimedNodeCache::new(), - ) - .expect("at-ingestion materialization timings apply") -} - /// The Count-Min summary of a Stage 1 candidate of `query`. pub fn cms_summary(query: &str) -> OperatorNode { use crate::pass1::per_computation_candidates::compose_logical_candidate; @@ -250,9 +183,9 @@ pub fn cms_summary(query: &str) -> OperatorNode { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); let root = lower_promql(query, target); // Inspect each local computation choice under the built-in deployment. - let per_computation = generate_per_computation_candidates( - &crate::stage0::generate_dag_of_logical_memo_groups(&[QueryRoot::Operator(root)]), - ) + let per_computation = generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ + QueryRoot::Operator(root), + ])) .unwrap() .without_unsupported_summaries(None); let summary = per_computation @@ -277,7 +210,7 @@ pub fn cms_summary(query: &str) -> OperatorNode { Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::Cms + }) if *kind == SketchAlgorithm::Cms ) }) .map(|node| (*node).clone()) diff --git a/crates/physical-asap-optimization/Cargo.toml b/crates/physical-asap-optimization/Cargo.toml index 9bc25ba9d..dc4220f5e 100644 --- a/crates/physical-asap-optimization/Cargo.toml +++ b/crates/physical-asap-optimization/Cargo.toml @@ -6,29 +6,7 @@ edition = "2021" # #509 Stage 2, physical ASAP-aware optimization: materialization and # physical operator implementation of each Stage 1 candidate, and the # PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`). Depends only on -# asap-types (the IR Stage 1 emits) and the shared accuracy model -# (asap-accuracy, guarantee recomposition of a selected plan), never on -# Stage 1's crate, plan selection or the executor. Tests build their inputs -# with Stage 1. +# asap-types, never on Stage 1's crate, plan selection or the executor. [dependencies] -asap-accuracy = { path = "../accuracy" } asap-types = { path = "../types" } -# The tumbling-window retention of a realized plan (`AssignmentRealizer::realize`). -asap-cost = { path = "../cost" } -rmp-serde = "1" -serde_json = "1" thiserror = "2" - -[features] -# Exposes `test_support` (single-candidate realization, operator -# implementation, tumbling window coverage, and `test_support::catalog`: catalog -# internals plan selection's unit tests exercise with its solver), for other -# crates' tests. Tests also enumerate whole workloads with the reference pricer. -test-support = ["reference-pricer"] -# `optimize_physical_asap`: the whole-workload enumeration behind the -# developer tools' reference pricing (`asap-devtools`). Planning does not use it. -reference-pricer = [] - -[dev-dependencies] -asap-types = { path = "../types", features = ["test-support"] } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs deleted file mode 100644 index c5b8ade0a..000000000 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ /dev/null @@ -1,425 +0,0 @@ -//! Finite, unpriced execution configurations over typed immediate candidates. -//! -//! This does not enumerate consumer subsets. Current summary eligibility is a -//! conjunction of individual-reader restrictions and pairwise interval/phase -//! compatibility; selected-reader constraints encode those conditions. Complete -//! Stage 2 validation still checks operator legality and upstream timing. -//! Potential shared non-summary nodes also receive query materialization; Stage 2 -//! checks highest-shared eligibility on the selected graph, not all possible consumers. -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; -use std::collections::HashMap; - -use crate::summary_materialization_eligibility; -use asap_types::ir::operator::{agg_has_exact_per_entity_kernel, Reduction}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use super::inventory::CandidateInventory; -use asap_types::ir::QueryRoot; -use asap_types::stages::{ - output_references, MemoGroupId, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - PruneReason, PrunedCandidate, ReaderConstraint, -}; - -/// [`PhysicalCandidate::may_keep_across_evaluations_per_sole_reader_query`] for `node` configured -/// with `materialization`: for each query in `entries`, whether -/// `summary_materialization_eligibility` with that query as the only reader -/// allows keeping it across evaluations. Empty unless `node` is a tumbling-window chain, -/// `materialization` is `MatAtQueryKeptAcrossEvaluations(Memory)` and the -/// deployment keeps query-time results. -pub fn may_keep_across_evaluations_per_sole_reader_query( - node: &OperatorNode, - materialization: Materialization, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Vec { - if materialization != Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) - || !matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - || !query_time_results_retained - { - return Vec::new(); - } - (0..entries.len()) - .map(|reader| { - summary_materialization_eligibility( - node, - &[reader], - entries, - data, - query_time_results_retained, - ) - .is_ok_and(|eligibility| eligibility.may_keep_across_evaluations) - }) - .collect() -} - -#[derive(Debug, thiserror::Error)] -pub enum ConfigurationError { - #[error("invalid source candidate inventory: {0}")] - InvalidInventory(String), - #[error("candidate {candidate} already has timing or materialization assigned")] - AssignedExecution { candidate: usize }, -} - -/// Each operator node of `source` with every timing and materialization -/// Stage 2 allows, one candidate each. `query_roots` are the memo's; their -/// group references are `source`'s output references. -pub fn enumerate_candidate_configurations( - source: &CandidateInventory, - query_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - raw_data_retained: bool, -) -> Result { - let invalid = |message: &str| ConfigurationError::InvalidInventory(message.into()); - source.validate().map_err(|error| invalid(&error))?; - let output_queries: Vec = output_references(query_roots) - .into_iter() - .map(|(query, _)| query) - .collect(); - let source_count = source.operator_nodes_of_candidates_for_memo_groups.len(); - if output_queries.len() != source.query_output_nodes.len() - || output_queries.iter().any(|&query| query >= entries.len()) - { - return Err(invalid("template or output/query mapping is incomplete")); - } - if source - .implementation_roots_of_group - .iter() - .flatten() - .any(|id| id.0 >= source_count) - { - return Err(invalid("group references missing source candidate")); - } - let mut query_outputs = vec![Vec::new(); entries.len()]; - let mut potential = vec![vec![false; entries.len()]; source_count]; - let mut may_produce_query_output = vec![false; source_count]; - for (output, &query) in output_queries.iter().enumerate() { - query_outputs[query].push(output); - let mut pending = source.query_output_nodes[output].clone(); - for root in &pending { - may_produce_query_output[root.0] = true; - } - while let Some(candidate) = pending.pop() { - if !potential[candidate.0][query] { - potential[candidate.0][query] = true; - pending.extend( - source.producer_nodes_per_input[candidate.0] - .iter() - .flat_map(|input| input.iter().copied()), - ); - } - } - } - let mut candidates = Vec::new(); - let mut reader_constraints = Vec::new(); - let mut pruned_candidates = Vec::new(); - let mut source_configurations = vec![Vec::new(); source_count]; - for source_id in 0..source_count { - let node = &source.operator_nodes_of_candidates_for_memo_groups[source_id].physical_node; - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized { - return Err(ConfigurationError::AssignedExecution { - candidate: source_id, - }); - } - let summary = matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ); - let forced_query = matches!( - node.asap(), - Some( - ASAPOp::SummaryEstimate { .. } - | ASAPOp::EvaluatePopulation { .. } - | ASAPOp::MaintainPopulation { .. } - ) - ); - let queries = potential[source_id] - .iter() - .enumerate() - .filter_map(|(query, reached)| reached.then_some(query)) - .collect::>(); - let mut modes = vec![(Materialization::NotMaterialized, ExecutionTiming::QueryTime)]; - if summary { - modes.push(( - Materialization::MatAtQuery(Medium::Memory), - ExecutionTiming::QueryTime, - )); - if queries.iter().any(|&query| { - summary_mode_allowed( - node, - &[query], - Materialization::MatAtIngestion(Medium::Memory), - entries, - data, - query_time_results_retained, - ) - }) { - modes.push(( - Materialization::MatAtIngestion(Medium::Memory), - ExecutionTiming::IngestionTime, - )); - } - if matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. })) - && queries.iter().any(|&query| { - summary_mode_allowed( - node, - &[query], - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - entries, - data, - query_time_results_retained, - ) - }) - { - modes.push(( - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ExecutionTiming::QueryTime, - )); - } - } else if !forced_query { - modes.push(( - Materialization::NotMaterialized, - ExecutionTiming::IngestionTime, - )); - } - // Potential reach only generates the choice. Selected-reader cardinality - // and Stage 2's highest-shared rule determine actual eligibility. - if !summary && queries.len() >= 2 { - modes.push(( - Materialization::MatAtQuery(Medium::Memory), - ExecutionTiming::QueryTime, - )); - } - // Configurations the deployment cannot run are not generated; each is - // recorded as pruned. - let group = source.operator_nodes_of_candidates_for_memo_groups[source_id].group(); - let mut prune_modes = |reason, applies: &dyn Fn(ExecutionTiming) -> bool| { - modes.retain(|&(materialization, timing)| { - let pruned = applies(timing); - if pruned { - pruned_candidates.push(PrunedCandidate { - group, - candidate: PhysicalCandidateKey { - operator_node: OperatorNodeOfCandidateForMemoGroupId(source_id), - timing, - materialization, - }, - reason, - }); - } - !pruned - }); - }; - if !per_entity_aggregate_executable(node) { - prune_modes(PruneReason::PerEntityAggregateNotExecutable, &|_| true); - } - // Without raw data retained, only ingestion-time work reads raw rows. - if !raw_data_retained && matches!(node.operator, Operator::NonASAP(NonASAPOp::Scan { .. })) - { - prune_modes(PruneReason::RawDataNotRetained, &|timing| { - timing != ExecutionTiming::IngestionTime - }); - } - for (materialization, timing) in modes { - let id = PhysicalCandidateId(candidates.len()); - source_configurations[source_id].push(id); - candidates.push(PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(source_id), - timing, - materialization, - producer_candidates_per_input: Vec::new(), - may_produce_query_output: may_produce_query_output[source_id] - && timing == ExecutionTiming::QueryTime, - may_keep_across_evaluations_per_sole_reader_query: - may_keep_across_evaluations_per_sole_reader_query( - node, - materialization, - entries, - data, - query_time_results_retained, - ), - }); - if materialization == Materialization::MatAtQuery(Medium::Memory) { - if !summary { - reader_constraints.push(ReaderConstraint::HighestSharedQueryMaterialization { - candidate: id, - query_outputs: query_outputs.clone(), - }); - } - reader_constraints.push(ReaderConstraint::MinimumQueryReaders { - candidate: id, - query_outputs: query_outputs.clone(), - minimum: 2, - }); - } - if summary { - let eligible = queries - .iter() - .copied() - .filter(|&query| { - summary_mode_allowed( - node, - &[query], - materialization, - entries, - data, - query_time_results_retained, - ) - }) - .collect::>(); - let outputs = eligible - .iter() - .flat_map(|&query| query_outputs[query].iter().copied()) - .collect(); - reader_constraints.push(ReaderConstraint::AllowedOutputs { - candidate: id, - outputs, - }); - // The current Stage 2 rules require common intervals/phases, - // whose conflicts are pairwise. No arbitrary set predicate is - // approximated by this encoding. - for (position, &left) in eligible.iter().enumerate() { - for &right in &eligible[position + 1..] { - if !summary_mode_allowed( - node, - &[left, right], - materialization, - entries, - data, - query_time_results_retained, - ) { - for &left_output in &query_outputs[left] { - for &right_output in &query_outputs[right] { - reader_constraints.push( - ReaderConstraint::IncompatibleOutputs { - candidate: id, - left: left_output, - right: right_output, - }, - ); - } - } - } - } - } - } - } - } - let mut producer_sets = ProducerSetInterner::default(); - // Compatibility depends on consumer timing and producer mode/timing only. - // Evaluate each source configuration once per timing, then reuse this exact - // index; reader eligibility remains expressed by separate graph constraints. - let mut timing_index = Vec::with_capacity(source_configurations.len()); - for configurations in &source_configurations { - let mut by_timing = [ProducerCandidates::default(), ProducerCandidates::default()]; - for (timing_index, timing) in [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime] - .into_iter() - .enumerate() - { - let mut compatible = Vec::new(); - for &producer in configurations { - if compatible_timing(timing, &candidates[producer.0]) { - compatible.push(producer); - } - } - by_timing[timing_index] = producer_sets.intern(compatible); - } - timing_index.push(by_timing); - } - let mut input_cache: HashMap<(ProducerCandidates, ExecutionTiming), ProducerCandidates> = - HashMap::new(); - for candidate in &mut candidates { - let mut requirements = Vec::new(); - for input in &source.producer_nodes_per_input[candidate.operator_node.0] { - let key = (input.clone(), candidate.timing); - let compatible_producers = if let Some(producer_candidates) = input_cache.get(&key) { - producer_candidates.clone() - } else { - let timing = match candidate.timing { - ExecutionTiming::QueryTime => 0, - ExecutionTiming::IngestionTime => 1, - }; - let mut compatible = Vec::new(); - for producer in input { - compatible.extend(timing_index[producer.0][timing].iter().copied()); - } - let producer_candidates = producer_sets.intern(compatible); - input_cache.insert(key, producer_candidates.clone()); - producer_candidates - }; - requirements.push(compatible_producers); - } - candidate.producer_candidates_per_input = requirements; - } - let mut result = PhysicalASAPMemoCandidates::new( - source.operator_nodes_of_candidates_for_memo_groups.clone(), - candidates, - query_roots.to_vec(), - reader_constraints, - ); - result.pruned_candidates = pruned_candidates; - super::validate_candidates(&result).map_err(|error| invalid(&error))?; - Ok(result) -} - -/// Whether the executor can run `node`: false only for an exact per-entity -/// `Aggregate` (a per-series range reduction over raw data samples) that -/// has more than one measure, a measure without an exact per-entity kernel -/// ([`agg_has_exact_per_entity_kernel`]), a measure filter or a `HAVING`. -/// The rule reads one operator node, so Stage 2 does not generate the node -/// rather than Stage 3 rejecting each plan that reads it. -fn per_entity_aggregate_executable(node: &OperatorNode) -> bool { - match &node.operator { - Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - having, - .. - }) => { - matches!(measures.as_slice(), [measure] if agg_has_exact_per_entity_kernel(measure)) - && filters.iter().all(Option::is_none) - && having.is_none() - } - _ => true, - } -} - -pub fn compatible_timing(consumer_timing: ExecutionTiming, producer: &PhysicalCandidate) -> bool { - consumer_timing == producer.timing - || (consumer_timing == ExecutionTiming::QueryTime - && matches!(producer.materialization, Materialization::MatAtIngestion(_))) -} - -fn summary_mode_allowed( - node: &OperatorNode, - readers: &[usize], - mode: Materialization, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> bool { - let Ok(eligibility) = summary_materialization_eligibility( - node, - readers, - entries, - data, - query_time_results_retained, - ) else { - return false; - }; - eligibility.range_compatibility.is_ok() - && match mode { - Materialization::NotMaterialized | Materialization::MatAtQuery(_) => true, - Materialization::MatAtIngestion(_) => eligibility.at_ingestion.is_ok(), - Materialization::MatAtQueryKeptAcrossEvaluations(_) => { - eligibility.may_keep_across_evaluations - } - } -} diff --git a/crates/physical-asap-optimization/src/catalog/factoring.rs b/crates/physical-asap-optimization/src/catalog/factoring.rs deleted file mode 100644 index 4ee9e4b68..000000000 --- a/crates/physical-asap-optimization/src/catalog/factoring.rs +++ /dev/null @@ -1,249 +0,0 @@ -//! Memoize identical producer interfaces and quotient equivalent terminal readers. -//! Internal computations are never merged. -use std::collections::HashMap; - -use asap_types::ir::OperatorNode; -use asap_types::stages::MemoGroupId; - -use asap_types::stages::{CandidateForMemoGroupId, PhysicalCandidateId}; -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; - -#[derive(Hash, PartialEq, Eq)] -pub struct MemoGroupOutputKey(Vec); - -impl MemoGroupOutputKey { - /// Include everything output_contract_satisfied reads from its expected - /// node. Byte-exact keys preserve parameters that PartialEq or JSON may - /// erase, and include closedness and every required uniqueness key. - pub fn new(group: MemoGroupId, expected: &OperatorNode) -> Result { - rmp_serde::to_vec(&(group.0, &expected.result_kind, &expected.schema)) - .map(Self) - .map_err(|error| error.to_string()) - } -} - -#[derive(Default)] -pub struct MemoGroupOutputToCompatibleProducersCache { - results: HashMap, - lists: ProducerSetInterner, - #[cfg(feature = "test-support")] - disabled: bool, - #[cfg(feature = "test-support")] - hits: std::cell::Cell, -} - -impl MemoGroupOutputToCompatibleProducersCache { - pub fn get(&self, key: &MemoGroupOutputKey) -> Option { - #[cfg(feature = "test-support")] - if self.disabled { - return None; - } - let found = self.results.get(key).cloned(); - #[cfg(feature = "test-support")] - if found.is_some() { - self.hits.set(self.hits.get() + 1); - } - found - } - - #[cfg(feature = "test-support")] - pub fn hits(&self) -> usize { - self.hits.get() - } - - #[cfg(feature = "test-support")] - pub fn disabled() -> Self { - Self { - disabled: true, - ..Self::default() - } - } - - pub fn remember( - &mut self, - key: MemoGroupOutputKey, - producer_candidates: Vec, - ) -> ProducerCandidates { - let producer_candidates = self.lists.intern(producer_candidates); - self.results.insert(key, producer_candidates.clone()); - producer_candidates - } -} - -/// Quotient only terminal unary summary readers. Internal consumers and merge -/// operators retain their original identities. A selection binds the -/// factored reader independently per output reference (one selected node -/// each), so every selection of the unfactored readers maps to one of the -/// factored reader, and each binding of it is an unfactored reader with the -/// same complete interface. -/// -/// These operators bypass ConcreteRows' witness-based row certification. -/// certified_price checks every supplied producer; template-based proposal -/// prices remain advisory. Execution and accuracy are recomputed from selected -/// producers, never from the representative template's child witness. -/// -/// A kept node's `candidates_for_memo_group` becomes the union, ascending, -/// of the candidates of the nodes factored into it, and its -/// `accuracy_budget_parts` the largest of theirs. -pub fn factor_terminal_readers( - inventory: &mut super::inventory::CandidateInventory, -) -> Result { - use super::inventory::CandidateInventoryError; - use asap_types::ir::ASAPOp; - let invalid = |error: String| CandidateInventoryError::InvalidMemo(error); - let count = inventory.operator_nodes_of_candidates_for_memo_groups.len(); - let mut internal = vec![false; count]; - for inputs in &inventory.producer_nodes_per_input { - for input in inputs { - for producer in input { - internal[producer.0] = true; - } - } - } - let mut outputs = vec![Vec::new(); count]; - for (output, root) in inventory.query_output_nodes.iter().enumerate() { - for id in root { - outputs[id.0].push(output); - } - } - let mut classes = HashMap::, usize>::new(); - let mut representative: Vec<_> = (0..count).collect(); - let mut eligible = vec![false; count]; - for id in 0..count { - let node = &inventory.operator_nodes_of_candidates_for_memo_groups[id]; - if internal[id] - || outputs[id].is_empty() - || inventory.producer_nodes_per_input[id].len() != 1 - { - continue; - } - let template = &node.physical_node; - if !matches!( - template.asap(), - Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::FinalizeExactAccumulator { .. }) - ) { - continue; - } - let mut inputs = Vec::new(); - let operator = template.operator.map_children(|child| { - inputs.push(child.clone()); - 0u8 - }); - let [input] = inputs.as_slice() else { - continue; - }; - eligible[id] = true; - let key = rmp_serde::to_vec(&( - node.group().0, - &outputs[id], - operator, - &template.schema, - template.result_kind, - &template.guarantee, - template.timing, - template.materialization, - &input.schema, - input.result_kind, - &input.guarantee, - input.timing, - input.materialization, - )) - .map_err(|error| invalid(error.to_string()))?; - if let Some(&previous) = classes.get(&key) { - representative[id] = previous; - } else { - classes.insert(key, id); - } - } - let removed = representative - .iter() - .enumerate() - .filter(|(id, rep)| *id != **rep) - .count(); - if removed == 0 { - return Ok(0); - } - let mut unions = HashMap::>::new(); - let mut known = HashMap::>::new(); - let mut candidates_for_memo_group = HashMap::>::new(); - let mut accuracy_budget_parts = HashMap::::new(); - for (id, &rep) in representative.iter().enumerate() { - if eligible[id] { - let producer_candidates = unions.entry(rep).or_default(); - let known = known.entry(rep).or_default(); - for &producer_id in &inventory.producer_nodes_per_input[id][0] { - if known.insert(producer_id) { - producer_candidates.push(producer_id); - } - } - } - if id != rep { - let factored = &inventory.operator_nodes_of_candidates_for_memo_groups[id]; - candidates_for_memo_group - .entry(rep) - .or_default() - .extend(&factored.candidates_for_memo_group); - let parts = accuracy_budget_parts.entry(rep).or_insert(0); - *parts = (*parts).max(factored.accuracy_budget_parts); - } - } - let mut new_ids = vec![PhysicalCandidateId(0); count]; - let mut next = 0; - for (id, &rep) in representative.iter().enumerate() { - if id == rep { - new_ids[id] = PhysicalCandidateId(next); - next += 1; - } - } - for (id, &rep) in representative.iter().enumerate() { - new_ids[id] = new_ids[rep]; - } - let mut lists = ProducerSetInterner::default(); - let mut nodes = Vec::with_capacity(next); - let mut producer_nodes_per_input = Vec::with_capacity(next); - for (id, node) in inventory - .operator_nodes_of_candidates_for_memo_groups - .iter() - .enumerate() - { - if representative[id] != id { - continue; - } - let mut node = node.clone(); - if let Some(factored) = candidates_for_memo_group.remove(&id) { - node.candidates_for_memo_group.extend(factored); - node.candidates_for_memo_group.sort_unstable(); - node.candidates_for_memo_group.dedup(); - } - if let Some(parts) = accuracy_budget_parts.remove(&id) { - node.accuracy_budget_parts = node.accuracy_budget_parts.max(parts); - } - let mut inputs = inventory.producer_nodes_per_input[id].clone(); - if let Some(producer_candidates) = unions.remove(&id) { - inputs[0] = producer_candidates.into(); - } - for input in &mut inputs { - *input = lists.intern(input.iter().map(|id| new_ids[id.0]).collect::>()); - } - nodes.push(node); - producer_nodes_per_input.push(inputs); - } - let remap = |ids: &mut Vec| { - let mut seen = std::collections::HashSet::new(); - *ids = ids - .iter() - .map(|id| new_ids[id.0]) - .filter(|id| seen.insert(*id)) - .collect(); - }; - for root in &mut inventory.query_output_nodes { - remap(root); - } - for group in &mut inventory.implementation_roots_of_group { - remap(group); - } - inventory.operator_nodes_of_candidates_for_memo_groups = nodes; - inventory.producer_nodes_per_input = producer_nodes_per_input; - inventory.validate().map_err(invalid)?; - Ok(removed) -} diff --git a/crates/physical-asap-optimization/src/catalog/inventory.rs b/crates/physical-asap-optimization/src/catalog/inventory.rs deleted file mode 100644 index cc8daf559..000000000 --- a/crates/physical-asap-optimization/src/catalog/inventory.rs +++ /dev/null @@ -1,346 +0,0 @@ -//! Lower the logical candidate memo into typed immediate operator nodes and -//! the nodes that may produce each of their inputs. -//! -//! Consumer-dependent demand, composed accuracy, and physical materialization -//! are handled by later configuration and evaluation stages. Private exact -//! operators are lowered before decomposition; the resulting nodes still need -//! Stage 2 validation before execution. Nothing is priced. - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::OperatorNode; -use asap_types::physical::Materialization; -use asap_types::stages::Memo; -use asap_types::stages::MemoGroupId; - -use super::factoring::{MemoGroupOutputKey, MemoGroupOutputToCompatibleProducersCache}; -use asap_types::stages::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, PhysicalCandidateId, - ProducerCandidates, -}; - -/// The memo's implementation candidates lowered to operator nodes, before -/// timing and materialization. Node IDs index -/// `operator_nodes_of_candidates_for_memo_groups`; they become -/// [`asap_types::stages::OperatorNodeOfCandidateForMemoGroupId`]s of the -/// configured candidates. -#[derive(Debug)] -pub struct CandidateInventory { - /// One per operator node: each implementation root, then the private - /// nodes below it. - pub operator_nodes_of_candidates_for_memo_groups: Vec, - /// Per node, per input (in `map_children` order): the nodes that may - /// produce it (a private child, or the input group's compatible roots). - pub producer_nodes_per_input: Vec>, - /// Per output reference: the roots of its group whose output matches the - /// group's logical node. - pub query_output_nodes: Vec>, - /// Per group: its implementation roots, excluding each implementation's - /// private nodes. - pub implementation_roots_of_group: Vec>, -} - -impl CandidateInventory { - /// Every producer and query output names an existing node, once per list. - pub fn validate(&self) -> Result<(), String> { - let count = self.operator_nodes_of_candidates_for_memo_groups.len(); - if self.producer_nodes_per_input.len() != count { - return Err("producer lists are not indexed by node".into()); - } - for (node, inputs) in self.producer_nodes_per_input.iter().enumerate() { - for (input, producer_candidates) in inputs.iter().enumerate() { - super::validate_ids( - producer_candidates, - count, - &format!("candidate {node}, requirement {input}"), - )?; - } - } - for (output, nodes) in self.query_output_nodes.iter().enumerate() { - super::validate_ids(nodes, count, &format!("query root {output}"))?; - } - Ok(()) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum CandidateInventoryError { - #[error("invalid candidate memo: {0}")] - InvalidMemo(String), - #[error("unsupported candidate {candidate} in group {group}: {reason}")] - Unsupported { - group: usize, - candidate: usize, - reason: &'static str, - }, - #[error( - "physical operator implementation for candidate {candidate} in group {group}: {reason}" - )] - PhysicalImplementation { - group: usize, - candidate: usize, - reason: String, - }, -} - -/// The inventory with identical terminal readers factored into one node -/// (whose `candidates_for_memo_group` lists every candidate it can implement). -pub fn build_candidate_inventory( - memo: &Memo, - output_groups: &[MemoGroupId], -) -> Result { - let mut inventory = build_with_compatible_producers_cache( - memo, - output_groups, - &mut MemoGroupOutputToCompatibleProducersCache::default(), - )?; - super::factoring::factor_terminal_readers(&mut inventory)?; - Ok(inventory) -} - -pub fn build_with_compatible_producers_cache( - memo: &Memo, - output_groups: &[MemoGroupId], - compatible_producers_cache: &mut MemoGroupOutputToCompatibleProducersCache, -) -> Result { - let invalid = |message: &str| CandidateInventoryError::InvalidMemo(message.into()); - if memo.groups.len() != memo.candidates.len() { - return Err(invalid("group and candidate vector lengths differ")); - } - if output_groups - .iter() - .any(|group| group.0 as usize >= memo.groups.len()) - { - return Err(invalid("query output references missing group")); - } - let mut inventory = CandidateInventory { - operator_nodes_of_candidates_for_memo_groups: Vec::new(), - producer_nodes_per_input: Vec::new(), - query_output_nodes: Vec::new(), - implementation_roots_of_group: vec![Vec::new(); memo.groups.len()], - }; - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - let unsupported = |reason| CandidateInventoryError::Unsupported { - group, - candidate: index, - reason, - }; - let implemented: HashSet<_> = std::iter::once(MemoGroupId(group as u32)) - .chain(candidate.fused_groups.iter().copied()) - .collect(); - if implemented.len() != 1 + candidate.fused_groups.len() - || implemented - .iter() - .any(|id| id.0 as usize >= memo.groups.len()) - { - return Err(invalid( - "candidate names duplicate or missing absorbed groups", - )); - } - // Only the owner group is this candidate's output contract. Its fused - // groups describe absorbed computations inside its private subtree, - // not additional results it can produce. Private-node decomposition - // charges that work; independent consumers of an absorbed group - // still select one of that group's own candidates. Reverse support - // leaves unused absorbed groups inactive without exclusion rules. - if candidate.accuracy_budget_parts == 0 { - return Err(unsupported("zero accuracy budget parts")); - } - if candidate - .input_groups - .iter() - .any(|child| child.0 as usize >= memo.groups.len()) - { - return Err(invalid("candidate references missing input group")); - } - let inputs = candidate - .input_groups - .iter() - .map(|child| Rc::clone(&memo.groups[child.0 as usize].logical_node)) - .collect::>(); - if inputs - .iter() - .any(|input| Rc::ptr_eq(input, &candidate.root_node)) - { - return Err(unsupported( - "root aliases an input group without an operator", - )); - } - let implemented = crate::implement_candidate_operators(&candidate.root_node, &inputs) - .map_err(|error| CandidateInventoryError::PhysicalImplementation { - group, - candidate: index, - reason: error.to_string(), - })?; - let id = - PhysicalCandidateId(inventory.operator_nodes_of_candidates_for_memo_groups.len()); - inventory.implementation_roots_of_group[group].push(id); - inventory.operator_nodes_of_candidates_for_memo_groups.push( - OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group as u32), - index, - }], - is_implementation_root: true, - accuracy_budget_parts: candidate.accuracy_budget_parts, - physical_node: implemented, - }, - ); - inventory.producer_nodes_per_input.push(Vec::new()); - } - } - for (group, candidates) in memo.candidates.iter().enumerate() { - for (index, candidate) in candidates.iter().enumerate() { - let mut boundaries = HashMap::new(); - for &child in &candidate.input_groups { - let pointer = Rc::as_ptr(&memo.groups[child.0 as usize].logical_node); - if boundaries - .insert(pointer, child) - .is_some_and(|previous| previous != child) - { - return Err(invalid("one boundary node names different input groups")); - } - } - if boundaries.contains_key(&Rc::as_ptr(&candidate.root_node)) { - return Err(CandidateInventoryError::Unsupported { - group, - candidate: index, - reason: "root aliases an input group without an operator", - }); - } - let mut builder = ImmediateCandidateBuilder { - inventory: &mut inventory, - compatible_producers_cache: &mut *compatible_producers_cache, - boundaries, - private_nodes: HashMap::new(), - used_boundaries: HashSet::new(), - owner: CandidateForMemoGroupId { - group: MemoGroupId(group as u32), - index, - }, - owner_accuracy_budget_parts: candidate.accuracy_budget_parts, - }; - let root_id = builder.inventory.implementation_roots_of_group[group][index]; - let implemented = Rc::clone( - &builder - .inventory - .operator_nodes_of_candidates_for_memo_groups[root_id.0] - .physical_node, - ); - builder.add(&implemented, Some(root_id))?; - if candidate - .input_groups - .iter() - .any(|group| !builder.used_boundaries.contains(group)) - { - return Err(invalid( - "declared input group has no canonical node boundary", - )); - } - } - } - for group in output_groups { - let expected = &memo.groups[group.0 as usize].logical_node; - let mut compatible_candidates = Vec::new(); - for &id in &inventory.implementation_roots_of_group[group.0 as usize] { - let template = - &inventory.operator_nodes_of_candidates_for_memo_groups[id.0].physical_node; - if compatible_output(expected, template)? { - compatible_candidates.push(id); - } - } - inventory.query_output_nodes.push(compatible_candidates); - } - inventory.validate().map_err(|error| invalid(&error))?; - Ok(inventory) -} - -struct ImmediateCandidateBuilder<'a> { - inventory: &'a mut CandidateInventory, - compatible_producers_cache: &'a mut MemoGroupOutputToCompatibleProducersCache, - boundaries: HashMap<*const OperatorNode, MemoGroupId>, - private_nodes: HashMap<*const OperatorNode, PhysicalCandidateId>, - used_boundaries: HashSet, - /// The memo candidate being lowered; owner of every node it adds. - owner: CandidateForMemoGroupId, - /// The owner's `accuracy_budget_parts`, which its private nodes carry. - owner_accuracy_budget_parts: usize, -} - -impl ImmediateCandidateBuilder<'_> { - fn add( - &mut self, - node: &Rc, - reserved: Option, - ) -> Result { - if let Some(&existing) = self.private_nodes.get(&Rc::as_ptr(node)) { - return Ok(existing); - } - if node.timing.is_some() || node.materialization != Materialization::NotMaterialized { - return Err(CandidateInventoryError::Unsupported { - group: self.owner.group.0 as usize, - candidate: self.owner.index, - reason: "assigned execution timing or materialization", - }); - } - let mut requirements = Vec::new(); - for input in super::reconstruction::ordered_inputs(node) { - let producers = if let Some(&group) = self.boundaries.get(&Rc::as_ptr(&input)) { - self.used_boundaries.insert(group); - let key = MemoGroupOutputKey::new(group, &input) - .map_err(CandidateInventoryError::InvalidMemo)?; - if let Some(compatible) = self.compatible_producers_cache.get(&key) { - compatible - } else { - let mut compatible = Vec::new(); - for &candidate in - &self.inventory.implementation_roots_of_group[group.0 as usize] - { - let template = &self.inventory.operator_nodes_of_candidates_for_memo_groups - [candidate.0] - .physical_node; - if compatible_output(&input, template)? { - compatible.push(candidate); - } - } - self.compatible_producers_cache.remember(key, compatible) - } - } else { - vec![self.add(&input, None)?].into() - }; - requirements.push(producers); - } - let id = if let Some(id) = reserved { - self.inventory.producer_nodes_per_input[id.0] = requirements; - id - } else { - let id = PhysicalCandidateId( - self.inventory - .operator_nodes_of_candidates_for_memo_groups - .len(), - ); - self.inventory.producer_nodes_per_input.push(requirements); - self.inventory - .operator_nodes_of_candidates_for_memo_groups - .push(OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![self.owner], - is_implementation_root: false, - accuracy_budget_parts: self.owner_accuracy_budget_parts, - physical_node: Rc::clone(node), - }); - id - }; - self.private_nodes.insert(Rc::as_ptr(node), id); - Ok(id) - } -} - -fn compatible_output( - expected: &OperatorNode, - producer: &OperatorNode, -) -> Result { - super::reconstruction::output_contract_satisfied(expected, producer) - .map_err(CandidateInventoryError::InvalidMemo) -} diff --git a/crates/physical-asap-optimization/src/catalog/mod.rs b/crates/physical-asap-optimization/src/catalog/mod.rs index eb3069ead..d278afef4 100644 --- a/crates/physical-asap-optimization/src/catalog/mod.rs +++ b/crates/physical-asap-optimization/src/catalog/mod.rs @@ -1,48 +1,32 @@ -//! Stage 2's physical candidates: implement each memo candidate's -//! operators, enumerate its timing and materialization, record which -//! candidates can produce each input, and prune what cannot be used +//! Stage 2's physical candidates: expand each logical candidate into one +//! physical candidate per valid $(\tau, \ell, \sigma)$, and each logical edge +//! into the physical edges between the two candidates' alternatives //! ([`generate_physical_asap_memo_candidates`]); and realize one selected -//! assignment as a physical plan (#1047). The output type, +//! DAG as a physical plan (#1047). The output type, //! `PhysicalASAPMemoCandidates`, lives in `asap_types::stages`; Stage 3 //! selects from it. -mod configurations; -mod factoring; -mod inventory; -mod pruning; mod realize; -mod reconstruction; -pub use configurations::ConfigurationError; -pub use inventory::CandidateInventoryError; pub use realize::Stage2AssignmentRealizer; -use asap_types::stages::{ - LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, ReaderConstraint, -}; +use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; /// Why [`generate_physical_asap_memo_candidates`] failed. #[derive(Debug, thiserror::Error)] pub enum ErrorGeneratingPhysicalASAPMemoCandidates { - /// Lowering the memo's candidates to operator nodes failed. - #[error(transparent)] - Inventory(#[from] CandidateInventoryError), - /// Enumerating timings and materializations failed. - #[error(transparent)] - Configuration(#[from] ConfigurationError), /// Structural pruning found the candidates malformed. #[error("{0}")] Pruning(String), } -/// Stage 2: implement every operator node of every candidate in `logical`'s -/// memo, enumerate the timings and materializations the data and deployment -/// allow (`query_time_results_retained`; no query-time raw scans unless -/// `raw_data_retained`; no exact per-entity aggregate the executor cannot -/// run), record which candidates may produce each input, and prune what -/// cannot be used. Every configuration not generated and every candidate -/// pruned is recorded in the output's `pruned_candidates` with its -/// `PruneReason`. Prices nothing. +/// Stage 2: expand every logical candidate of `logical` into its physical +/// alternatives (decision-tree levels 4–6: timing, lifetime, schedule) the +/// data and deployment allow (`query_time_results_retained`; no query-time +/// raw scans unless `raw_data_retained`), and every logical edge into its +/// physical edges with $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$. Every +/// alternative not generated is recorded in the output's +/// `pruned_candidates` with its `PruneReason`. Prices nothing. pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], @@ -50,114 +34,12 @@ pub fn generate_physical_asap_memo_candidates( query_time_results_retained: bool, raw_data_retained: bool, ) -> Result { - let memo = &logical.memo; - let output_groups: Vec<_> = asap_types::stages::output_references(&memo.query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - let inventory = inventory::build_candidate_inventory(memo, &output_groups)?; - let configured = configurations::enumerate_candidate_configurations( - &inventory, - &memo.query_roots, + let _ = ( + logical, entries, data, query_time_results_retained, raw_data_retained, - )?; - pruning::prune(configured).map_err(ErrorGeneratingPhysicalASAPMemoCandidates::Pruning) -} - -/// Each output reference's candidates ([`PhysicalASAPMemoCandidates::query_output_candidates`]). -fn query_output_candidates(physical: &PhysicalASAPMemoCandidates) -> Vec> { - (0..physical.output_reference_count()) - .map(|output| physical.query_output_candidates(output).collect()) - .collect() -} - -/// `ids` name existing candidates (of `count`), each once. -fn validate_ids(ids: &[PhysicalCandidateId], count: usize, context: &str) -> Result<(), String> { - let mut seen = std::collections::HashSet::new(); - for id in ids { - if id.0 >= count { - return Err(format!("{context} references missing candidate {}", id.0)); - } - if !seen.insert(*id) { - return Err(format!("{context} repeats candidate {}", id.0)); - } - } - Ok(()) -} - -/// Every producer, operator node and reader constraint of `physical` names an -/// existing candidate or output reference, and each query reader constraint -/// partitions the output references. -fn validate_candidates(physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let count = physical.candidates.len(); - let outputs_count = physical.output_reference_count(); - let invalid = |message: &str| format!("invalid physical ASAP memo candidates: {message}"); - for constraint in &physical.reader_constraints { - let outputs = match constraint { - ReaderConstraint::MinimumQueryReaders { query_outputs, .. } - | ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - let outputs: Vec<_> = query_outputs.iter().flatten().copied().collect(); - let unique: std::collections::HashSet<_> = outputs.iter().copied().collect(); - if outputs.len() != outputs_count || unique.len() != outputs.len() { - return Err(invalid( - "query reader buckets must partition every output exactly once", - )); - } - outputs - } - ReaderConstraint::AllowedOutputs { outputs, .. } => outputs.clone(), - ReaderConstraint::IncompatibleOutputs { left, right, .. } => vec![*left, *right], - }; - if constraint.candidate().0 >= count - || outputs.iter().any(|&output| output >= outputs_count) - { - return Err(invalid( - "reader constraint references missing candidate or output", - )); - } - } - for (index, candidate) in physical.candidates.iter().enumerate() { - if candidate.operator_node.0 >= physical.operator_nodes_of_candidates_for_memo_groups.len() - { - return Err(invalid(&format!( - "candidate {index} references missing operator node" - ))); - } - for (input, producer_candidates) in - candidate.producer_candidates_per_input.iter().enumerate() - { - validate_ids( - producer_candidates, - count, - &format!("candidate {index}, requirement {input}"), - ) - .map_err(|message| invalid(&message))?; - } - } - Ok(()) -} - -/// Catalog internals that plan selection's unit tests exercise together with -/// its solver. -#[cfg(feature = "test-support")] -pub mod test_support { - pub use super::configurations::{ - compatible_timing, enumerate_candidate_configurations, - may_keep_across_evaluations_per_sole_reader_query, - }; - pub use super::factoring::{ - factor_terminal_readers, MemoGroupOutputKey, MemoGroupOutputToCompatibleProducersCache, - }; - pub use super::inventory::{ - build_candidate_inventory, build_with_compatible_producers_cache, CandidateInventory, - }; - pub use super::pruning::usable_candidates; - pub use super::realize::{materialize_selected_candidates, restore_query_roots}; - pub use super::reconstruction::{ - ordered_inputs, output_contract_satisfied, paired_node_identities, - reconstruct_selected_candidates, ReconstructedCandidates, - }; + ); + todo!("#1207 PR 5: physical expansion and physical edges") } diff --git a/crates/physical-asap-optimization/src/catalog/pruning.rs b/crates/physical-asap-optimization/src/catalog/pruning.rs deleted file mode 100644 index 4cde4cb9e..000000000 --- a/crates/physical-asap-optimization/src/catalog/pruning.rs +++ /dev/null @@ -1,195 +0,0 @@ -//! Remove provably unusable configurations before constructing solver variables. -//! No costs or dominance assumptions are involved: finite input realizability, -//! potential output reach and necessary reader counts are sufficient proofs. -use asap_types::stages::{ - PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalCandidateKey, PruneReason, - PrunedCandidate, ReaderConstraint, -}; -use asap_types::stages::{ProducerCandidates, ProducerSetInterner}; -use std::collections::HashMap; - -/// `configured` without the candidates `prune_reasons` rules out, its -/// IDs renumbered densely in their order; each removed one is appended to -/// its `pruned_candidates` with the reason. `configured` is already -/// validated: configuration enumeration ends by validating its output. -pub fn prune(configured: PhysicalASAPMemoCandidates) -> Result { - let count = configured.candidates.len(); - let reasons = prune_reasons(&configured); - let keep: Vec = reasons.iter().map(Option::is_none).collect(); - let mut pruned_candidates = configured.pruned_candidates.clone(); - for (candidate, reason) in configured.candidates.iter().zip(&reasons) { - if let Some(reason) = *reason { - pruned_candidates.push(PrunedCandidate { - group: configured.operator_nodes_of_candidates_for_memo_groups - [candidate.operator_node.0] - .group(), - candidate: PhysicalCandidateKey { - operator_node: candidate.operator_node, - timing: candidate.timing, - materialization: candidate.materialization, - }, - reason, - }); - } - } - let mut mapping = vec![None; count]; - let mut next = 0; - for (id, retained) in keep.iter().enumerate() { - if *retained { - mapping[id] = Some(PhysicalCandidateId(next)); - next += 1; - } - } - let mut producer_sets = ProducerSetInterner::default(); - let mut remapped: HashMap = HashMap::new(); - let candidates = configured - .candidates - .into_iter() - .enumerate() - .filter_map(|(id, mut candidate)| { - if !keep[id] { - return None; - } - for input in &mut candidate.producer_candidates_per_input { - *input = remapped - .entry(input.clone()) - .or_insert_with(|| { - producer_sets.intern( - input - .iter() - .filter_map(|id| mapping[id.0]) - .collect::>(), - ) - }) - .clone(); - } - Some(candidate) - }) - .collect(); - let reader_constraints = configured - .reader_constraints - .into_iter() - .filter_map(|mut constraint| { - let id = constraint.candidate_mut(); - *id = mapping[id.0]?; - Some(constraint) - }) - .collect(); - // Operator nodes are kept whole: `operator_node` indexes them unchanged. - let mut pruned = PhysicalASAPMemoCandidates::new( - configured.operator_nodes_of_candidates_for_memo_groups, - candidates, - configured.query_roots, - reader_constraints, - ); - pruned.pruned_candidates = pruned_candidates; - super::validate_candidates(&pruned)?; - Ok(pruned) -} - -/// Which candidates of `physical` [`prune`] keeps. -#[cfg(feature = "test-support")] -pub fn usable_candidates(physical: &PhysicalASAPMemoCandidates) -> Vec { - prune_reasons(physical) - .iter() - .map(Option::is_none) - .collect() -} - -/// Why each candidate of `physical` cannot be used, `None` for a usable -/// one: the reason of the first round that ruled it out, and within a -/// round the first of: an input with no buildable producer, no query output -/// reached, too few query readers. -fn prune_reasons(physical: &PhysicalASAPMemoCandidates) -> Vec> { - let count = physical.candidates.len(); - let query_output_candidates = super::query_output_candidates(physical); - let mut reasons: Vec> = vec![None; count]; - loop { - let enabled: Vec = reasons.iter().map(Option::is_none).collect(); - // Least fixed point: a cycle without a finite implementation is not a - // producer. A cycle with an independent escape candidate remains usable. - let mut realizable = vec![false; count]; - loop { - let mut changed = false; - for (id, candidate) in physical.candidates.iter().enumerate() { - if enabled[id] - && !realizable[id] - && candidate - .producer_candidates_per_input - .iter() - .all(|input| input.iter().any(|id| realizable[id.0])) - { - realizable[id] = true; - changed = true; - } - } - if !changed { - break; - } - } - let mut reached = vec![vec![false; query_output_candidates.len()]; count]; - for (output, root) in query_output_candidates.iter().enumerate() { - let mut pending = root.clone(); - while let Some(id) = pending.pop() { - if realizable[id.0] && !reached[id.0][output] { - reached[id.0][output] = true; - pending.extend( - physical.candidates[id.0] - .producer_candidates_per_input - .iter() - .flat_map(|input| input.iter().copied()), - ); - } - } - } - // Intersect all allowed-reader restrictions before counting potential - // distinct queries. Potential reach is an upper bound, never a demand. - for constraint in &physical.reader_constraints { - if let ReaderConstraint::AllowedOutputs { candidate, outputs } = constraint { - for (output, value) in reached[candidate.0].iter_mut().enumerate() { - *value &= outputs.contains(&output); - } - } - } - let mut round: Vec> = realizable - .iter() - .zip(&reached) - .zip(&enabled) - .map(|((realizable, outputs), enabled)| { - if !enabled || *realizable && outputs.iter().any(|value| *value) { - None - } else if !realizable { - Some(PruneReason::InputHasNoBuildableProducer) - } else { - Some(PruneReason::ReachesNoQueryOutput) - } - }) - .collect(); - for constraint in &physical.reader_constraints { - if let ReaderConstraint::MinimumQueryReaders { - candidate, - query_outputs, - minimum, - } = constraint - { - let queries = query_outputs - .iter() - .filter(|outputs| outputs.iter().any(|output| reached[candidate.0][*output])) - .count(); - if queries < *minimum { - round[candidate.0].get_or_insert(PruneReason::TooFewQueryReaders); - } - } - } - let mut changed = false; - for (reason, round) in reasons.iter_mut().zip(round) { - if reason.is_none() && round.is_some() { - *reason = round; - changed = true; - } - } - if !changed { - return reasons; - } - } -} diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs index f473709c1..9abbc6b8b 100644 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ b/crates/physical-asap-optimization/src/catalog/realize.rs @@ -1,140 +1,20 @@ -//! Apply an explicit selected materialization assignment through real Stage 2. -//! -//! No fallback modes are chosen. Physical rewrites or timing splits that change -//! candidate identity are rejected until represented by physical enumeration. -//! This validates execution legality, not cost estimates or accuracy targets. -use std::collections::HashMap; -use std::rc::Rc; - -use crate::optimize_physical_asap_chosen; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::Materialization; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; +//! Realize a selected DAG through Stage 2: each node's physical properties +//! come from its `PhysicalCandidate` (concepts doc, section 3, +//! "Realizer/export"). This validates execution legality, not cost estimates +//! or accuracy targets. +use asap_types::ir::QueryRoot; use asap_types::stages::{ AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateError, RealizedPlan, }; - -use super::reconstruction::{ - output_contract_satisfied, paired_node_identities, reconstruct_selected_candidates, -}; - -/// Realize `selected` with each selected node at its candidate's -/// materialization. Output -/// references must follow `map_operator_refs` traversal of the original -/// roots in workload order (one reference for an operator root). Repeated -/// scalar references remain separate positions; literal-only scalar roots -/// use none. -pub fn materialize_selected_candidates( - physical_candidates: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - original_roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result { - if entries.len() != original_roots.len() { - return Err("workload entries and query roots differ in length".into()); - } - let rebuilt = reconstruct_selected_candidates(physical_candidates, selected)?; - // `reconstruct_selected_candidates` checked every selected candidate exists. - let materializations: Vec<_> = selected - .candidate_of_node - .iter() - .map(|id| physical_candidates.candidates[id.0].materialization) - .collect(); - let mut node_by_pointer = HashMap::new(); - for (node, operator_node) in rebuilt.operator_node_of_selected_node.iter().enumerate() { - node_by_pointer.insert(Rc::as_ptr(operator_node), node); - } - let roots = restore_query_roots(original_roots, &rebuilt.outputs)?; - let mut assignment_error = None; - let mut seen = vec![false; materializations.len()]; - let physical = optimize_physical_asap_chosen( - "selected-candidate-connections", - &roots, - entries, - data, - query_time_results_retained, - &mut |logical, _, options| { - let node = node_by_pointer.get(&logical).copied(); - let requested = if let Some(node) = node { - seen[node] = true; - materializations[node] - } else { - Materialization::NotMaterialized - }; - if !options.contains(&requested) { - let message = format!( - "selected materialization {requested:?} is not offered for selected node {node:?}" - ); - assignment_error = Some(if node.is_some() { - PhysicalCandidateError::RejectedAssignment(message) - } else { - PhysicalCandidateError::InvalidOrUnsupported(message) - }); - return None; - } - Some(requested) - }, - ) - .map_err(|error| error.to_string())?; - if let Some(error) = assignment_error { - return Err(error); - } - let physical = physical.ok_or_else(|| { - PhysicalCandidateError::RejectedAssignment( - "selected materialization assignment violates Stage 2 timing or dependency rules" - .into(), - ) - })?; - physical.dag.validate().map_err(|error| error.to_string())?; - let physical_outputs = ordered_output_references(&physical.roots); - let identities = paired_node_identities(&rebuilt.outputs, &physical_outputs)?; - for (node, mode) in materializations.iter().enumerate() { - if *mode != Materialization::NotMaterialized && !seen[node] { - // The decoder already established node identity. Stage 2 did - // not offer this selected node (e.g. a lower shared non-summary), - // so the assignment is infeasible rather than an adapter failure. - return Err(PhysicalCandidateError::RejectedAssignment(format!( - "selected node {node} is not an eligible materialization node in the selected graph" - ))); - } - } - let operator_node_of_selected_node = rebuilt - .operator_node_of_selected_node - .into_iter() - .enumerate() - .map(|(node, operator_node)| { - let physical = identities - .get(&Rc::as_ptr(&operator_node)) - .cloned() - .ok_or_else(|| format!("selected node {node} has no physical identity"))?; - if physical.materialization != materializations[node] { - return Err(format!( - "selected node {node} materialization changed during physical reconstruction" - )); - } - Ok(physical) - }) - .collect::, String>>()?; - let range_evaluations: Vec<_> = entries.iter().map(|entry| entry.range_evaluation).collect(); - let tumbling_window_retention_ms = - asap_cost::tumbling_window_retention_ms_of_plan(&physical.dag, &range_evaluations); - Ok(RealizedPlan { - plan: physical, - tumbling_window_retention_ms, - operator_node_of_selected_node, - }) -} +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; /// Stage 2's [`AssignmentRealizer`], over the planning context: the /// workload's original query roots (one per entry of `entries`), its data, /// and whether the deployment keeps query-time results across /// evaluations. The facade builds it, passes it to Stage 3, and builds the /// selected plan with it. -#[derive(Debug, Clone, Copy)] pub struct Stage2AssignmentRealizer<'a> { pub original_roots: &'a [QueryRoot], pub entries: &'a [QueryWorkloadEntry], @@ -143,183 +23,12 @@ pub struct Stage2AssignmentRealizer<'a> { } impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { - /// `materialize_selected_candidates`; each node also keeps its - /// candidate's timing. fn realize( &self, physical: &PhysicalASAPMemoCandidates, selected: &DagOfSelectedCandidates, ) -> Result { - let realized = materialize_selected_candidates( - physical, - selected, - self.original_roots, - self.entries, - self.data, - self.query_time_results_retained, - )?; - for (id, node) in selected - .candidate_of_node - .iter() - .zip(&realized.operator_node_of_selected_node) - { - let timing = physical.candidates[id.0].timing; - if node.timing != Some(timing) { - return Err(PhysicalCandidateError::InvalidOrUnsupported(format!( - "candidate {} requested timing {timing:?}, Stage 2 produced {:?}", - id.0, node.timing - ))); - } - } - Ok(realized) - } -} - -fn ordered_output_references(roots: &[QueryRoot]) -> Vec> { - asap_types::stages::output_references(roots) - .into_iter() - .map(|(_, node)| node) - .collect() -} - -pub fn restore_query_roots( - original: &[QueryRoot], - outputs: &[Rc], -) -> Result, String> { - let references = ordered_output_references(original); - if references.len() != outputs.len() { - return Err("output references do not match original query-root mapping".into()); - } - for (reference, output) in references.iter().zip(outputs) { - if !output_contract_satisfied(reference, output)? { - return Err(format!( - "selected output reference has incompatible result or schema: expected {:?} {:?}, received {:?} {:?}", - reference.result_kind, reference.schema, output.result_kind, output.schema - )); - } - } - let mut next = outputs.iter(); - Ok(original - .iter() - .map(|root| match root { - QueryRoot::Operator(_) => QueryRoot::Operator(Rc::clone(next.next().unwrap())), - QueryRoot::Scalar(expression) => QueryRoot::Scalar( - expression.map_operator_refs(&mut |_| Rc::clone(next.next().unwrap())), - ), - }) - .collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; - use asap_logical_asap_optimization::test_support::try_populate_pass1; - use asap_logical_asap_optimization::test_support::{ - entry, lower_promql, sharing_rules_and_shared_state_groups, - }; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::stages::{ - LogicalASAPMemoCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId, - }; - use asap_types::workload::QueryRecurrence; - - /// Add a node running `candidate` and, for each input, a node of its - /// first unmaterialized producer (else its first producer), - /// recursively; returns the new node. - fn select_from( - physical: &PhysicalASAPMemoCandidates, - candidate: PhysicalCandidateId, - selected: &mut DagOfSelectedCandidates, - ) -> SelectedNodeId { - let node = SelectedNodeId(selected.candidate_of_node.len()); - selected.candidate_of_node.push(candidate); - for (input, producer_candidates) in physical.candidates[candidate.0] - .producer_candidates_per_input - .iter() - .enumerate() - { - let producer = producer_candidates - .iter() - .copied() - .find(|s| { - physical.candidates[s.0].materialization == Materialization::NotMaterialized - }) - .unwrap_or(producer_candidates[0]); - let producer = select_from(physical, producer, selected); - selected.edges.push(SelectedEdge { - producer, - consumer: node, - input, - }); - } - node - } - - // The trait object returns what the catalog function returns with each - // candidate's own materialization. - #[test] - fn stage2_realizer_matches_direct_catalog_calls() { - let root = QueryRoot::Operator(lower_promql( - "sum by (job) (sum_over_time(m[1m]))", - AccuracyTarget::exact(), - )); - let roots = vec![root.clone()]; - let initial = generate_dag_of_logical_memo_groups(&[root]); - let per_computation = - generate_per_computation_candidates(&generate_dag_of_logical_memo_groups(&[ - roots[0].clone() - ])) - .unwrap(); - let memo = try_populate_pass1(initial, &per_computation).unwrap(); - let logical = LogicalASAPMemoCandidates { - memo: sharing_rules_and_shared_state_groups(memo).unwrap(), - }; - let entries = vec![entry(QueryRecurrence::Unknown)]; - let data = DataWorkload::default(); - let physical = crate::catalog::generate_physical_asap_memo_candidates( - &logical, &entries, &data, false, true, - ) - .unwrap(); - let selected = physical - .query_output_candidates(0) - .map(|candidate| { - let mut selected = DagOfSelectedCandidates { - candidate_of_node: Vec::new(), - edges: Vec::new(), - node_of_output_reference: Vec::new(), - }; - let root = select_from(&physical, candidate, &mut selected); - selected.node_of_output_reference.push(root); - selected - }) - .find(|selected| reconstruct_selected_candidates(&physical, selected).is_ok()) - .expect("some selection reconstructs"); - assert!(selected.candidate_of_node.len() > 1); - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries: &entries, - data: &data, - query_time_results_retained: false, - }; - let realizer: &dyn AssignmentRealizer = &realizer; - - let direct = - materialize_selected_candidates(&physical, &selected, &roots, &entries, &data, false) - .unwrap(); - let through_trait = realizer.realize(&physical, &selected).unwrap(); - assert_eq!( - format!("{:?}", direct.plan), - format!("{:?}", through_trait.plan) - ); - assert_eq!( - direct.operator_node_of_selected_node, - through_trait.operator_node_of_selected_node - ); - assert_eq!( - direct.tumbling_window_retention_ms, - through_trait.tumbling_window_retention_ms - ); + let _ = (physical, selected); + todo!("#1207 PR 6: build the selected DAG, each node at its PhysicalCandidate's properties") } } diff --git a/crates/physical-asap-optimization/src/catalog/reconstruction.rs b/crates/physical-asap-optimization/src/catalog/reconstruction.rs deleted file mode 100644 index fd7f0068b..000000000 --- a/crates/physical-asap-optimization/src/catalog/reconstruction.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Reconstruct selected immediate operators, preserving candidate identity. -//! -//! This decoder checks typed connections, not Stage 2 execution legality or -//! query accuracy targets. The caller must run Stage 2 and the plan validator. -use std::collections::HashMap; -use std::rc::Rc; - -use asap_accuracy::propagation::recompose_guarantees; -use asap_types::ir::{OperatorNode, QueryRoot}; - -use asap_types::stages::{ - DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, -}; - -/// Enumerate in exactly the traversal order used when rebinding. `children()` -/// puts direct children before scalar references; `map_children` follows fields. -pub fn ordered_inputs(node: &OperatorNode) -> Vec> { - let mut inputs = Vec::new(); - node.operator.map_children(|child| { - inputs.push(Rc::clone(child)); - Rc::clone(child) - }); - inputs -} - -/// Open schemas do not promise completeness; an equivalent derived output may -/// strengthen that promise to closed. Supplied uniqueness must imply every -/// required key; additional proven keys are allowed. Every field, configuration -/// parameter and time axis remains exact. Serialization includes sketch error/failure -/// metadata deliberately omitted by some parameter PartialEq implementations. -pub fn output_contract_satisfied( - expected: &OperatorNode, - output: &OperatorNode, -) -> Result { - let mut supplied = output.schema.clone(); - if !expected.schema.closed { - supplied.closed = false; - } - if !expected.schema.unique_keys.iter().all(|required| { - supplied - .unique_keys - .iter() - .any(|provided| provided.iter().all(|column| required.contains(column))) - }) { - return Ok(false); - } - supplied.unique_keys = expected.schema.unique_keys.clone(); - Ok(expected.result_kind == output.result_kind - && expected.schema == supplied - && serde_json::to_string(&expected.schema).map_err(|error| error.to_string())? - == serde_json::to_string(&supplied).map_err(|error| error.to_string())?) -} - -pub struct ReconstructedCandidates { - pub outputs: Vec>, - /// The operator node each selected node became, indexed by - /// [`SelectedNodeId`](asap_types::stages::SelectedNodeId), with - /// guarantees recomputed; each a distinct operator node. - pub(crate) operator_node_of_selected_node: Vec>, -} - -/// The nodes of `selected` rebuilt over their selected producers, one -/// operator node each, with guarantees recomposed. Checks the selection's -/// structure, not its cost. -pub fn reconstruct_selected_candidates( - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, -) -> Result { - super::validate_candidates(physical)?; - selected.validate(physical)?; - let count = selected.candidate_of_node.len(); - let mut inputs = selected - .candidate_of_node - .iter() - .map(|candidate| { - vec![ - None; - physical.candidates[candidate.0] - .producer_candidates_per_input - .len() - ] - }) - .collect::>(); - let mut consumers = vec![Vec::new(); count]; - for edge in &selected.edges { - inputs[edge.consumer.0][edge.input] = Some(edge.producer.0); - consumers[edge.producer.0].push(edge.consumer.0); - } - let mut remaining = inputs.iter().map(Vec::len).collect::>(); - let mut ready = Vec::new(); - for (node, candidate) in selected.candidate_of_node.iter().enumerate() { - let template = physical.physical_node(*candidate); - if template.children().len() != inputs[node].len() { - return Err(format!( - "candidate {} template inputs do not match producer requirements", - candidate.0 - )); - } - if remaining[node] == 0 { - ready.push(node); - } - } - let mut rebuilt: Vec>> = vec![None; count]; - while let Some(node) = ready.pop() { - let id = selected.candidate_of_node[node].0; - let template = physical.physical_node(PhysicalCandidateId(id)); - let expected = ordered_inputs(template); - let mut producers = Vec::with_capacity(inputs[node].len()); - for (input, producer) in inputs[node].iter().enumerate() { - let producer = rebuilt[producer.unwrap()].as_ref().unwrap(); - if !output_contract_satisfied(&expected[input], producer)? { - return Err(format!( - "candidate {id} input {input} has incompatible schema or state configuration" - )); - } - producers.push(Rc::clone(producer)); - } - let mut next = 0; - // Re-derive structural metadata over the selected inputs, preserving - // only explicit naming overrides. Logical rewrite templates can retain - // pre-rewrite keys/nullability; copying their entire schema is invalid. - let rebuilt_node = Rc::new( - template - .with_new_children(|_| { - let producer = Rc::clone(&producers[next]); - next += 1; - producer - }) - .map_err(|e| format!("candidate {id}: {e}"))? - .with_guarantee(template.guarantee.clone()), - ); - rebuilt_node - .validate_structure() - .map_err(|e| format!("candidate {id}: {e}"))?; - rebuilt[node] = Some(rebuilt_node); - for &consumer in &consumers[node] { - remaining[consumer] -= 1; - if remaining[consumer] == 0 { - ready.push(consumer); - } - } - } - let rebuilt = rebuilt - .into_iter() - .map(|node| node.ok_or_else(|| "selected nodes form a cycle".to_string())) - .collect::, _>>()?; - let roots = selected - .node_of_output_reference - .iter() - .map(|node| QueryRoot::Operator(Rc::clone(&rebuilt[node.0]))) - .collect::>(); - let outputs = recompose_guarantees(&roots) - .map_err(|e| e.to_string())? - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(node), - QueryRoot::Scalar(_) => Err("unexpected scalar during operator reconstruction".into()), - }) - .collect::, String>>()?; - let old_outputs = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Rc::clone(node), - QueryRoot::Scalar(_) => unreachable!("operator references were constructed above"), - }) - .collect::>(); - let identities = paired_node_identities(&old_outputs, &outputs)?; - let operator_node_of_selected_node = rebuilt - .into_iter() - .map(|node| { - identities - .get(&Rc::as_ptr(&node)) - .cloned() - .ok_or_else(|| "selected node lost during guarantee recomposition".to_string()) - }) - .collect::, _>>()?; - Ok(ReconstructedCandidates { - outputs, - operator_node_of_selected_node, - }) -} - -/// Match corresponding graph positions without structural interning. Reject -/// splitting or merging identities, and rewrites that change immediate operators. -pub fn paired_node_identities( - before: &[Rc], - after: &[Rc], -) -> Result>, String> { - if before.len() != after.len() { - return Err("output reference count changed".into()); - } - let mut identities = HashMap::<*const OperatorNode, Rc>::new(); - let mut reverse = HashMap::new(); - let mut pending = before - .iter() - .cloned() - .zip(after.iter().cloned()) - .collect::>(); - while let Some((old, new)) = pending.pop() { - if let Some(existing) = identities.get(&Rc::as_ptr(&old)) { - if !Rc::ptr_eq(existing, &new) { - return Err("selected node was split into multiple physical nodes".into()); - } - continue; - } - if reverse - .insert(Rc::as_ptr(&new), Rc::as_ptr(&old)) - .is_some_and(|previous| previous != Rc::as_ptr(&old)) - { - return Err("distinct selected nodes were merged".into()); - } - let old_operator = old.operator.map_children(|_| ()); - let new_operator = new.operator.map_children(|_| ()); - // Compare all serialized parameters, including sketch probability and - // non-finite scalar values. PartialEq rejects an unchanged NaN, while - // JSON erases the distinction between NaN and either infinity. - if rmp_serde::to_vec(&old_operator).map_err(|e| e.to_string())? - != rmp_serde::to_vec(&new_operator).map_err(|e| e.to_string())? - { - return Err("operator implementation changed; explicit physical candidate enumeration is required".into()); - } - let old_children = ordered_inputs(&old); - let new_children = ordered_inputs(&new); - if old_children.len() != new_children.len() { - return Err("candidate input count changed".into()); - } - pending.extend(old_children.into_iter().zip(new_children)); - identities.insert(Rc::as_ptr(&old), new); - } - Ok(identities) -} diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index aed2e101c..252aecf16 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -1,2437 +1,18 @@ -//! #509 Stage 2, physical ASAP-aware optimization: materialization and -//! physical operator implementation. +//! #509 Stage 2, physical ASAP-aware optimization (concepts doc, +//! section 3, "Stage 2: physical alternatives"). //! //! The planner facade calls [`generate_physical_asap_memo_candidates`] on -//! Pass 2's memo (`asap_types::stages::LogicalASAPMemoCandidates`): every -//! operator node of every memo candidate with each allowed timing and -//! materialization, the producers of each input and the pruned candidates +//! Pass 2's output (`asap_types::stages::LogicalASAPMemoCandidates`): each +//! logical candidate expanded into one physical candidate per valid +//! $(\tau, \ell, \sigma)$, each logical edge into its physical edges with +//! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives //! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from -//! that output, and [`Stage2AssignmentRealizer`] builds the selected -//! assignment as a physical plan. The rules below apply to both, and to the -//! whole-workload reference pricer `optimize_physical_asap` (feature -//! `reference-pricer`, for the developer tools), which turns one -//! CandidateLogicalASAPDAG into its CandidatePhysicalASAPDAGs. The crate -//! depends on no other stage crate. -//! -//! ## Materialization -//! -//! Each summary build (a `SummaryAgg` or a `SummaryTumblingWindowChain`) is one -//! materialization candidate (`MaterializationCandidate`) whose -//! output materialization may store (#509 §Materialization, Algorithm 3). -//! So is the highest node of each other sub-DAG that two or more queries -//! read (e.g. a `Scan` that a 1 m and a 5 m window read): one a root or a -//! node only one query reads consumes. Such a node has only -//! `NotMaterialized` and `MatAtQuery(Memory)`, so a shared sub-DAG adds one -//! option, not one per node. A summary build has up to four options: -//! -//! * `NotMaterialized`: rebuilt at query time by every query that reads it; -//! * `MatAtIngestion(Memory)`: maintained as data arrives and kept for the -//! queries, when the build is eligible (`IngestionPruneReason`); -//! * `MatAtQuery(Memory)`: built once at query time for every query of one -//! evaluation that reads it and reused only within that evaluation, -//! offered for a build two or more queries read; -//! * `MatAtQueryKeptAcrossEvaluations(Memory)`: built at query time and -//! kept across evaluations (#509 Example 4's B3), offered when the -//! deployment keeps query-time results (the caller's -//! `query_time_results_retained`) for a -//! tumbling-window chain that builds one new tumbling window per evaluation -//! (`builds_one_tumbling_window_per_evaluation`): each evaluation builds the -//! newest tumbling window and reuses the others. Both query-time options are offered -//! when both apply, and Stage 3 prices each. -//! -//! A build at ingestion time needs every build below it at ingestion time too -//! (#509 Algorithm 3, line 13), and every shared non-summary node below it -//! `NotMaterialized`; other assignments are not generated. Every -//! other combination of options is one physical candidate, the -//! all-`NotMaterialized` one first: nothing is capped. A node that one -//! query reads at ingestion time and another at query time is copied, one -//! copy per timing (`split_shared_by_timing`). -//! -//! A build may run at ingestion time (S4) only when: -//! -//! * the data workload is continuously ingesting or mixed (unknown arrival -//! counts as not ingesting); -//! * every query reading it repeats and is predictable; -//! * it has no filter: an ingestion-time build reads raw data samples, which -//! carry none of the columns a filter reads; -//! * if it is a counter state (`Rate`, `Increase`, `IRate`), whose update -//! needs each series' samples in time order, the data is continuously -//! ingesting and every source below it is a time series. Assumption: -//! continuously ingesting PromQL data arrives in time order per series, -//! as Prometheus assumes (it rejects out-of-order samples). Table (SQL) -//! sources and mixed data, whose at-rest part may load in any order, are -//! not taken to be ordered; -//! * it is a tumbling-window chain whose readers all evaluate on its tumbling window -//! boundaries, anchored at the epoch (each fixed interval a multiple of the -//! tumbling-window width, and each known phase on that grid), or a whole-window build whose readers -//! all run at one fixed interval with agreeing declared phases, over a -//! window that is not shifted and whose evaluations all end on the grid -//! of its width anchored at the epoch (the interval and a declared phase -//! are multiples of the window; a reader with no declared phase is taken -//! to evaluate at multiples of its interval, as a chain's is), and none -//! is a range evaluation of more than one step per request (one window -//! per request cannot serve every step). -//! So a window equal to its interval (`m[5m]` every 5 minutes) is one -//! tumbling window maintained at ingestion time; -//! * everything below it can run at ingestion time. -//! -//! ## Range evaluations -//! -//! A query whose workload entry declares a -//! [`RangeEvaluation`](asap_types::workload::RangeEvaluation) evaluates at -//! every `step_ms` from `end − span_ms` forward to each request's end. -//! Every step reads the -//! tumbling windows of a tumbling-window chain, so each tumbling-window width must divide the step -//! of every range-evaluated reader. Stage 1 offers only such widths -//! (`QueryWorkloadEntry::evaluation_grid_ms`); a candidate with a chain off -//! a reader's step grid fails with -//! [`PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep`] and the caller -//! prunes it with that reason. -//! -//! ## Physical operator implementation -//! -//! The runtime has one implementation per logical operator except exact -//! top-k, which it cannot run as `Aggregate{[TopK]}`. Stage 2 records that -//! choice in the DAG by rewriting it to a per-group sort followed by a -//! per-group limit, the shape the PromQL frontend uses for generic `topk`. -//! A `Project` over the limited rows restores the logical `[keys.., topk_k]` -//! columns, so an exact root has the same column names as a summary -//! candidate's and as the frontend declares. Over PromQL rows it also keeps -//! the series identity, after them: `topk` answers whole input series, so -//! each answer carries the chosen series' full label set. PromQL rows without -//! one (e.g. of `sum by (job)`) keep their other string label columns instead, -//! and SQL rows their other group keys (the ranked item, e.g. `host` of -//! `GROUP BY host ORDER BY COUNT(*) DESC LIMIT k`). -//! `topk_k` carries the ranked value's type and nullability (the frontend -//! declares a non-null `Utf8` placeholder); casting the value to `Utf8` would -//! not make it the same answer. -//! An `Aggregate{[TopK]}` over a per-entity reduction, or with other -//! measures, a filter or `HAVING`, has no such rewrite, and its candidate -//! fails with a [`PhysicalASAPOptimizationError`]; the caller prunes that -//! candidate alone, with the error as its reason. -//! A summary needs no rewrite: `SummaryAgg` → `SummaryEstimate` already is -//! build → estimate. Every node keeps the guarantee Stage 1 composed for it. +//! that output, and [`Stage2AssignmentRealizer`] builds the selected DAG as a +//! physical plan. The crate depends on no other stage crate. + mod catalog; -mod split_by_timing; pub use catalog::{ - generate_physical_asap_memo_candidates, CandidateInventoryError, ConfigurationError, - ErrorGeneratingPhysicalASAPMemoCandidates, Stage2AssignmentRealizer, -}; - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::Source; -use asap_types::ir::operator::{AggIntent, Reduction}; -use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; -use asap_types::ir::schema::{ExactKind, FieldDataType}; -use asap_types::ir::{ - apply_materialization_timings, is_filtered_build, ASAPOp, MaterializationAssignment, - NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr, - SchemaDerivationError, SortKey, -}; -use asap_types::physical::Materialization; -use asap_types::physical::Medium; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{ - DataArrival, DataWorkload, Predictability, QueryRecurrence, QueryWorkloadEntry, RepeatedDemand, + generate_physical_asap_memo_candidates, ErrorGeneratingPhysicalASAPMemoCandidates, + Stage2AssignmentRealizer, }; -use split_by_timing::split_shared_by_timing; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum PhysicalASAPOptimizationError { - #[error(transparent)] - Structure(#[from] SchemaDerivationError), - #[error(transparent)] - Timing(#[from] ExecutionDataStateError), - #[error("exact top-k has no Sort + Limit implementation: {0}")] - ExactTopK(String), - /// Bad candidate for the workload: a tumbling-window chain whose tumbling window - /// width does not divide the step of a query that reads it as a range - /// request, so some step would end inside a tumbling window and could not read - /// the tumbling windows. Stage 1 offers only tumbling windows on every reader's step grid; - /// the candidate is pruned with this reason. - #[error( - "tumbling-window width {tumbling_window_width_ms} ms does not divide the {step_ms} ms step of \ - a range evaluation that reads it" - )] - TumblingWindowWidthOffRangeStep { - /// The chain's tumbling-window width. - tumbling_window_width_ms: u64, - /// The reader's range-evaluation step. - step_ms: u64, - }, -} - -/// Why a summary build cannot be materialized at ingestion time (S4). -/// Shared by complete-plan Stage 2 and explicit selected-reader eligibility. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum IngestionPruneReason { - /// The data workload is at rest, or its arrival is unknown. - NotIngesting, - /// A query reading it is not repeated. - NotRepeated, - /// A query reading it is not predictable. - NotPredictable, - /// It has a filter, which only a query-time build can apply. - Filtered, - /// It is a counter state, whose update needs each series' samples in - /// time order, over data not taken to arrive in that order: only - /// continuously ingesting time-series (PromQL) sources are. - CounterNeedsTimeOrder, - /// A whole-window build whose readers do not all run at one fixed - /// interval and one phase, or whose window is shifted or not a range. - WindowNotFixed, - /// A whole-window build whose evaluations do not all end on the grid of - /// its window's width anchored at the epoch, where the backend builds - /// it (e.g. `m[20s]` every 20 s at phase 5 s, every 30 s, or `m[15m]` - /// every 5 minutes, which tumbling windows serve): the interval or the declared - /// phase is not a multiple of the window. - WindowOffEpochGrid, - /// A whole-window build read by a range evaluation of more than one - /// step per request: it keeps one window per request, not one per step. - RangeEvaluatedWholeWindow, - /// A tumbling-window chain read at a known phase off its tumbling-window boundaries, - /// which are anchored at the epoch (e.g. 20 s tumbling windows read at phase 5 s): - /// built once each, its tumbling windows cannot end at every evaluation. - TumblingWindowsOffPhase, - /// A tumbling-window chain read at a fixed interval that its width does - /// not divide (e.g. 15 s tumbling windows read every 10 s): built once - /// each on the grid anchored at the epoch, its tumbling windows cannot - /// end at every evaluation. - TumblingWindowsOffInterval, - /// Something below it can only run at query time. - UpstreamAtQueryTime(String), - /// It is not a summary build (e.g. a shared `Scan`): only a summary's - /// state is maintained as data arrives. - NotASummaryBuild, -} - -/// Workload eligibility for one summary and only its selected query readers. -/// These are local options, not a complete physical assignment. Ingestion still -/// requires selected upstream implementations to pass timing/down-closure checks. -#[derive(Debug)] -pub(crate) struct SummaryMaterializationEligibility { - pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - pub(crate) may_keep_across_evaluations: bool, - /// A tumbling window's selected readers must also meet their range-step grids. - pub(crate) range_compatibility: Result<(), PhysicalASAPOptimizationError>, -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum SummaryEligibilityError { - #[error("selected reader {query} has no workload entry")] - UnknownReader { query: usize }, - #[error("materialization eligibility requires a summary build or tumbling-window chain")] - NotSummary, - #[error("materialization eligibility requires at least one selected reader")] - NoSelectedReaders, - #[error("tumbling-window width must be positive")] - ZeroPaneWidth, -} - -/// Evaluate an implemented summary against explicit selected query indices. -/// Never supply the union of potential readers. Duplicate indices count once; -/// missing workload entries are rejected as malformed reader references. The node must carry the actual bound source/filter/window subtree, -/// and `data` describes that source's arrival. This function does not implement -/// operators, assign upstream timing, or prove complete execution legality. -pub(crate) fn summary_materialization_eligibility( - node: &OperatorNode, - selected_readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result { - if !is_summary_build(node) { - return Err(SummaryEligibilityError::NotSummary); - } - if selected_readers.is_empty() { - return Err(SummaryEligibilityError::NoSelectedReaders); - } - if matches!( - node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 0, - .. - }) - ) { - return Err(SummaryEligibilityError::ZeroPaneWidth); - } - if let Some(&query) = selected_readers - .iter() - .find(|&&query| query >= entries.len()) - { - return Err(SummaryEligibilityError::UnknownReader { query }); - } - let mut readers = selected_readers.to_vec(); - readers.sort_unstable(); - readers.dedup(); - Ok(summary_reader_eligibility( - node, - &readers, - entries, - data, - query_time_results_retained, - )) -} - -fn summary_reader_eligibility( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> SummaryMaterializationEligibility { - let recurrences: Option> = readers - .iter() - .map(|&query| entries.get(query).map(|entry| &entry.recurrence)) - .collect(); - let may_keep_across_evaluations = query_time_results_retained - && match (node.asap(), recurrences) { - ( - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }), - Some(recurrences), - ) => builds_one_tumbling_window_per_evaluation(*tumbling_window_width_ms, recurrences), - _ => false, - }; - SummaryMaterializationEligibility { - at_ingestion: ingestion_eligibility(node, readers, entries, data), - may_keep_across_evaluations, - range_compatibility: check_summary_range_steps(node, readers, entries), - } -} - -/// One summary build, or one highest shared non-summary node, Stage 2 may -/// store, with its materialization options. -/// It is not a whole-workload plan like [`PhysicalWorkloadPlan`]: it is Stage -/// 2's enumeration bookkeeping, and the physical candidates are the -/// combinations of the materialization candidates' options. For example, -/// over one metric: -/// -/// ```text -/// q1: quantile_over_time(0.99, latency[5m]) every 1m -/// q2: max_over_time(latency[5m]) every 1m -/// ``` -/// -/// If Stage 1 chose KLL tumbling windows for q1 and exact max tumbling windows for q2, Stage 2 -/// finds two materialization candidates: -/// -/// | materialization candidate | readers | options | -/// |----------------------------------|---------|---------------------------------| -/// | Kll ×5 tumbling windows (q1) | q1 | NotMaterialized, MatAtIngestion | -/// | Max acc ×5 tumbling windows (q2) | q2 | NotMaterialized, MatAtIngestion | -/// -/// One read by two or more queries also gets `MatAtQuery` (built once per -/// evaluation and reused), and so does the highest node of a shared -/// non-summary sub-DAG, e.g. a `Scan` both queries read when Stage 1 -/// shares it. The physical candidates are the product of the -/// options, here 2 × 2 = 4, less the assignments that put a build at -/// ingestion time over one that is not; Stage 3 prices each. -/// `at_ingestion` records why ingestion is ruled out when it is (e.g. -/// `Filtered`, `NotRepeated`). Each physical candidate carries the -/// resulting assignment ([`PhysicalWorkloadPlan::materialization`]). -#[derive(Debug, Clone)] -pub(crate) struct MaterializationCandidate { - /// The build in the implemented DAG. - pub(crate) node: Rc, - /// The queries (indices into the roots) that read it. - pub(crate) readers: Vec, - /// `Ok` when `MatAtIngestion` is an option. - pub(crate) at_ingestion: Result<(), IngestionPruneReason>, - /// Whether `MatAtQueryKeptAcrossEvaluations` is an option: the - /// deployment keeps query-time results and it is a tumbling-window chain - /// that builds one new tumbling window per evaluation. - pub(crate) may_keep_across_evaluations: bool, -} - -impl MaterializationCandidate { - /// Its options, `NotMaterialized` first: `MatAtIngestion` when eligible, - /// `MatAtQuery` when two or more queries read it, and - /// `MatAtQueryKeptAcrossEvaluations` when it may be kept across - /// evaluations. - pub(crate) fn options(&self) -> Vec { - let mut options = vec![Materialization::NotMaterialized]; - if self.at_ingestion.is_ok() { - options.push(Materialization::MatAtIngestion(Medium::Memory)); - } - if self.readers.len() > 1 { - options.push(Materialization::MatAtQuery(Medium::Memory)); - } - if self.may_keep_across_evaluations { - options.push(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Memory, - )); - } - options - } -} - -/// Whether a tumbling-window chain of `tumbling_window_width_ms`-wide tumbling windows read by -/// queries repeating as `readers` builds exactly one new tumbling window per -/// evaluation: there is a reader, every reader repeats at a fixed interval -/// equal to the tumbling-window width, and the readers whose evaluation phase is known -/// share one phase (modulo the interval). Its other tumbling windows are the previous -/// evaluations' newest, so a deployment that keeps query-time results -/// reuses them (#509 Example 4's B3). -/// -/// Co-scheduling assumption: readers with the same interval and no known -/// phase run at the same instants as each other and as the readers with a -/// known phase, as plan selection (`asap-plan-selection`) assumes when it counts -/// one evaluation per interval. Readers at different known phases would -/// each need their own newest tumbling window, so they do not share one kept chain. -pub(crate) fn builds_one_tumbling_window_per_evaluation<'a>( - tumbling_window_width_ms: u64, - readers: impl IntoIterator, -) -> bool { - if tumbling_window_width_ms == 0 { - return false; - } - let mut phase_ms = None; - let mut any_reader = false; - for recurrence in readers { - any_reader = true; - if recurrence.fixed_interval_ms() != Some(tumbling_window_width_ms) { - return false; - } - if let QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) = recurrence - { - let phase = evaluation_phase.0 % tumbling_window_width_ms; - if *phase_ms.get_or_insert(phase) != phase { - return false; - } - } - } - any_reader -} - -/// Stage 2 for one CandidateLogicalASAPDAG: implement every operator -/// (physical operator implementation), then one physical candidate per -/// materialization assignment of its mat_candidates, the all-`NotMaterialized` one -/// first. `entries[i]` is the workload entry of query `i` (`roots[i]`): -/// its recurrence and predictability decide whether state may be kept from -/// ingestion time; a missing entry is unknown. `id` is left empty for the -/// caller to name. Sharing between `roots` is preserved: one cache serves the -/// whole workload. `query_time_results_retained` says whether a query-time -/// result may be kept across evaluations. -/// -/// The reference pricer: only the developer tools (and tests) enumerate a -/// whole workload this way; planning goes through -/// [`generate_physical_asap_memo_candidates`]. -#[cfg(any(test, feature = "reference-pricer"))] -pub fn optimize_physical_asap( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result, PhysicalASAPOptimizationError> { - let (implemented, _, mat_candidates) = - implement_and_find_candidates(roots, entries, data, query_time_results_retained)?; - materialization_assignments(&mat_candidates) - .map(|assignment| { - assigned_candidate(from_logical, &implemented, &mat_candidates, assignment) - }) - .collect() -} - -/// Stage 2 for one CandidateLogicalASAPDAG and one materialization -/// assignment: the physical candidate [`optimize_physical_asap`] lists for -/// it. `choose(readers)` gives the option for each materialization -/// candidate (each summary build in discovery order, query by query, each -/// query's builds as a walk from its root meets them, then each highest -/// shared non-summary node in the same order), `readers` being the queries -/// that read it. `None` when -/// a chosen option is not one of the build's, or the assignment -/// materializes a build at ingestion time above one at query time. -#[cfg(feature = "test-support")] -pub fn optimize_physical_asap_assigned( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut dyn FnMut(&[usize]) -> Option, -) -> Result, PhysicalASAPOptimizationError> { - optimize_physical_asap_chosen( - from_logical, - roots, - entries, - data, - query_time_results_retained, - &mut |_, readers, _| choose(readers), - ) -} - -/// [`optimize_physical_asap_chosen`]'s choice per materialization -/// candidate: from the logical node it implements, its readers and its -/// options. `None` rejects the assignment. -pub(crate) type ChooseMaterialization<'a> = - dyn FnMut(*const OperatorNode, &[usize], &[Materialization]) -> Option + 'a; - -/// Stage 2 for one CandidateLogicalASAPDAG and one materialization -/// assignment chosen node by node: `choose(logical, readers, options)` -/// gives the option for each materialization candidate, `logical` being -/// the address of the logical node of `roots` it implements (null when it -/// implements none, e.g. a node the implementation adds) and `readers` the -/// queries that read it. `None` when `choose` rejects, a chosen option is -/// not one of the candidate's, or the assignment materializes a build at ingestion time -/// above one at query time. The search that decodes one plan from a model -/// solution uses it to build that plan alone, not every assignment. -pub(crate) fn optimize_physical_asap_chosen( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut ChooseMaterialization<'_>, -) -> Result, PhysicalASAPOptimizationError> { - let (implemented, cache, mat_candidates) = - implement_and_find_candidates(roots, entries, data, query_time_results_retained)?; - let logical_of: HashMap<*const OperatorNode, *const OperatorNode> = cache - .iter() - .map(|(logical, implemented)| (Rc::as_ptr(implemented), *logical)) - .collect(); - let mut assignment = Vec::with_capacity(mat_candidates.len()); - for mat_candidate in &mat_candidates { - let logical = logical_of - .get(&Rc::as_ptr(&mat_candidate.node)) - .copied() - .unwrap_or(std::ptr::null()); - let options = mat_candidate.options(); - let Some(m) = - choose(logical, &mat_candidate.readers, &options).filter(|m| options.contains(m)) - else { - return Ok(None); - }; - assignment.push(m); - } - let nodes: Vec<_> = mat_candidates.iter().map(|c| Rc::clone(&c.node)).collect(); - if !down_closed(&mat_candidates, &candidates_below(&nodes), &assignment) { - return Ok(None); - } - assigned_candidate(from_logical, &implemented, &mat_candidates, assignment).map(Some) -} - -/// The physical candidate of the implemented roots `implemented` with -/// `assignment[c]` chosen for `mat_candidates[c]`. -fn assigned_candidate( - from_logical: &str, - implemented: &[QueryRoot], - mat_candidates: &[MaterializationCandidate], - assignment: Vec, -) -> Result { - let materialized: Vec<_> = mat_candidates - .iter() - .zip(&assignment) - .filter(|(_, m)| **m != Materialization::NotMaterialized) - .map(|(mat_candidate, m)| (mat_candidate, *m)) - .collect(); - let mut options = MaterializationAssignment::with_default(Materialization::NotMaterialized); - for (mat_candidate, materialization) in &materialized { - options.set(&mat_candidate.node, *materialization); - } - let operators: Vec> = implemented - .iter() - .flat_map(|root| root.operator_refs().into_iter().cloned()) - .collect(); - let (split, options) = split_shared_by_timing(&operators, &options); - let replaced: HashMap<_, _> = operators - .iter() - .map(Rc::as_ptr) - .zip(split.iter().cloned()) - .collect(); - let split_roots = map_roots(implemented, &mut |node| { - Ok(replaced[&Rc::as_ptr(node)].clone()) - })?; - let mut timing = NodeToTimedNodeCache::new(); - let timed = map_roots(&split_roots, &mut |node| { - Ok(apply_materialization_timings(node, &options, &mut timing)?) - })?; - let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; - let materialization = dag - .nodes - .iter() - .filter(|n| { - matches!( - n.payload, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) || n.materialization != Materialization::NotMaterialized - }) - .map(|n| (n.id, n.materialization)) - .collect(); - Ok(PhysicalWorkloadPlan { - id: String::new(), - from_logical: from_logical.to_string(), - roots: timed, - dag, - materialization, - }) -} - -/// `roots` implemented, the implementation of each logical node, and the -/// materialization candidates of the implemented roots. -fn implement_and_find_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Result< - ( - Vec, - NodeToImplementedNodeCache, - Vec, - ), - PhysicalASAPOptimizationError, -> { - let mut cache = HashMap::new(); - let implemented = map_roots(roots, &mut |node| implement(node, &mut cache))?; - let mat_candidates = - materialization_candidates(&implemented, entries, data, query_time_results_retained); - check_tumbling_windows_on_range_steps(&mat_candidates, entries)?; - Ok((implemented, cache, mat_candidates)) -} - -/// `Err` for the first tumbling-window chain whose tumbling-window width does not -/// divide the step of a reader's range evaluation -/// ([`PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep`]): that step -/// would end inside a tumbling window. -fn check_tumbling_windows_on_range_steps( - mat_candidates: &[MaterializationCandidate], - entries: &[QueryWorkloadEntry], -) -> Result<(), PhysicalASAPOptimizationError> { - for candidate in mat_candidates { - check_summary_range_steps(&candidate.node, &candidate.readers, entries)?; - } - Ok(()) -} - -fn check_summary_range_steps( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], -) -> Result<(), PhysicalASAPOptimizationError> { - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) = node.asap() - else { - return Ok(()); - }; - for &query in readers { - let Some(range) = entries - .get(query) - .and_then(|entry| entry.range_evaluation) - .filter(|range| range.evaluations_per_request() > 1) - else { - continue; - }; - if range.step_ms % tumbling_window_width_ms.max(&1) != 0 { - return Err( - PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { - tumbling_window_width_ms: *tumbling_window_width_ms, - step_ms: range.step_ms, - }, - ); - } - } - Ok(()) -} - -/// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`). -fn is_summary_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) -} - -/// The materialization candidates of `roots` (implemented), with their -/// readers, whether each may run at ingestion time and whether it may be -/// kept from query time across evaluations: every summary build, in -/// discovery order, then every highest shared non-summary node, in -/// discovery order. A shared node is one two or more queries read; the -/// highest are those a root or an unshared node reads (e.g. a `Scan` that -/// two windows read). Only a highest one is offered `MatAtQuery`, so a -/// shared sub-DAG adds one candidate, not one per node. -pub(crate) fn materialization_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, -) -> Vec { - let mut reached: Vec> = Vec::new(); - let mut readers: HashMap<*const OperatorNode, Vec> = HashMap::new(); - let mut consumers: HashMap<*const OperatorNode, Vec<*const OperatorNode>> = HashMap::new(); - let mut root_nodes: HashSet<*const OperatorNode> = HashSet::new(); - for (query, root) in roots.iter().enumerate() { - for operator in root.operator_refs() { - root_nodes.insert(Rc::as_ptr(operator)); - for node in OperatorNode::reachable(operator) { - let entry = readers.entry(Rc::as_ptr(&node)).or_default(); - if entry.is_empty() { - reached.push(Rc::clone(&node)); - for child in node.children() { - consumers - .entry(Rc::as_ptr(child)) - .or_default() - .push(Rc::as_ptr(&node)); - } - } - if !entry.contains(&query) { - entry.push(query); - } - } - } - } - let shared = |node: *const OperatorNode| readers[&node].len() > 1; - let highest_shared = |node: &Rc| { - let ptr = Rc::as_ptr(node); - shared(ptr) - && (root_nodes.contains(&ptr) - || consumers - .get(&ptr) - .is_some_and(|c| c.iter().any(|&consumer| !shared(consumer)))) - }; - let (nodes, shared_results): (Vec<_>, Vec<_>) = - reached.into_iter().partition(|n| is_summary_build(n)); - let below = candidates_below(&nodes); - let builds = nodes.iter().enumerate().map(|(c, node)| { - let readers = readers[&Rc::as_ptr(node)].clone(); - let eligibility = - summary_reader_eligibility(node, &readers, entries, data, query_time_results_retained); - let at_ingestion = eligibility.at_ingestion.and_then(|()| { - // Only this build and the builds below it at ingestion time. - let mut trial = - MaterializationAssignment::with_default(Materialization::NotMaterialized); - for &v in below[c].iter().chain([&c]) { - trial.set(&nodes[v], Materialization::MatAtIngestion(Medium::Memory)); - } - apply_materialization_timings(node, &trial, &mut NodeToTimedNodeCache::new()) - .map(|_| ()) - .map_err(|e| IngestionPruneReason::UpstreamAtQueryTime(e.to_string())) - }); - MaterializationCandidate { - node: Rc::clone(node), - readers, - at_ingestion, - may_keep_across_evaluations: eligibility.may_keep_across_evaluations, - } - }); - // A node fixed at ingestion time cannot be built at query time. - let shared_results = shared_results - .into_iter() - .filter(|n| highest_shared(n) && n.timing != Some(ExecutionTiming::IngestionTime)) - .map(|node| MaterializationCandidate { - readers: readers[&Rc::as_ptr(&node)].clone(), - node, - at_ingestion: Err(IngestionPruneReason::NotASummaryBuild), - may_keep_across_evaluations: false, - }); - builds.chain(shared_results).collect() -} - -/// Whether every scan below `node` reads a time series (a PromQL source). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - -/// The phase, modulo `tumbling_window_width_ms` (which must be nonzero), of -/// every evaluation of `demand`, if known. A plain fixed interval has cadence but no phase, so -/// it is unknown. An explicit schedule has one only when every timestamp in -/// it has the same phase. -fn plan_tumbling_window_phase( - demand: &RepeatedDemand, - tumbling_window_width_ms: u64, -) -> Option { - match demand { - RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - } => Some(evaluation_phase.0 % tumbling_window_width_ms), - RepeatedDemand::Scheduled(times) => { - let first = times.first().map(|t| t.0 % tumbling_window_width_ms); - first.filter(|phase| { - times - .iter() - .all(|t| t.0 % tumbling_window_width_ms == *phase) - }) - } - RepeatedDemand::FixedInterval(_) | RepeatedDemand::EstimatedRate(_) => None, - } -} - -/// The workload rules of S4 for `node` (module doc). -fn ingestion_eligibility( - node: &OperatorNode, - readers: &[usize], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, -) -> Result<(), IngestionPruneReason> { - use IngestionPruneReason as Why; - if !matches!( - data.arrival, - DataArrival::Ingesting { .. } | DataArrival::Mixed { .. } - ) { - return Err(Why::NotIngesting); - } - let demands: Vec> = - readers.iter().map(|&q| entries.get(q)).collect(); - if demands - .iter() - .any(|d| !d.is_some_and(|d| matches!(d.recurrence, QueryRecurrence::Repeated(_)))) - { - return Err(Why::NotRepeated); - } - if demands - .iter() - .any(|d| !d.is_some_and(|d| matches!(d.predictability, Predictability::Predictable { .. }))) - { - return Err(Why::NotPredictable); - } - if is_filtered_build(node) { - return Err(Why::Filtered); - } - let (family, chain_tumbling_window_width_ms) = match node.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => (family, None), - Some(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - .. - }) => (family, Some(*tumbling_window_width_ms)), - _ => unreachable!("a materialization candidate is a summary build"), - }; - if matches!( - family, - FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase | ExactKind::IRate, _) - ) { - // Assumption (module doc): continuously ingesting PromQL data - // arrives in time order per series, as Prometheus assumes. - let ordered = - matches!(data.arrival, DataArrival::Ingesting { .. }) && reads_only_time_series(node); - if !ordered { - return Err(Why::CounterNeedsTimeOrder); - } - } - if let Some(tumbling_window_width_ms) = chain_tumbling_window_width_ms { - // A chain's tumbling windows are built once each, on the grid anchored at the - // epoch (the plan carries no tumbling-window origin): every reader must - // evaluate on its boundaries. A reader at a fixed interval needs an - // interval the width divides, whatever its phase; a reader of known - // phase also needs that phase on the grid. A reader of unknown phase - // is taken to run at multiples of its interval (phase 0), as for a - // whole-window build below. - for d in demands.iter().flatten() { - let QueryRecurrence::Repeated(demand) = &d.recurrence else { - continue; - }; - if d.recurrence - .fixed_interval_ms() - .is_some_and(|interval_ms| !interval_ms.is_multiple_of(tumbling_window_width_ms)) - { - return Err(Why::TumblingWindowsOffInterval); - } - if tumbling_window_width_ms == 0 { - return Err(Why::TumblingWindowsOffPhase); - } - let Some(phase_ms) = plan_tumbling_window_phase(demand, tumbling_window_width_ms) - else { - if matches!(demand, RepeatedDemand::Scheduled(_)) { - // A schedule whose times are not all on one tumbling-window phase. - return Err(Why::TumblingWindowsOffPhase); - } - continue; - }; - if phase_ms != 0 { - return Err(Why::TumblingWindowsOffPhase); - } - } - return Ok(()); - } - if demands.iter().flatten().any(|d| { - d.range_evaluation - .is_some_and(|range| range.evaluations_per_request() > 1) - }) { - return Err(Why::RangeEvaluatedWholeWindow); - } - // A whole-window build: one tumbling window per evaluation, built (like - // a chain's tumbling windows) on the grid of its width anchored at the epoch, so - // every evaluation must end on that grid. The readers share one fixed - // interval and their declared phases agree; a reader with no declared - // phase is taken to evaluate at multiples of its interval (phase 0), as - // a chain's is, which lies on the grid once the interval does. - let mut interval_ms = None; - let mut phase_ms = None; - for d in demands.into_iter().flatten() { - let (interval, phase) = match &d.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - (u64::from(interval.0), None) - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => ( - u64::from(interval.0), - evaluation_phase.0.checked_rem(u64::from(interval.0)), - ), - _ => return Err(Why::WindowNotFixed), - }; - if interval == 0 { - return Err(Why::WindowNotFixed); - } - if *interval_ms.get_or_insert(interval) != interval { - return Err(Why::WindowNotFixed); - } - if let Some(phase) = phase { - if *phase_ms.get_or_insert(phase) != phase { - return Err(Why::WindowNotFixed); - } - } - } - let Some(interval_ms) = interval_ms else { - return Err(Why::WindowNotFixed); - }; - let Some(NonASAPOp::TimeRange { - range, - kind: asap_types::ir::TimeRangeKind::Range, - child, - }) = node.children()[0].non_asap() - else { - return Err(Why::WindowNotFixed); - }; - if matches!(child.non_asap(), Some(NonASAPOp::TimeShift { .. })) { - return Err(Why::WindowNotFixed); - } - let window_ms = u64::try_from(range.as_millis()) - .ok() - .filter(|&ms| ms > 0) - .ok_or(Why::WindowNotFixed)?; - let on_grid = |ms: u64| ms.is_multiple_of(window_ms); - if on_grid(interval_ms) && phase_ms.is_none_or(on_grid) { - Ok(()) - } else { - Err(Why::WindowOffEpochGrid) - } -} - -/// For each of `nodes`, the indices of the others below it. -fn candidates_below(nodes: &[Rc]) -> Vec> { - nodes - .iter() - .map(|node| { - let reachable = OperatorNode::reachable(node); - (0..nodes.len()) - .filter(|&v| { - !Rc::ptr_eq(&nodes[v], node) - && reachable.iter().any(|n| Rc::ptr_eq(n, &nodes[v])) - }) - .collect() - }) - .collect() -} - -/// Whether, below every ingestion-time build of `assignment`, every build -/// is at ingestion time too and every shared non-summary node is -/// `NotMaterialized`: ingestion work reads it as it arrives, and its -/// query-time readers read a copy ([`split_shared_by_timing`]). -/// Simplification: that copy is not offered `MatAtQuery` even when two -/// query-time readers remain. `below` is [`candidates_below`] of -/// `mat_candidates`. -fn down_closed( - mat_candidates: &[MaterializationCandidate], - below: &[Vec], - assignment: &[Materialization], -) -> bool { - let at_ingestion = |m: &Materialization| m.timing() == ExecutionTiming::IngestionTime; - assignment.iter().enumerate().all(|(c, m)| { - !at_ingestion(m) - || below[c].iter().all(|&v| { - if is_summary_build(&mat_candidates[v].node) { - at_ingestion(&assignment[v]) - } else { - assignment[v] == Materialization::NotMaterialized - } - }) - }) -} - -/// Every assignment of an option to each materialization candidate (mixed -/// radix, the last varying fastest, all `NotMaterialized` first) that is -/// [`down_closed`]. -#[cfg(any(test, feature = "reference-pricer"))] -fn materialization_assignments( - mat_candidates: &[MaterializationCandidate], -) -> impl Iterator> + '_ { - let options: Vec> = mat_candidates.iter().map(|c| c.options()).collect(); - let nodes: Vec<_> = mat_candidates.iter().map(|c| Rc::clone(&c.node)).collect(); - let below = candidates_below(&nodes); - let mut digits = Some(vec![0usize; mat_candidates.len()]); - std::iter::from_fn(move || loop { - let current = digits.as_mut()?; - let assignment: Vec = current - .iter() - .zip(&options) - .map(|(&d, options)| options[d]) - .collect(); - // Advance (mixed radix); `None` after the last assignment. - let mut carried = true; - for (digit, options) in current.iter_mut().zip(&options).rev() { - *digit += 1; - if *digit < options.len() { - carried = false; - break; - } - *digit = 0; - } - if carried { - digits = None; - } - if down_closed(mat_candidates, &below, &assignment) { - return Some(assignment); - } - digits.as_ref()?; - }) -} - -/// `roots` with every operator node they read replaced by `f`'s result. -fn map_roots( - roots: &[QueryRoot], - f: &mut impl FnMut(&Rc) -> Result, PhysicalASAPOptimizationError>, -) -> Result, PhysicalASAPOptimizationError> { - roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => Ok(QueryRoot::Operator(f(node)?)), - QueryRoot::Scalar(expr) => { - let mut mapped = HashMap::new(); - for node in expr.operator_refs() { - mapped.insert(Rc::as_ptr(node), f(node)?); - } - Ok(QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - mapped[&Rc::as_ptr(node)].clone() - }))) - } - }) - .collect() -} - -/// Lower the private operators of one candidate while preserving its already -/// canonical input-group nodes. This reuses the complete-plan implementation -/// rules (currently exact top-k -> Project/Limit/Sort), without assigning timing -/// or materialization and without recursively implementing another group's input. -/// Returned new operators must be enumerated and priced before plan selection. -pub(crate) fn implement_candidate_operators( - root: &Rc, - inputs: &[Rc], -) -> Result, PhysicalASAPOptimizationError> { - // The inventory rejects a root aliasing an input; an input outside the - // subtree is unused here and rejected by the inventory as `InvalidMemo`. - debug_assert!(!inputs.iter().any(|input| Rc::ptr_eq(input, root))); - let mut implementations = inputs - .iter() - .map(|input| (Rc::as_ptr(input), Rc::clone(input))) - .collect(); - implement(root, &mut implementations) -} - -/// Each node's implementation. -type NodeToImplementedNodeCache = HashMap<*const OperatorNode, Rc>; - -fn implement( - node: &Rc, - cache: &mut NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - implement(child, cache)?; - } - let implemented = match node.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) - if measures.iter().any(|m| matches!(m, AggIntent::TopK { .. })) => - { - implement_topk(node, cache)? - } - _ => rebuilt(node, cache)?, - }; - cache.insert(Rc::as_ptr(node), implemented.clone()); - Ok(implemented) -} - -/// Exact top-k as `Limit(Sort)` per group, which outputs the ranked rows of -/// the child, under a `Project` restoring the logical `[keys.., topk_k]` -/// output (the ranked value under `topk_k`): a root returns those columns, -/// and a node reading the top-k resolved its columns against them. -fn implement_topk( - node: &Rc, - cache: &NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - let unsupported = |why: &str| PhysicalASAPOptimizationError::ExactTopK(why.to_string()); - let Some(NonASAPOp::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - }) = node.non_asap() - else { - unreachable!("implement_topk is called on an Aggregate") - }; - let Reduction::Reduce(keys) = reduction else { - return Err(unsupported("per-entity reduction")); - }; - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Err(unsupported("measures besides the top-k")); - }; - if filters.iter().any(Option::is_some) { - return Err(unsupported("filtered measure")); - } - if having.is_some() { - return Err(unsupported("HAVING")); - } - let child = cache[&Rc::as_ptr(child)].clone(); - let value = asap_types::ir::schema::topk::ranked_column(&child) - .map_err(|e| unsupported(&format!("no ranked column: {e}")))?; - let sorted = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(value), - ascending: false, - nulls_first: false, - }], - partition_by: keys.clone(), - child: child.clone(), - }))? - .with_guarantee(child.guarantee.clone()), - ); - let limited = Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: keys.clone(), - child: sorted, - }))? - .with_guarantee(node.guarantee.clone()), - ); - let output_name = node.schema.fields[keys.len()].name.clone(); - let cols = asap_types::ir::schema::topk::ranked_row_projection( - &child, - &Reduction::Reduce(keys.clone()), - &output_name, - ) - .map_err(|error| unsupported(&format!("ranked rows: {error}")))?; - Ok(Rc::new( - OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: limited, - }))? - .with_guarantee(node.guarantee.clone()), - )) -} - -/// `node` over its implemented children; the same `Rc` when none changed. -fn rebuilt( - node: &Rc, - cache: &NodeToImplementedNodeCache, -) -> Result, PhysicalASAPOptimizationError> { - let input = |child: &Rc| cache[&Rc::as_ptr(child)].clone(); - let changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &input(child))); - Ok(if changed { - Rc::new( - node.with_new_children(|child| input(child))? - .with_guarantee(node.guarantee.clone()), - ) - } else { - node.clone() - }) -} - -/// Stage 2 internals that other crates' tests call: building one physical -/// candidate for a chosen assignment, the catalog steps, and the timing split. -#[cfg(feature = "test-support")] -pub mod test_support { - use asap_types::ir::QueryRoot; - use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - - use crate::{PhysicalASAPOptimizationError, PhysicalWorkloadPlan}; - - pub use crate::catalog::test_support as catalog; - pub use crate::split_by_timing::split_shared_by_timing; - - /// See `optimize_physical_asap_chosen` in this crate. - pub fn optimize_physical_asap_chosen( - from_logical: &str, - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - query_time_results_retained: bool, - choose: &mut crate::ChooseMaterialization<'_>, - ) -> Result, PhysicalASAPOptimizationError> { - crate::optimize_physical_asap_chosen( - from_logical, - roots, - entries, - data, - query_time_results_retained, - choose, - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_asap_optimization::test_support::lower_promql; - use asap_types::ir::physical_export::PhysicalASAPNodeId; - use asap_types::ir::physical_export::PhysicalASAPOperatorPayload as Payload; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::ASAPOp; - - // Candidate-local implementation must leave another group's canonical root untouched. - #[test] - fn candidate_operator_lowering_preserves_input_boundaries() { - let root = lower_promql( - "topk by (job) (3, sum_over_time(m[1m]))", - AccuracyTarget::exact(), - ); - let boundary = Rc::clone(root.children()[0]); - let lowered = - implement_candidate_operators(&root, std::slice::from_ref(&boundary)).unwrap(); - assert_eq!(lowered.schema, root.schema); - let nodes = OperatorNode::reachable(&lowered); - assert!(nodes.iter().any(|node| node.operator.kind_name() == "Sort")); - assert!(nodes - .iter() - .any(|node| node.operator.kind_name() == "Limit")); - assert!(nodes.iter().any(|node| Rc::ptr_eq(node, &boundary))); - } - - // Logical and exact ranked rows retain the full series identity contract. - #[test] - fn topk_logical_contract_preserves_series_identity() { - let template = lower_promql("topk(3, sum_over_time(m[1m]))", AccuracyTarget::exact()); - let source = asap_logical_asap_optimization::test_support::metric_scan(&[ - "job", - "instance", - asap_types::ir::schema::PROMQL_SERIES_IDENTITY, - ]); - let root = Rc::new(template.with_new_children(|_| Rc::clone(&source)).unwrap()); - let boundary = Rc::clone(root.children()[0]); - let lowered = implement_candidate_operators(&root, &[boundary]).unwrap(); - assert_eq!(lowered.schema, root.schema); - assert_eq!( - names(&root), - [ - "topk_3", - "job", - "instance", - asap_types::ir::schema::PROMQL_SERIES_IDENTITY - ] - ); - assert!(root - .schema - .fields - .iter() - .any(|field| field.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY)); - assert_eq!( - root.schema.fields[0].dtype, - asap_types::ir::schema::FieldDataType::Plain(asap_types::ir::schema::DataType::Float64) - ); - } - - /// The all-`NotMaterialized` candidate, with no workload declared. - fn first_candidate(roots: &[QueryRoot]) -> PhysicalWorkloadPlan { - optimize_physical_asap("L1", roots, &[], &DataWorkload::default(), false) - .unwrap() - .remove(0) - } - - fn operator_root(candidate: &PhysicalWorkloadPlan, i: usize) -> Rc { - match &candidate.roots[i] { - QueryRoot::Operator(node) => node.clone(), - QueryRoot::Scalar(_) => panic!("operator root"), - } - } - - fn names(node: &OperatorNode) -> Vec { - node.schema.fields.iter().map(|f| f.name.clone()).collect() - } - - /// The `Limit` under an exact top-k's restoring `Project`. - fn topk_limit(project: &OperatorNode) -> Rc { - let NonASAPOp::Project { child, .. } = project.expect_non_asap() else { - panic!("project, found {}", project.operator.kind_name()) - }; - child.clone() - } - - /// The single sort key of the `Sort` under the `Limit` at `limit`. - fn topk_sort_key(limit: &OperatorNode) -> (ScalarExpr, bool) { - let NonASAPOp::Limit { child: sort, .. } = limit.expect_non_asap() else { - panic!("limit, found {}", limit.operator.kind_name()) - }; - let NonASAPOp::Sort { keys, .. } = sort.expect_non_asap() else { - panic!("sort below limit") - }; - let [key] = keys.as_slice() else { - panic!("one sort key") - }; - (key.expr.clone(), key.ascending) - } - - /// Example 1's shape, `topk by (job) (k, per-series rows)`, sorts each - /// group by the sample value, descending. - #[test] - fn promql_topk_over_series_sorts_by_value_descending() { - let root = lower_promql( - "topk by (job) (10, sum_over_time(m[1m]))", - AccuracyTarget::exact(), - ); - let candidate = first_candidate(&[root.clone().into()]); - let physical = operator_root(&candidate, 0); - assert_eq!(names(&physical), names(&root)); - let limit = topk_limit(&physical); - assert_eq!(limit.schema.fields[1].name, "value"); - assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); - } - - /// `topk(k, sum by (job) (..))` sorts by the summed measure. - #[test] - fn promql_topk_over_grouped_aggregate_sorts_by_its_measure() { - let root = lower_promql("topk(5, sum by (job) (m))", AccuracyTarget::exact()); - let candidate = first_candidate(&[root.into()]); - let physical = operator_root(&candidate, 0); - // The answers keep their `job` label. - assert_eq!(names(&physical), ["topk_5", "job"]); - let limit = topk_limit(&physical); - assert_eq!(names(&limit), ["job", "sum"]); - assert_eq!(topk_sort_key(&limit), (ScalarExpr::Column(1), false)); - } - - /// A node reading an exact top-k keeps resolving the columns it bound - /// against the logical `[keys.., topk_k]` output: `sum by (job)` still - /// groups by `job`, not by whatever column now sits at its index. - #[test] - fn consumer_of_exact_topk_reads_its_logical_columns() { - let root = lower_promql( - "sum by (job) (topk by (job) (10, sum_over_time(m[1m])))", - AccuracyTarget::exact(), - ); - let candidate = first_candidate(&[root.clone().into()]); - let physical = operator_root(&candidate, 0); - assert_eq!(names(&physical), names(&root)); - let NonASAPOp::Aggregate { child: input, .. } = physical.expect_non_asap() else { - panic!("sum root") - }; - let NonASAPOp::Project { - cols, child: limit, .. - } = input.expect_non_asap() - else { - panic!("project over the top-k rows") - }; - let read: Vec<_> = cols - .iter() - .map(|item| { - let ScalarExpr::Column(c) = item.expr else { - panic!("column") - }; - ( - item.alias.clone().unwrap(), - limit.schema.fields[c].name.clone(), - ) - }) - .collect(); - assert_eq!( - read, - [ - ("job".to_string(), "job".to_string()), - ("topk_10".to_string(), "value".to_string()) - ] - ); - assert_eq!(topk_sort_key(limit), (ScalarExpr::Column(1), false)); - } - - /// An `Aggregate{[TopK]}` that Sort + Limit cannot express (another - /// measure, `HAVING`) fails, so its candidate is pruned with the reason. - #[test] - fn topk_with_extra_measure_or_having_fails() { - use asap_logical_asap_optimization::test_support::{aggregate, metric_scan}; - use asap_types::ir::scalar::expr_ir::ScalarValue; - use asap_types::ir::Predicate; - let topk = AggIntent::TopK { - k: 3, - accuracy: AccuracyTarget::exact(), - }; - let count = AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }; - let by_job = || Reduction::by(vec![2]); - let cases = [ - aggregate( - by_job(), - vec![topk.clone(), count], - vec![], - None, - metric_scan(&["job"]), - ), - aggregate( - by_job(), - vec![topk], - vec![], - Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - metric_scan(&["job"]), - ), - ]; - for root in cases { - let error = - optimize_physical_asap("L1", &[root.into()], &[], &DataWorkload::default(), false) - .unwrap_err(); - assert!( - matches!(error, PhysicalASAPOptimizationError::ExactTopK(_)), - "{error}" - ); - } - } - - /// Exact `topk by (job)` becomes Limit(Sort) partitioned by `job`, and a - /// child shared by two roots stays one node (one `Rc`, one DAG node). - #[test] - fn exact_topk_becomes_sort_then_limit_and_keeps_shared_child() { - let exact = AccuracyTarget::exact(); - let topk = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", exact.clone()); - let sum = lower_promql("sum by (job) (sum_over_time(m[1m]))", exact); - // Share Q2's per-series child with Q1, as Stage 1 sharing would. - let NonASAPOp::Aggregate { child: shared, .. } = topk.expect_non_asap() else { - panic!("topk aggregate") - }; - let sum = Rc::new(sum.with_new_children(|_| shared.clone()).unwrap()); - let candidate = first_candidate(&[topk.into(), sum.into()]); - let root = |i: usize| match &candidate.roots[i] { - QueryRoot::Operator(node) => node.clone(), - QueryRoot::Scalar(_) => panic!("operator root"), - }; - - let NonASAPOp::Limit { - n: Some(10), - partition_by, - child: sort, - .. - } = topk_limit(&root(0)).expect_non_asap().clone() - else { - panic!("limit under the root") - }; - assert!(!partition_by.keys().is_empty()); - let NonASAPOp::Sort { - child: below_sort, - partition_by: sort_partition, - .. - } = sort.expect_non_asap() - else { - panic!("sort below limit") - }; - assert_eq!(*sort_partition, partition_by); - let NonASAPOp::Aggregate { - child: below_sum, .. - } = root(1).expect_non_asap().clone() - else { - panic!("sum root") - }; - assert!(Rc::ptr_eq(below_sort, &below_sum)); - - let dag = &candidate.dag; - let kind = |id: PhysicalASAPNodeId| match &dag - .nodes - .iter() - .find(|n| n.id == id) - .unwrap() - .payload - { - Payload::NonASAP(NonASAPOp::Sort { .. }) => "sort", - Payload::NonASAP(NonASAPOp::Limit { .. }) => "limit", - _ => "other", - }; - let producer_of = |consumer| { - dag.edges - .iter() - .find(|e| e.consumer == consumer) - .unwrap() - .producer - }; - let limit_id = producer_of(dag.roots[0]); - assert_eq!(kind(limit_id), "limit"); - let sort_id = producer_of(limit_id); - assert_eq!(kind(sort_id), "sort"); - assert_eq!(producer_of(sort_id), producer_of(dag.roots[1])); - } - - /// With summaries chosen, every node, the summary build included, runs at - /// query time. - #[test] - fn everything_runs_at_query_time() { - let root = lower_promql( - "topk by (job) (10, sum_over_time(m[1m]))", - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // A summary (the last candidate) for every target. - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| t.choice_count() - 1) - .collect(); - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap() - .into_iter() - .collect(); - let candidate = first_candidate(&roots); - assert!(candidate - .dag - .nodes - .iter() - .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })))); - assert!(candidate - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - } - - // ── Materialization ────────────────────────────────────────────────── - - use asap_types::workload::{DataArrival, RepetitionInterval, TimestampMs}; - - const MINUTE: u64 = 60_000; - - /// The composed logical candidate of `query` that builds its last - /// summary option (DDSketch for a quantile) for every target. - fn summary_root(query: &str) -> QueryRoot { - let root = lower_promql( - query, - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(asap_types::ir::properties::AccuracyMetric::Rank), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| t.choice_count() - 1) - .collect(); - asap_logical_asap_optimization::test_support::compose_logical_candidate(&inventory, &choice) - .unwrap() - .remove(0) - } - - /// `root` with its summary build in 1-min tumbling windows. - fn in_tumbling_windows(root: QueryRoot) -> QueryRoot { - let roots = [root]; - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &[Some(MINUTE)], - ); - assert_eq!(rule_candidates.len(), 1, "{rule_candidates:?}"); - let mut pass2_candidates = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap(); - pass2_candidates.pop().unwrap().roots.unwrap().remove(0) - } - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - /// The options `candidate` chose other than `NotMaterialized`, by node. - fn materialized(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .materialization - .values() - .copied() - .filter(|m| *m != Materialization::NotMaterialized) - .collect() - } - - // An unselected unpredictable query must not restrict an ingestion candidate. - #[test] - fn selected_reader_eligibility_ignores_optional_unpredictable_consumers() { - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let node = root - .operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .unwrap(); - let entries = vec![ - every_minute(), - QueryWorkloadEntry { - predictability: Predictability::AdHoc, - ..every_minute() - }, - ]; - let data = arriving(); - let query_time_results_retained = false; - let selected = summary_materialization_eligibility( - &node, - &[0, 0], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert_eq!(selected.at_ingestion, Ok(())); - assert!(matches!( - summary_materialization_eligibility(&node, &[0, entries.len()], &entries, &data, query_time_results_retained), - Err(SummaryEligibilityError::UnknownReader { query }) if query == entries.len() - )); - let combined = summary_materialization_eligibility( - &node, - &[0, 1], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert_eq!( - combined.at_ingestion, - Err(IngestionPruneReason::NotPredictable) - ); - // The complete-DAG path uses the same selected-reader workload rules. - let candidates = - materialization_candidates(&[root], &entries, &data, query_time_results_retained); - let original = candidates - .iter() - .find(|candidate| { - matches!( - candidate.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .unwrap(); - assert_eq!(original.at_ingestion, selected.at_ingestion); - assert_eq!( - original.may_keep_across_evaluations, - selected.may_keep_across_evaluations - ); - } - - // Phase and range-step restrictions apply only to the selected reader set. - #[test] - fn selected_reader_eligibility_checks_phase_and_range_compatibility() { - use asap_types::workload::RangeEvaluation; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let node = root - .operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .unwrap(); - let at = |phase| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase), - }), - ..every_minute() - }; - let entries = vec![ - at(0), - at(MINUTE / 2), - QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: MINUTE / 2, - span_ms: 10 * MINUTE, - }), - ..at(0) - }, - ]; - let query_time_results_retained = true; - let data = arriving(); - let one = summary_materialization_eligibility( - &node, - &[0], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(one.may_keep_across_evaluations); - assert!(one.range_compatibility.is_ok()); - let different_phase = summary_materialization_eligibility( - &node, - &[0, 1], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(!different_phase.may_keep_across_evaluations); - assert_eq!( - different_phase.at_ingestion, - Err(IngestionPruneReason::TumblingWindowsOffPhase) - ); - let off_grid = summary_materialization_eligibility( - &node, - &[0, 2], - &entries, - &data, - query_time_results_retained, - ) - .unwrap(); - assert!(matches!( - off_grid.range_compatibility, - Err(PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { .. }) - )); - assert!(matches!( - summary_materialization_eligibility( - &node, - &[], - &entries, - &data, - query_time_results_retained - ), - Err(SummaryEligibilityError::NoSelectedReaders) - )); - } - - /// A predictable query evaluated every minute. - fn every_minute() -> QueryWorkloadEntry { - QueryWorkloadEntry { - predictability: Predictability::Predictable { known_at: None }, - ..asap_logical_asap_optimization::test_support::entry(QueryRecurrence::Repeated( - RepeatedDemand::FixedInterval(RepetitionInterval(MINUTE as u32)), - )) - } - } - - fn arriving() -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - ..Default::default() - } - } - - /// The distinct nodes of `kind` the candidate's roots read. - fn node_of(candidate: &PhysicalWorkloadPlan, kind: &str) -> Vec> { - let mut nodes: Vec> = Vec::new(); - for root in candidate.roots.iter().flat_map(QueryRoot::operator_refs) { - for node in OperatorNode::reachable(root) { - if node.operator.kind_name() == kind && !nodes.iter().any(|n| Rc::ptr_eq(n, &node)) - { - nodes.push(node); - } - } - } - nodes - } - - /// #509 Example 4, B1 and B2: a tumbling-window chain read by a repeating, - /// predictable query over arriving data is either rebuilt at query time - /// or maintained at ingestion time. Maintained, the chain and the scan - /// below it run at ingestion time, and the merge and estimate at query - /// time. - #[test] - fn a_tumbling_window_chain_over_arriving_data_may_be_maintained_at_ingestion_time() { - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let candidates = - optimize_physical_asap("L1", &[root], &[every_minute()], &arriving(), false).unwrap(); - assert_eq!(candidates.len(), 2); - let [b2, b1] = &candidates[..] else { - unreachable!() - }; - assert!(!b2.materializes_any()); - assert!(b2 - .dag - .nodes - .iter() - .all(|n| n.output_state.timing == ExecutionTiming::QueryTime)); - assert_eq!(materialized(b1), [AT_INGESTION]); - let [chain] = &node_of(b1, "SummaryTumblingWindowChain")[..] else { - panic!("one chain") - }; - assert_eq!( - chain.materialization, - Materialization::MatAtIngestion(Medium::Memory) - ); - for below in OperatorNode::reachable(chain) { - assert_eq!(below.timing, Some(ExecutionTiming::IngestionTime)); - } - for kind in ["SummaryMerge", "SummaryEstimate"] { - for node in node_of(b1, kind) { - assert_eq!(node.timing, Some(ExecutionTiming::QueryTime), "{kind}"); - } - } - } - - /// A chain of 1-min tumbling windows, anchored at the epoch, read by two queries - /// every minute is maintained at ingestion time when they evaluate on - /// its tumbling-window boundaries (phases 0 and 2 min, or one phase unknown), not - /// when one is at phase 30 s: each tumbling window is built once, so it would read - /// tumbling windows 30 s off. - #[test] - fn a_tumbling_window_chain_is_maintained_only_on_its_readers_phases() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let at = |phase_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase_ms), - }), - ..every_minute() - }; - let at_ingestion = |demands: [QueryWorkloadEntry; 2]| { - let mut cache = HashMap::new(); - // One chain read by both queries. - let implemented = map_roots(&[root.clone(), root.clone()], &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &demands, &arriving(), false) - .into_iter() - .find(|c| { - matches!( - c.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .expect("the chain is a materialization candidate") - .at_ingestion - }; - assert_eq!(at_ingestion([at(0), at(2 * MINUTE)]), Ok(())); - assert_eq!(at_ingestion([at(0), every_minute()]), Ok(())); - assert_eq!( - at_ingestion([at(0), at(30_000)]), - Err(Why::TumblingWindowsOffPhase) - ); - assert_eq!( - at_ingestion([at(30_000), every_minute()]), - Err(Why::TumblingWindowsOffPhase) - ); - } - - /// A chain of 1-min tumbling windows, anchored at the epoch, read every - /// 90 s ends a window at only every other evaluation (#1173), with or - /// without a declared phase on the grid: it is maintained at ingestion - /// time only when every reader's interval is a multiple of its width. - #[test] - fn a_tumbling_window_chain_is_maintained_only_on_its_readers_intervals() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let every = |interval_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(interval_ms as u32), - )), - ..every_minute() - }; - let every_at_phase_0 = |interval_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval_ms as u32), - evaluation_phase: TimestampMs(0), - }), - ..every_minute() - }; - let at_ingestion = |demands: [QueryWorkloadEntry; 2]| { - let mut cache = HashMap::new(); - // One chain read by both queries. - let implemented = map_roots(&[root.clone(), root.clone()], &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &demands, &arriving(), false) - .into_iter() - .find(|c| { - matches!( - c.node.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .expect("the chain is a materialization candidate") - .at_ingestion - }; - assert_eq!(at_ingestion([every_minute(), every(2 * MINUTE)]), Ok(())); - assert_eq!( - at_ingestion([every_minute(), every(90_000)]), - Err(Why::TumblingWindowsOffInterval) - ); - assert_eq!( - at_ingestion([every_minute(), every_at_phase_0(90_000)]), - Err(Why::TumblingWindowsOffInterval) - ); - } - - /// S4: maintaining state at ingestion time needs arriving data and - /// queries that repeat and are predictable; otherwise the chain only - /// runs at query time, and the materialization candidate says why. - #[test] - fn ingestion_time_needs_arriving_data_and_repeated_predictable_queries() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let at_rest = DataWorkload { - arrival: DataArrival::AtRest, - ..Default::default() - }; - let unknown = QueryWorkloadEntry { - predictability: Predictability::Unknown, - ..every_minute() - }; - let once = QueryWorkloadEntry { - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - ..every_minute() - }; - for (demand, data, why) in [ - (every_minute(), at_rest, Why::NotIngesting), - (every_minute(), DataWorkload::default(), Why::NotIngesting), - (unknown, arriving(), Why::NotPredictable), - (once, arriving(), Why::NotRepeated), - ] { - let roots = [root.clone()]; - let candidates = - optimize_physical_asap("L1", &roots, std::slice::from_ref(&demand), &data, false) - .unwrap(); - assert_eq!(candidates.len(), 1, "{why:?}"); - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = materialization_candidates(&implemented, &[demand], &data, false); - assert_eq!(mat_candidates.len(), 1); - assert_eq!(mat_candidates[0].at_ingestion, Err(why)); - } - } - - /// A counter state (`rate`, `increase`) needs each series' samples in - /// time order. Continuously ingesting time-series (PromQL) data is taken - /// to arrive in that order, as Prometheus assumes, so its counter tumbling windows - /// may be maintained at ingestion time. - #[test] - fn counter_tumbling_windows_over_ingesting_promql_data_run_at_ingestion_time() { - for query in ["rate(m[5m])", "increase(m[5m])"] { - let roots = [in_tumbling_windows(summary_root(query))]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &arriving(), false); - assert_eq!(mat_candidates[0].at_ingestion, Ok(()), "{query}"); - let candidates = - optimize_physical_asap("L1", &roots, &[every_minute()], &arriving(), false) - .unwrap(); - assert_eq!(candidates.len(), 2, "{query}"); - } - } - - /// Only continuously ingesting time-series data is taken to arrive in - /// time order: a counter over a table source, or over mixed data (whose - /// at-rest part may be loaded in any order), stays at query time. - #[test] - fn counter_tumbling_windows_over_other_sources_stay_at_query_time() { - use IngestionPruneReason as Why; - let root = in_tumbling_windows(summary_root("rate(m[5m])")); - let QueryRoot::Operator(node) = &root else { - panic!("{root:?}") - }; - let table = QueryRoot::Operator(to_table(node)); - let mixed = DataWorkload { - arrival: DataArrival::Mixed { - rows_per_second: 1_000.0, - }, - ..Default::default() - }; - for (root, data) in [(table, arriving()), (root.clone(), mixed)] { - let roots = [root]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - let mat_candidates = - materialization_candidates(&implemented, &[every_minute()], &data, false); - assert_eq!( - mat_candidates[0].at_ingestion, - Err(Why::CounterNeedsTimeOrder) - ); - } - } - - /// `node` with every time-series scan turned into a table scan of the - /// same name and schema. - fn to_table(node: &Rc) -> Rc { - let mut out = (**node).clone(); - out.operator = node.operator.map_children(to_table); - if let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &mut out.operator { - if let Source::TimeSeries { metric } = source { - *source = Source::Table { - table_ref: metric.clone(), - }; - } - } - Rc::new(out) - } - - /// A whole-window build is maintained only for one window per - /// evaluation on the epoch grid of its width, where the backend builds - /// it: a fixed interval that is a multiple of the window, a phase (if - /// known) on that grid, and no filter. `m[1m]` every minute with no - /// declared phase is one tumbling 1-minute window per evaluation, taken - /// to run on the grid like a chain's reader of unknown phase; at phase - /// 30 s its windows would end off the grid, and every 30 s or every - /// 90 s some evaluations would. - #[test] - fn a_whole_window_build_needs_evaluations_on_the_epoch_grid_of_its_window() { - use IngestionPruneReason as Why; - let root = summary_root("quantile_over_time(0.99, m[1m])"); - let at_phase = |interval_ms: u32, phase_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(interval_ms), - evaluation_phase: TimestampMs(phase_ms), - }), - ..every_minute() - }; - let at = |interval_ms: u32| at_phase(interval_ms, 0); - let mat_candidates = |root: &QueryRoot, demand: QueryWorkloadEntry| { - let mut cache = HashMap::new(); - let implemented = map_roots(std::slice::from_ref(root), &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &[demand], &arriving(), false) - .remove(0) - .at_ingestion - }; - assert_eq!(mat_candidates(&root, at(MINUTE as u32)), Ok(())); - assert_eq!(mat_candidates(&root, every_minute()), Ok(())); - assert_eq!(mat_candidates(&root, at(2 * MINUTE as u32)), Ok(())); - assert_eq!( - mat_candidates(&root, at_phase(MINUTE as u32, 3 * MINUTE)), - Ok(()) - ); - assert_eq!( - mat_candidates(&root, at_phase(MINUTE as u32, 30_000)), - Err(Why::WindowOffEpochGrid) - ); - assert_eq!( - mat_candidates(&root, at(30_000)), - Err(Why::WindowOffEpochGrid) - ); - assert_eq!( - mat_candidates(&root, at(90_000)), - Err(Why::WindowOffEpochGrid) - ); - // Several readers of one build: a declared phase on the grid and an - // undeclared one agree; two declared phases, or two intervals, do not. - let mat_candidates_of = |demands: &[QueryWorkloadEntry]| { - let roots = vec![root.clone(); demands.len()]; - let mut cache = HashMap::new(); - let implemented = map_roots(&roots, &mut |n| implement(n, &mut cache)).unwrap(); - materialization_candidates(&implemented, demands, &arriving(), false) - .into_iter() - .find(|c| matches!(c.node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("the whole-window build is a materialization candidate") - .at_ingestion - }; - assert_eq!( - mat_candidates_of(&[at_phase(MINUTE as u32, MINUTE), every_minute()]), - Ok(()) - ); - assert_eq!( - mat_candidates_of(&[ - at(MINUTE as u32), - at_phase(MINUTE as u32, 2 * MINUTE + MINUTE / 2) - ]), - Err(Why::WindowNotFixed) - ); - assert_eq!( - mat_candidates_of(&[at(MINUTE as u32), at(2 * MINUTE as u32)]), - Err(Why::WindowNotFixed) - ); - let candidates = optimize_physical_asap( - "L1", - std::slice::from_ref(&root), - &[at(MINUTE as u32)], - &arriving(), - false, - ) - .unwrap(); - assert_eq!(materialized(&candidates[1]), [AT_INGESTION]); - - // The same build with a filter. - let QueryRoot::Operator(node) = &root else { - unreachable!() - }; - let build = OperatorNode::reachable(node) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - .. - }) = build.operator.clone() - else { - unreachable!() - }; - let filtered = Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - filter: Some(asap_types::ir::Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::expr_ir::ScalarValue::Boolean(true), - ))), - }), - build.schema.clone(), - ) - .with_guarantee(build.guarantee.clone()), - ); - assert_eq!( - mat_candidates(&QueryRoot::Operator(filtered), at(MINUTE as u32)), - Err(Why::Filtered) - ); - } - - /// A whole-window build read by a range evaluation of several steps per - /// request is not maintained at ingestion time: it would hold one - /// window per request, not one per step. A one-step range (`span_ms` - /// 0) is one evaluation per request and stays eligible. - #[test] - fn a_range_evaluated_whole_window_build_stays_at_query_time() { - use asap_types::workload::RangeEvaluation; - let root = summary_root("quantile_over_time(0.99, m[1m])"); - let at_minute = |span_ms: u64| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(0), - }), - range_evaluation: Some(RangeEvaluation { - step_ms: MINUTE, - span_ms, - }), - ..every_minute() - }; - let at_ingestion = |entry: QueryWorkloadEntry| { - let mut cache = HashMap::new(); - let implemented = map_roots(std::slice::from_ref(&root), &mut |n| { - implement(n, &mut cache) - }) - .unwrap(); - materialization_candidates(&implemented, &[entry], &arriving(), false) - .remove(0) - .at_ingestion - }; - assert_eq!( - at_ingestion(at_minute(60 * MINUTE)), - Err(IngestionPruneReason::RangeEvaluatedWholeWindow) - ); - assert_eq!(at_ingestion(at_minute(0)), Ok(())); - } - - /// A tumbling-window chain whose width does not divide a range-evaluated reader's - /// step is not offered: the candidate fails with the widths, and the - /// caller prunes it. 1-min tumbling windows serve 2-min steps. - #[test] - fn a_tumbling_window_chain_off_the_range_step_grid_is_rejected() { - use asap_types::workload::RangeEvaluation; - let root = in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")); - let stepping = |step_ms: u64| QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms, - span_ms: 60 * MINUTE, - }), - ..every_minute() - }; - let implement_for = |entry: QueryWorkloadEntry| { - optimize_physical_asap( - "L1", - std::slice::from_ref(&root), - &[entry], - &arriving(), - false, - ) - }; - let error = implement_for(stepping(30_000)).unwrap_err(); - assert!( - matches!( - error, - PhysicalASAPOptimizationError::TumblingWindowWidthOffRangeStep { - tumbling_window_width_ms: MINUTE, - step_ms: 30_000, - } - ), - "{error}" - ); - assert_eq!( - error.to_string(), - "tumbling-window width 60000 ms does not divide the 30000 ms step of a range \ - evaluation that reads it" - ); - assert_eq!(implement_for(stepping(2 * MINUTE)).unwrap().len(), 2); - // A one-step range (span 0) is one evaluation per request, at the - // refresh: its step is not read. - let one_step = QueryWorkloadEntry { - range_evaluation: Some(RangeEvaluation { - step_ms: 30_000, - span_ms: 0, - }), - ..every_minute() - }; - assert_eq!(implement_for(one_step).unwrap().len(), 2); - } - - /// A build two queries read is also offered built once per evaluation - /// for both (`MatAtQuery`); a build one query reads is not. - #[test] - fn a_build_two_queries_read_may_be_built_once_per_evaluation() { - let QueryRoot::Operator(p99) = summary_root("quantile_over_time(0.99, m[1m])") else { - unreachable!() - }; - let build = OperatorNode::reachable(&p99) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - let p50 = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&build), - query: asap_types::ir::schema::SketchStatistic::Quantile { q: 0.5 }, - })) - .unwrap(), - ); - let roots = [QueryRoot::Operator(p99), QueryRoot::Operator(p50)]; - let candidates = - optimize_physical_asap("L1", &roots, &[], &DataWorkload::default(), false).unwrap(); - assert_eq!(candidates.len(), 2); - assert_eq!( - materialized(&candidates[1]), - [Materialization::MatAtQuery(Medium::Memory)] - ); - let [build] = &node_of(&candidates[1], "SummaryAgg")[..] else { - panic!("one shared build") - }; - assert_eq!( - build.materialization, - Materialization::MatAtQuery(Medium::Memory) - ); - } - - /// Two queries share one scan: one reads it through tumbling windows maintained at - /// ingestion time, the other at query time. The scan is split, one copy - /// per timing; with nothing materialized it stays one node. The shared - /// scan may also be built once per evaluation (`MatAtQuery`), but not - /// under ingestion-time tumbling windows, where only one query reads it at query - /// time. - #[test] - fn a_scan_two_queries_read_at_two_timings_is_split() { - let QueryRoot::Operator(tumbling_windows) = - in_tumbling_windows(summary_root("quantile_over_time(0.99, m[5m])")) - else { - unreachable!() - }; - let scan = OperatorNode::reachable(&tumbling_windows) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - let ranged = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(5 * MINUTE), - kind: asap_types::ir::TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap(); - let exact = asap_logical_asap_optimization::test_support::aggregate( - Reduction::by(vec![]), - vec![AggIntent::Max { col: None }], - vec![], - None, - ranged, - ); - let roots = [ - QueryRoot::Operator(tumbling_windows), - QueryRoot::Operator(exact), - ]; - let candidates = optimize_physical_asap( - "L1", - &roots, - &[every_minute(), every_minute()], - &arriving(), - false, - ) - .unwrap(); - assert_eq!(candidates.len(), 3); - let scans = |c: &PhysicalWorkloadPlan| { - let mut scans: Vec<_> = c - .dag - .nodes - .iter() - .filter(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Scan { .. }))) - .map(|n| (n.output_state.timing, n.materialization)) - .collect(); - scans.sort_by_key(|(t, _)| *t == ExecutionTiming::QueryTime); - scans - }; - let not_materialized = Materialization::NotMaterialized; - assert_eq!( - scans(&candidates[0]), - [(ExecutionTiming::QueryTime, not_materialized)] - ); - assert_eq!( - scans(&candidates[1]), - [( - ExecutionTiming::QueryTime, - Materialization::MatAtQuery(Medium::Memory) - )] - ); - assert_eq!( - scans(&candidates[2]), - [ - (ExecutionTiming::IngestionTime, not_materialized), - (ExecutionTiming::QueryTime, not_materialized) - ] - ); - } - - /// Only the highest shared non-summary nodes are materialization - /// candidates, each with `NotMaterialized` and `MatAtQuery` only: a - /// 5 m range two queries read (not the scan below it, which only that - /// shared range reads for them), and the scan, which a third query - /// also reads through its own 1 m range. - #[test] - fn only_the_highest_shared_non_summary_nodes_are_candidates() { - let QueryRoot::Operator(p99) = summary_root("quantile_over_time(0.99, m[5m])") else { - unreachable!() - }; - let scan = OperatorNode::reachable(&p99) - .into_iter() - .find(|n| matches!(n.non_asap(), Some(NonASAPOp::Scan { .. }))) - .unwrap(); - let ranged = |ms: u64| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(ms), - kind: asap_types::ir::TimeRangeKind::Range, - child: Rc::clone(&scan), - })) - .unwrap() - }; - let aggregate = |intent: AggIntent, input: Rc| { - QueryRoot::Operator(asap_logical_asap_optimization::test_support::aggregate( - Reduction::by(vec![]), - vec![intent], - vec![], - None, - input, - )) - }; - let five_minutes = ranged(5 * MINUTE); - let roots = [ - aggregate(AggIntent::Max { col: None }, Rc::clone(&five_minutes)), - aggregate(AggIntent::Min { col: None }, five_minutes), - aggregate(AggIntent::Max { col: None }, ranged(MINUTE)), - ]; - let mat_candidates = - materialization_candidates(&roots, &[], &DataWorkload::default(), false); - let found: Vec<_> = mat_candidates - .iter() - .map(|c| (c.node.operator.kind_name(), c.readers.clone(), c.options())) - .collect(); - let options = vec![ - Materialization::NotMaterialized, - Materialization::MatAtQuery(Medium::Memory), - ]; - assert_eq!( - found, - [ - ("TimeRange", vec![0, 1], options.clone()), - ("Scan", vec![0, 1, 2], options) - ] - ); - } - - /// Readers share one kept tumbling-window chain only when they evaluate at the - /// same instants: two readers every minute at phases 0 and 2 min do, - /// at phases 0 and 30 s they do not (each would need its own newest - /// tumbling window). A reader with no known phase is assumed to run with the - /// others. - #[test] - fn readers_at_different_phases_do_not_build_one_tumbling_window_per_evaluation() { - let at = |phase_ms: u64| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(MINUTE as u32), - evaluation_phase: TimestampMs(phase_ms), - }) - }; - let unknown_phase = every_minute().recurrence; - assert!(builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &at(2 * MINUTE)] - )); - assert!(!builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &at(30_000)] - )); - assert!(builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(30_000), &unknown_phase] - )); - assert!(!builds_one_tumbling_window_per_evaluation( - MINUTE, - [&at(0), &unknown_phase, &at(30_000)] - )); - } - - // A known evaluation phase is reduced modulo the width; a bare interval has - // none; a schedule has one only when all its times share it. - #[test] - fn tumbling_window_phase_of_a_repeated_demand() { - use asap_types::workload::{RepetitionInterval, TimestampMs}; - let at = |phase| RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase), - }; - let scheduled = |times: &[u64]| { - RepeatedDemand::Scheduled(times.iter().copied().map(TimestampMs).collect()) - }; - let phase = |demand| plan_tumbling_window_phase(&demand, 60_000); - assert_eq!(phase(at(56_000)), Some(56_000)); - assert_eq!(phase(at(176_000)), Some(56_000)); - assert_eq!( - phase(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))), - None - ); - assert_eq!(phase(scheduled(&[56_000, 176_000])), Some(56_000)); - assert_eq!(phase(scheduled(&[56_000, 150_000])), None); - } -} diff --git a/crates/physical-asap-optimization/src/split_by_timing.rs b/crates/physical-asap-optimization/src/split_by_timing.rs deleted file mode 100644 index 72dad6d63..000000000 --- a/crates/physical-asap-optimization/src/split_by_timing.rs +++ /dev/null @@ -1,375 +0,0 @@ -//! Splitting a shared sub-DAG by execution timing before Stage 2 applies a -//! materialization assignment ([`split_shared_by_timing`]). - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{timing_under_consumer, MaterializationAssignment, OperatorNode}; - -/// Each (node, timing) pair's rebuilt node, with the original it copies. -type Copies = HashMap<(*const OperatorNode, ExecutionTiming), (Rc, Rc)>; - -/// Copy, for one consumer, every sub-DAG that `assignment` would reach with -/// two different timings from `roots` (one query each, or the roots of one -/// query), so that a workload whose CSE shared a `Scan` between an -/// ingestion-time summary and a query-time computation, in one query or -/// across queries, can still be timed. Only the conflicting sub-DAGs are -/// copied, one copy per timing; a sub-DAG reached with one timing stays one -/// `Rc`, shared by the roots that read it. Returns the (possibly rewritten) -/// roots, in order, and `assignment` remapped onto them: a rebuilt node -/// keeps its original's explicit materialization. -pub fn split_shared_by_timing( - roots: &[Rc], - assignment: &MaterializationAssignment, -) -> (Vec>, MaterializationAssignment) { - // First pass: the set of timings each node is reached with. - let mut reached: HashMap<*const OperatorNode, Vec> = HashMap::new(); - let mut forced = HashMap::new(); - fn collect( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - reached: &mut HashMap<*const OperatorNode, Vec>, - forced: &mut HashMap<*const OperatorNode, bool>, - ) { - let timing = timing_under_consumer(node, consumer, assignment, forced); - let entry = reached.entry(Rc::as_ptr(node)).or_default(); - if entry.contains(&timing) { - return; - } - entry.push(timing); - for child in node.children() { - collect(child, timing, assignment, reached, forced); - } - } - for root in roots { - collect( - root, - ExecutionTiming::QueryTime, - assignment, - &mut reached, - &mut forced, - ); - } - if reached.values().all(|timings| timings.len() <= 1) { - return (roots.to_vec(), assignment.clone()); - } - // Second pass: rebuild, giving each (node, timing) pair its own copy, - // kept with its original. - let mut copies: Copies = HashMap::new(); - fn rebuild( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - reached: &HashMap<*const OperatorNode, Vec>, - copies: &mut Copies, - forced: &mut HashMap<*const OperatorNode, bool>, - ) -> Rc { - let timing = timing_under_consumer(node, consumer, assignment, forced); - let key = (Rc::as_ptr(node), timing); - if let Some((_, done)) = copies.get(&key) { - return Rc::clone(done); - } - let conflicted = reached - .get(&Rc::as_ptr(node)) - .is_some_and(|timings| timings.len() > 1); - let mut changed = conflicted; - let operator = node.operator.map_children(|child| { - let rebuilt = rebuild(child, timing, assignment, reached, copies, forced); - changed |= !Rc::ptr_eq(&rebuilt, child); - rebuilt - }); - let out = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), - ) - } else { - Rc::clone(node) - }; - copies.insert(key, (Rc::clone(node), Rc::clone(&out))); - out - } - let split = roots - .iter() - .map(|root| { - rebuild( - root, - ExecutionTiming::QueryTime, - assignment, - &reached, - &mut copies, - &mut forced, - ) - }) - .collect(); - let mut remapped = assignment.clone(); - for (original, copy) in copies.values() { - if let Some(materialization) = assignment.get(original) { - remapped.set(copy, materialization); - } - } - (split, remapped) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::operator_properties::{Reduction, Source}; - use asap_types::ir::properties::{ExecutionDataState, ExecutionDataStateError}; - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; - use asap_types::ir::{ - apply_materialization_timings, ASAPOp, NodeToTimedNodeCache, NonASAPOp, Operator, - }; - use asap_types::physical::{Materialization, Medium}; - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - fn scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) - } - - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - fn estimate(child: Rc) -> Rc { - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }), - Schema::lifted( - vec![Field::plain("quantile_0_99", DataType::Float64, false)], - None, - ), - ) - .with_guarantee(None), - ) - } - - fn max(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn concat(children: Vec>) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { children })).unwrap() - } - - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child })) - .unwrap() - } - - fn child(node: &Rc) -> Rc { - Rc::clone(node.children()[0]) - } - - /// One shared sub-DAG reached as ingestion-time input and as query-time - /// input cannot be executed once for both; splitting it by timing first - /// makes the plan timeable. - #[test] - fn a_shared_subtree_reached_at_two_timings_conflicts() { - let shared = scan(); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - max(estimate(agg(Rc::clone(&shared), kll()))), - max(Rc::clone(&shared)), - ], - })) - .unwrap(); - assert_eq!( - apply_materialization_timings( - &root, - &MaterializationAssignment::with_default(AT_INGESTION), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - let (split, assignment) = split_shared_by_timing( - std::slice::from_ref(&root), - &MaterializationAssignment::with_default(AT_INGESTION), - ); - assert!(apply_materialization_timings( - &split[0], - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .is_ok()); - } - - /// Splitting rebuilds the ancestors of a copied sub-DAG; an explicit - /// assignment on one of them follows it onto the rebuilt node. - #[test] - fn split_keeps_explicit_assignments() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let root = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![max(estimate(Rc::clone(&summary))), max(Rc::clone(&shared))], - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - let (split, assignment) = split_shared_by_timing(std::slice::from_ref(&root), &assignment); - let timed = - apply_materialization_timings(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let summary = child(&child(&child(&timed))); - assert!(matches!( - summary.operator, - Operator::ASAP(ASAPOp::SummaryAgg { .. }) - )); - assert_eq!(summary.materialization, AT_INGESTION); - assert_eq!(summary.timing, Some(ExecutionTiming::IngestionTime)); - } - - /// Two queries share a scan: one reads it through a summary built at - /// ingestion time, the other at query time. Splitting the workload's - /// roots together copies the scan for one phase, so one timing pass - /// over both roots succeeds; a node both queries reach at one timing - /// stays one node. - #[test] - fn split_across_roots_copies_only_what_two_queries_read_at_two_timings() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let by_summary = max(estimate(Rc::clone(&summary))); - let raw = max(Rc::clone(&shared)); - let common = max(estimate(agg(scan(), kll()))); - let roots = [ - concat(vec![Rc::clone(&by_summary), Rc::clone(&common)]), - concat(vec![raw, Rc::clone(&common)]), - ]; - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - let timed_unsplit = { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache)) - .collect::, _>>() - }; - assert!(matches!( - timed_unsplit, - Err(ExecutionDataStateError::ConflictingTiming { .. }) - )); - let (split, assignment) = split_shared_by_timing(&roots, &assignment); - assert_eq!(split.len(), 2); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = split - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache).unwrap()) - .collect(); - let summary = child(&child(&child(&timed[0]))); - assert_eq!(summary.materialization, AT_INGESTION); - assert_eq!(child(&summary).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed[1])).timing, - Some(ExecutionTiming::QueryTime) - ); - assert!(Rc::ptr_eq(timed[0].children()[1], timed[1].children()[1])); - } - - /// Splitting a workload that also shares a scan at two timings keeps a - /// conflicting assignment on a fixed-timing node: the split roots still - /// fail with the conflict instead of dropping the assignment. - #[test] - fn a_fixed_timing_conflict_survives_the_split() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let pair = concat(vec![ - max(estimate(Rc::clone(&summary))), - max(Rc::clone(&shared)), - ]); - let state = agg(Rc::clone(&shared), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, AT_INGESTION); - assignment.set(&state, AT_INGESTION); - assignment.set(&read, AT_INGESTION); - let (split, assignment) = split_shared_by_timing(&[pair, read], &assignment); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Result, _> = split - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache)) - .collect(); - assert_eq!( - timed.err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - materialization: AT_INGESTION, - } - ) - ); - } -} diff --git a/crates/plan-selection/Cargo.toml b/crates/plan-selection/Cargo.toml index ac64258a5..ab85b889e 100644 --- a/crates/plan-selection/Cargo.toml +++ b/crates/plan-selection/Cargo.toml @@ -9,33 +9,12 @@ edition = "2021" # the facade's `AssignmentRealizer`: no dependency on Stages 1 or 2, nor on # the executor (#1054 X1, #1047 P7). Tests build their inputs with them. [dependencies] -asap_sketchlib = { workspace = true } asap-types = { path = "../types" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } thiserror = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" -rmp-serde = "1" good_lp = { version = "=1.15.3", default-features = false, features = ["highs"] } highs = "=2.4.0" - -[features] -# `reference_pricer`: the whole-workload selector over enumerated plans, for -# the developer tools' reference pricing (`asap-devtools`). -reference-pricer = [] -# `lp-diagnosis`: with `ASAP_STAGE3_LP_DIAGNOSIS=`, each selection MILP -# solve also reports where its LP relaxation loses its bound (measurement -# only, for the developer tools; a diagnosed run's timings are not -# comparable with an undiagnosed one's). -lp-diagnosis = [] - -[dev-dependencies] -asap-types = { path = "../types", features = ["test-support"] } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } -asap-frontend-promql = { path = "../frontend-promql" } -# The executor's charge for each summary state, checked against the bytes -# plan selection prices for it plus the accumulator's own fields (#947). diff --git a/crates/plan-selection/src/accuracy_binding.rs b/crates/plan-selection/src/accuracy_binding.rs deleted file mode 100644 index 853440ad0..000000000 --- a/crates/plan-selection/src/accuracy_binding.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! Binding the statistics a root guarantee leaves symbolic: from the data -//! workload and the operand-domain contracts. - -use std::collections::HashMap; -use std::rc::Rc; - -use asap_accuracy::composition::{ - parse_raw_data_samples_sign_statistic, parse_source_norm_statistic, INPUT_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, STREAM_L1_NORM, -}; -use asap_accuracy::estimators::{hll, univmon}; -use asap_accuracy::propagation::{ - operand_domain_statistic, operand_guarantee, operand_of, project_column_guarantee, - root_guarantee, -}; -use asap_accuracy::OperandDomain; -use asap_types::ir::operator::operator_properties::BinaryOpKind; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric}; -use asap_types::ir::scalar::ScalarValue; -use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::workload::DataWorkload; - -use crate::SelectionModels; -use asap_types::workload::bounds::declared_non_negative; - -/// [`root_guarantee`] with the statistics Stage 1 left symbolic bound: -/// - the input row count, `series` ([`series_read`](asap_types::workload::bounds::series_read) of `root`); -/// - each source's stream norm ([`source_norm_statistic`]), from the data -/// workload's per-source `stream_norms`; -/// - classic HLL's failure probability and the chance a UnivMon's top layer -/// is incomplete, from the data workload's per-source -/// `max_distinct_per_evaluation`; -/// - a product's or quotient's bound, from the deployment's enforced -/// operand domains ([`operand_domain_bounds`]). -/// -/// Every data-workload fact is declared: the caller drops facts that no -/// longer hold before planning. Any other statistic stays unknown, and an -/// unknown statistic never meets a target. -pub(crate) fn bound_root_guarantee( - root: &QueryRoot, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Option { - let domains = operand_domain_bounds(root, series, data, models); - let bind = |guarantee: &AccuracyGuarantee| { - bind_statistics(guarantee, &|statistic| { - workload_statistic(statistic, series, data) - .or_else(|| domains.get(statistic).copied().flatten()) - }) - }; - match root_guarantee(root) { - Some(guarantee) => Some(bind(&guarantee)), - None => asap_accuracy::propagation::bind_guarantees(root, &bind), - } -} - -/// `guarantee` with the statistics `value` supplies bound. A sign gate -/// ([`raw_data_samples_sign_statistic`](asap_accuracy::composition::raw_data_samples_sign_statistic)) -/// is an indicator that only 1 (proven) opens: any other value leaves it -/// unbound, so it can never read as a perfect bound. -pub(crate) fn bind_statistics( - guarantee: &AccuracyGuarantee, - value: &dyn Fn(&str) -> Option, -) -> AccuracyGuarantee { - guarantee.bind(&|statistic| { - let supplied = value(statistic); - match parse_raw_data_samples_sign_statistic(statistic) { - Some(_) => supplied.filter(|v| *v == 1.0), - None => supplied, - } - }) -} - -/// A data-workload statistic Stage 1 left symbolic: the input row count -/// (`series`, the series of the sources the query reads), one source's -/// stream norm, or classic HLL's failure probability or a UnivMon's -/// incomplete-top-layer probability given one source's distinct bound, -/// from source facts that hold at any planning time, or the sign of one -/// metric's samples, from its declared type. -pub(crate) fn workload_statistic( - statistic: &str, - series: Option, - data: &DataWorkload, -) -> Option { - // The sign of one metric's samples: proven (1) when the metric is - // declared a counter or declared non-negative; nothing is inferred from - // its name. - if let Some(metric) = parse_raw_data_samples_sign_statistic(statistic) { - return declared_non_negative(data, metric).then_some(1.0); - } - if statistic == INPUT_ROW_COUNT { - return series.map(|rows| rows as f64); - } - if let Some((precision, relative_error, source)) = - hll::parse_classic_failure_statistic(statistic) - { - let bound = data.sources.get(source)?.max_distinct_per_evaluation?; - return Some(hll::classic_failure_probability( - precision, - relative_error, - bound, - )); - } - if let Some((heap_size, layers, source)) = - univmon::parse_incomplete_top_layer_statistic(statistic) - { - let bound = data.sources.get(source)?.max_distinct_per_evaluation?; - return Some(univmon::incomplete_top_layer_probability( - heap_size, layers, bound, - )); - } - let (norm, source) = parse_source_norm_statistic(statistic)?; - let norms = data.sources.get(source)?.stream_norms?; - match norm { - STREAM_L1_NORM => norms.l1, - _ => norms.l2, - } -} - -/// One operand of a product or quotient, as Stage 3 finds its domain. -enum DomainOperand<'a> { - /// A planned operator: the deployment's `OperandDomain` for it. - Node(&'a Rc), - /// A literal: itself. - Literal(f64), - /// Anything else: no domain. - Other, -} - -impl<'a> DomainOperand<'a> { - fn of(expr: &'a ScalarExpr) -> Self { - match expr { - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - Self::Node(node) - } - ScalarExpr::Literal(ScalarValue::Float64(value)) => Self::Literal(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => Self::Literal(*value as f64), - _ => Self::Other, - } - } - - fn domain(&self, models: &SelectionModels<'_>) -> Option { - match self { - Self::Node(node) => models.evidence.operand_domain(node), - Self::Literal(value) => Some(OperandDomain { - lower: *value, - upper: *value, - }), - Self::Other => None, - } - } -} - -/// A product or quotient whose bound Stage 1 left symbolic on its operands' -/// domains ([`operand_domain_statistic`]). -struct DomainOperation<'a> { - statistic: &'static str, - operands: [(DomainOperand<'a>, AccuracyGuarantee); 2], -} - -/// Each operand-domain statistic in `root`, bound to the largest bound -/// among the operations that leave it symbolic, or `None` (unbound) when -/// any of them has no bound. Operations are found where the bound -/// originates (the same rule Stage 1 uses), in vector arithmetic, a scalar -/// root, and projection expressions. `series` is the root's input row -/// count ([`series_read`](asap_types::workload::bounds::series_read)). -fn operand_domain_bounds( - root: &QueryRoot, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> HashMap<&'static str, Option> { - let mut operations = Vec::new(); - if let QueryRoot::Scalar(expr) = root { - scalar_domain_operations(expr, &|_| None, &mut operations); - } - let nodes: Vec> = root - .operator_refs() - .iter() - .flat_map(|node| OperatorNode::reachable(node)) - .collect(); - for node in &nodes { - match node.non_asap() { - Some(NonASAPOp::BinaryOp { - operator: - BinaryOperator { - kind: BinaryOpKind::Arithmetic(op), - .. - }, - lhs, - rhs, - .. - }) => { - let (Some(l), Some(r)) = (operand_guarantee(lhs), operand_guarantee(rhs)) else { - continue; - }; - if let Some(statistic) = operand_domain_statistic(op, &l, &r) { - operations.push(DomainOperation { - statistic, - operands: [(DomainOperand::Node(lhs), l), (DomainOperand::Node(rhs), r)], - }); - } - } - Some(NonASAPOp::Project { cols, child, .. }) => { - for item in cols { - scalar_domain_operations( - &item.expr, - &|column| project_column_guarantee(child, column), - &mut operations, - ); - } - } - _ => {} - } - } - let mut bounds: HashMap<&'static str, Option> = HashMap::new(); - for operation in &operations { - let bound = domain_bound(operation, series, data, models); - let entry = bounds.entry(operation.statistic).or_insert(Some(0.0)); - *entry = entry.zip(bound).map(|(a, b)| a.max(b)); - } - bounds -} - -/// The operations in `expr` (and its sub-expressions) whose bound Stage 1 -/// left symbolic on operand domains. -fn scalar_domain_operations<'a>( - expr: &'a ScalarExpr, - column: &dyn Fn(usize) -> Option, - operations: &mut Vec>, -) { - if let ScalarExpr::Arithmetic { - op, left, right, .. - } = expr - { - if let (Some(l), Some(r)) = (operand_of(left, column), operand_of(right, column)) { - if let Some(statistic) = operand_domain_statistic(op, &l, &r) { - operations.push(DomainOperation { - statistic, - operands: [(DomainOperand::of(left), l), (DomainOperand::of(right), r)], - }); - } - } - } - for child in expr.children() { - scalar_domain_operations(child, column, operations); - } -} - -/// The bound of one product or quotient from its operands' domains: -/// - a product: `|x|·e_y + |y|·e_x + e_x·e_y`; -/// - a quotient: `(|x|·e_y + |y|·e_x) / (m·(m − e_y))` with `m` the least -/// `|y|`, which must exceed `e_y` (the denominator's domain is bounded -/// away from zero). -/// -/// `|x|`, `|y|` are the domains' largest magnitudes, and `e` each -/// operand's absolute error (a relative or cardinality bound times that -/// magnitude) after binding its data-workload statistics (`series`, the -/// root's input row count). `None` when an operand has no domain or an -/// unbound error. -fn domain_bound( - operation: &DomainOperation<'_>, - series: Option, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Option { - let [(x, gx), (y, gy)] = &operation.operands; - let (dx, dy) = (x.domain(models)?, y.domain(models)?); - let bind = - |g: &AccuracyGuarantee| g.bind(&|statistic| workload_statistic(statistic, series, data)); - let (gx, gy) = (bind(gx), bind(gy)); - let valid = - |d: &OperandDomain| d.lower.is_finite() && d.upper.is_finite() && d.lower <= d.upper; - if !(valid(&dx) && valid(&dy)) { - return None; - } - let magnitude = |d: &OperandDomain| d.lower.abs().max(d.upper.abs()); - let (mx, my) = (magnitude(&dx), magnitude(&dy)); - let error = |g: &AccuracyGuarantee, m: f64| { - let bound = g.spec.error.evaluate()?; - match g.spec.accuracy_metric { - _ if g.is_exact() => Some(0.0), - AccuracyMetric::ABSOLUTE => Some(bound), - AccuracyMetric::Relative => Some(bound * m), - _ => None, - } - }; - let (ex, ey) = (error(&gx, mx)?, error(&gy, my)?); - let bound = if operation.statistic == PRODUCT_OPERAND_DOMAINS { - mx * ey + my * ex + ex * ey - } else { - let least = if dy.lower > 0.0 { - dy.lower - } else if dy.upper < 0.0 { - -dy.upper - } else { - return None; - }; - if least <= ey { - return None; - } - (mx * ey + my * ex) / (least * (least - ey)) - }; - bound.is_finite().then_some(bound) -} diff --git a/crates/plan-selection/src/checks.rs b/crates/plan-selection/src/checks.rs deleted file mode 100644 index 3ac348d12..000000000 --- a/crates/plan-selection/src/checks.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Validity checks on one candidate: accuracy targets and latency. What -//! the executor or deployment cannot run (per-entity aggregates without a -//! kernel, query-time scans of raw data not retained) Stage 2 prunes. - -use asap_cost::{family_name, PlanCost, QueryLatency}; -use asap_types::ir::properties::{ - AccuracyGuarantee, AccuracyMetric, AccuracyScope, AccuracyTarget, -}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, WeightDomain}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use crate::accuracy_binding::bound_root_guarantee; -use crate::report::{AcceptedAccuracy, RejectionCause}; -use crate::SelectionModels; -use asap_types::workload::bounds::{confirmed_weights, series_read, values_per_result}; - -/// The first query whose latency per evaluation is over its -/// `max_latency_ms` bound or, with no bound broken, over its repeat -/// interval (a query evaluated every `I` ms whose evaluation takes longer -/// than `I` falls further behind with every evaluation), as a rejection -/// cause and reason. -pub(crate) fn latency_violation( - cost: &PlanCost, - entries: &[QueryWorkloadEntry], -) -> Option<(RejectionCause, String)> { - let per_query = || cost.usage.latency_per_query.iter().zip(entries).enumerate(); - let latency_text = |query: usize, latency: &QueryLatency| { - format!( - "q{}: latency {:.1} ms per evaluation (compaction {:.1} ms + query {:.1} ms)", - query + 1, - latency.total_ms(), - latency.compaction_ms, - latency.query_ms - ) - }; - let over_bound = - per_query().find_map( - |(query, (latency, entry))| match entry.requirements.max_latency_ms { - Some(bound) if latency.total_ms() > bound => Some(( - RejectionCause::LatencyBoundExceeded, - format!( - "{} is over its {bound} ms bound", - latency_text(query, latency) - ), - )), - _ => None, - }, - ); - over_bound.or_else(|| { - per_query().find_map(|(query, (latency, entry))| { - match entry.recurrence.fixed_interval_ms() { - Some(interval_ms) if latency.total_ms() > interval_ms as f64 => Some(( - RejectionCause::LatencyOverRepeatInterval, - format!( - "{} is over its {interval_ms} ms repeat interval, so its evaluations \ - cannot keep up", - latency_text(query, latency) - ), - )), - _ => None, - } - }) - }) -} - -/// Each entry's accuracy target, as the per-query targets Stage 3 checks. -pub(crate) fn accuracy_targets(entries: &[QueryWorkloadEntry]) -> Vec { - entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect() -} - -/// What Stage 3 accepted for each query ([`AcceptedAccuracy`]), or the -/// first query whose output is not shown to meet its target, as a reason. An output meets its target when its theoretical root guarantee -/// (composed in Stage 1 with the built-in model) satisfies it, or when the -/// deployment's empirical evidence shows that this candidate's output does -/// ([`AccuracyEvidenceProvider::output_guarantee`]). A -/// certified estimate inside the plan is not enough on its own: a difference -/// or ratio of certified estimates can be unbounded. The root guarantee's -/// symbolic statistics are bound first ([`bound_root_guarantee`]); one still -/// unknown leaves the target unmet. Count-Min is admitted only over update -/// weights proven non-negative, which its bound and the executor assume: an open -/// [`WeightDomain::RawDataSamplesOfMetric`] requirement is met when `data` declares -/// the metric a counter or non-negative ([`confirmed_weights`]), and unmet -/// otherwise. -/// -/// A target that sets no metric accepts a guarantee in any metric that meets -/// its ε, δ and scope ([`AccuracyModel::satisfies`]); one that sets a metric -/// accepts only guarantees in it (or convertible to it). -/// -/// The Count-Min weight check runs first and rejects as -/// [`RejectionCause::CountMinWeightsNotProvenNonNegative`]: the executor -/// cannot run such a plan. Then the deployment's accuracy model is asked -/// ([`AccuracyModel::accepts_output_regardless_of_guarantee`], through -/// `SelectionModels::accuracy`): an output it accepts is accepted -/// whatever its guarantee says, and the guarantee check's rejection, if any, -/// is only recorded ([`AcceptedAccuracy::guarantee_check_rejection`]). -/// Otherwise, an output with no guarantee from either -/// source (Stage 1 had no rule, and the evidence has none) is rejected as -/// [`RejectionCause::NoAccuracyGuarantee`], and any other miss as -/// [`RejectionCause::AccuracyTargetMissed`]. -pub(crate) fn accuracy_check( - candidate: &PhysicalWorkloadPlan, - targets: &[AccuracyTarget], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Result, CheckFailure> { - let mut accepted = Vec::with_capacity(targets.len()); - for (query, (root, target)) in candidate.roots.iter().zip(targets).enumerate() { - let query = query + 1; - let mut sketches: Vec = Vec::new(); - for node in root - .operator_refs() - .iter() - .flat_map(|node| OperatorNode::reachable(node)) - { - let Operator::ASAP( - ASAPOp::SummaryAgg { family, input, .. } - | ASAPOp::SummaryTumblingWindowChain { family, input, .. }, - ) = &node.operator - else { - continue; - }; - if let FieldDataType::Sketch(kind, _) = family { - let name = format!("{:?}", kind.algorithm()); - if !sketches.contains(&name) { - sketches.push(name); - } - } - if matches!(family, FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap)) - && !matches!( - confirmed_weights(&input.weight_domain, data), - WeightDomain::NonNegative { .. } - ) - { - let unconfirmed = match &input.weight_domain { - WeightDomain::RawDataSamplesOfMetric { metric } => format!( - " (samples of `{metric}`, which is not declared a counter or \ - non-negative)" - ), - WeightDomain::CounterDerivativeOfMetric { metric } => format!( - " (rates or increases of `{metric}`, which is not declared a counter)" - ), - _ => String::new(), - }; - // Count-Min (plain, CmsWithHeap or Hydra Count-Min) over - // weights not proven non-negative does not run: the executor - // refuses CmsWithHeap and HydraCms when it builds the plan, - // and plain Count-Min adds a negative weight, which voids its - // never-underestimate bound. So - // this rejects the plan whatever the deployment's accuracy - // model accepts. - return Err(CheckFailure { - cause: RejectionCause::CountMinWeightsNotProvenNonNegative, - reason: format!( - "q{query}: {} needs non-negative update weights, and these are not \ - proven non-negative{unconfirmed}", - family_name(family) - ), - size_independent: true, - }); - } - } - // The guarantee check: the theoretical guarantee, then the - // deployment's evidence. - let checked = guarantee_check(query, root, target, data, models); - // The deployment's model may accept the output directly, whatever - // its guarantee says; the check's result is then only recorded. - if models - .accuracy - .accepts_output_regardless_of_guarantee(root, target) - { - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: None, - accepted_regardless_of_guarantee: true, - guarantee_check_rejection: checked.err().map(|failure| failure.cause), - sketches, - }); - continue; - } - accepted.push(AcceptedAccuracy { - target_metric: target.accuracy_metric, - accepted_metric: checked?, - accepted_regardless_of_guarantee: false, - guarantee_check_rejection: None, - sketches, - }); - } - Ok(accepted) -} - -/// A check a candidate failed: why, with its numbers, and whether every -/// size of the same plan's summaries fails it too. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct CheckFailure { - pub(crate) cause: RejectionCause, - pub(crate) reason: String, - /// `true` when no other size of the plan's summaries can pass the check - /// (e.g. no rule composes a guarantee, or a bound stays unknown) and the - /// deployment's accuracy model and evidence cannot judge another size - /// otherwise; `false` when that is not proven. - pub(crate) size_independent: bool, -} - -/// Whether query `query`'s output `root` meets `target` by its theoretical -/// root guarantee or, failing that, the deployment's evidence -/// ([`accuracy_check`]): the metric of the guarantee that met it (`None` -/// for an exact output), or why not, as -/// [`RejectionCause::NoAccuracyGuarantee`] or -/// [`RejectionCause::AccuracyTargetMissed`]. -pub(crate) fn guarantee_check( - query: usize, - root: &QueryRoot, - target: &AccuracyTarget, - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> Result, CheckFailure> { - // A guarantee per value is stated over the values the target's scope - // covers: the whole result's `G`, or at most `⌊1 / share⌋` groups. - let series = series_read(root, data); - let values = values_per_result(root, series, data); - let over_scope = |g: AccuracyGuarantee| g.over_scope(target.accuracy_scope, values); - let guarantee = bound_root_guarantee(root, series, data, models).map(over_scope); - let meets = |g: &AccuracyGuarantee| models.accuracy.satisfies(g, target); - // The deployment's evidence is read only when the theoretical - // guarantee does not meet the target. - let evidence = || models.evidence.output_guarantee(root).map(over_scope); - let (met, evidence) = match guarantee.clone().filter(meets) { - Some(met) => (Some(met), None), - None => { - let evidence = evidence(); - (evidence.clone().filter(meets), evidence) - } - }; - if let Some(met) = met { - return Ok((!met.is_exact()).then_some(met.spec.accuracy_metric)); - } - // Whether the deployment's model or evidence may judge another size of - // the plan's summaries otherwise. - let model_or_evidence_may_judge = models.accuracy.may_accept_regardless_of_guarantee() - || models.evidence.may_provide_evidence(); - if guarantee.is_none() && evidence.is_none() { - return Err(CheckFailure { - cause: RejectionCause::NoAccuracyGuarantee, - reason: format!( - "q{query}: the output has no guarantee, and the deployment has no evidence \ - for it, so it cannot meet target {target:?}" - ), - // No rule composes one, whatever the sizes. - size_independent: !model_or_evidence_may_judge, - }); - } - // A bound or failure probability that stays unknown once bound is - // unknown at every size; a known one may shrink with larger summaries. - let unknown = |g: &AccuracyGuarantee| { - g.spec.error.evaluate().is_none() - || (target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none()) - }; - let size_independent = evidence.is_none() - && guarantee.as_ref().is_some_and(unknown) - && !model_or_evidence_may_judge; - Err(CheckFailure { - cause: RejectionCause::AccuracyTargetMissed, - size_independent, - reason: match guarantee { - Some(g) - if values.is_none() - && target.accuracy_scope == AccuracyScope::WholeResult - && target.failure_probability.is_some() - && g.spec.failure_probability.evaluate().is_none() => - { - format!( - "q{query}: the output's guarantee holds for each value, and the number \ - of values one evaluation returns is not bound (no declared series \ - count for a source a PromQL vector reads, and no declared \ - `distinct_values_per_label` for every group key nor a \ - `distinct_values_per_label_set` covering them, or a key that is not \ - an unchanged source column), so it does not meet \ - whole-result target {target:?}, and \ - the deployment has no evidence that it meets it (a target on the groups \ - holding a share of the records, `groups_with_record_share_at_least`, \ - covers a bound number of values)" - ) - } - Some(g) => format!( - "q{query}: the output guarantees a {:?} bound {:?}, failure probability \ - {:?}, which misses target {target:?}, and the deployment has no evidence \ - that it meets it", - g.spec.accuracy_metric, - g.spec.error.evaluate(), - g.spec.failure_probability.evaluate(), - ), - None => format!( - "q{query}: the output has no guarantee, and the deployment has no \ - evidence that it meets target {target:?}" - ), - }, - }) -} diff --git a/crates/plan-selection/src/groups.rs b/crates/plan-selection/src/groups.rs deleted file mode 100644 index 1421f5d2a..000000000 --- a/crates/plan-selection/src/groups.rs +++ /dev/null @@ -1,2163 +0,0 @@ -//! #1095 C3: what the owner-scoped candidate MILP (§2.0) is built from. Given -//! Stage 2's physical candidates (one per operator node, timing and -//! materialization; Stage 2 makes no copies), it computes: -//! -//! 1. **Candidate dominance** ([`prune_dominated_candidates`]): a candidate no optimal -//! plan under cost model 1 uses is deleted, with its links. -//! 2. **Per-candidate data** ([`CandidateData`]): whether the candidate outputs -//! approximate values and the budget parts it is sized for, and its usage -//! per execution, standing cost, held bytes and own latency, every number -//! from [`asap_cost::node_usage`]. -//! 3. **Owners** ([`OwnerRegion`]): one per query and one per possible -//! instance of each shareable candidate, each with its region of stateless -//! candidates, the links inside it and the shareable candidates it may read. -//! 4. **Accuracy weights** ([`AccuracyWeight`]) per query and approximate -//! candidate, and pairs excluded together, from the composition of the -//! query's outputs. -//! -//! Nothing calls it yet; the MILP (C4) will. -//! -//! ## Usage per execution and its context -//! -//! A candidate's usage is [`asap_cost::node_usage`] of its operator node at one -//! execution per second (`rate = (1, 1.0)`); its standing cost is the same -//! call's cost at no execution (`rate = (1, 0.0)`), and its execution cost -//! the difference. `node_usage` also needs facts the candidate alone does not -//! fix, its *context*: -//! - the outputs of the producers it reads (one per input; the producer -//! candidates of an input may output different statistics); -//! - for a query-time scan, the window each reader reads, through time -//! shifts ([`window_read_through`], which follows -//! [`PhysicalASAPDAG::scan_window`] for a plan with that one reader); -//! - for a pane chain kept beyond one evaluation, the panes it keeps, which -//! grow with the range-evaluation span of the queries reaching it -//! ([`asap_cost::tumbling_window_retention_ms`], [`asap_cost::retained_tumbling_windows`]). -//! -//! Every combination of these options is costed; a combination -//! `asap-cost` cannot cost is skipped -//! (no plan can use it). When the numbers agree, the usage is -//! plan-independent; otherwise [`CandidateData::usage_depends_on`] says which -//! options changed it, and [`CandidateData::usage`] holds the first costed -//! combination's numbers. An ingestion-time node's maintained window is -//! read from the first producer candidate of each input -//! ([`asap_cost::maintained_window_ms`]). -//! -//! Candidate links can close a cycle (on #1036 workload-01, a widened scan's -//! `Project` that reads its own memo group). Candidates are costed producers first; on a -//! cycle, a candidate is costed from its producers already costed and marked -//! [`UsageContext::ProducerOnACycle`]. - -//! ## Dominance (exact under cost model 1) -//! -//! Candidate `k` is deleted when another candidate `k'` (the *dominating* candidate): -//! - is in the same memo group, at the same timing and materialization, -//! with the same held state; -//! - has the same producer candidates per input, the same readers (every -//! (reader candidate, input) that may read `k` may read `k'`, and nothing -//! else), the same output references it may produce, the same reader -//! constraints and keep flags; -//! - has the same `(approx, parts)`; -//! - has a guarantee at least as strong ([`guarantee_at_least_as_strong`]); -//! - outputs the same schema, sources, state family and unchanged columns -//! with an operator of the same kind (pane chains: the same panes), so its -//! readers are costed with the same formulas; -//! - and has every number at most `k`'s: CPU operations and scanned bytes -//! per execution, latency (work per execution), output rows and bytes, -//! kept memory (average and peak), disk bytes, held bytes and raw data -//! kept. -//! -//! Both candidates' usage must be plan-independent (otherwise the comparison -//! would hold for one context only). Readers' usage is assumed -//! non-decreasing in their inputs' rows and bytes. Among candidates that -//! dominate each other the one with the smallest id is kept. Every link to -//! a deleted candidate is deleted; its readers still read the dominating candidate. -//! -//! ## Accuracy weights -//! -//! `w(q, k)` says whether query `q` may read approximate candidate `k` (a -//! sketch [`ASAPOp::SummaryAgg`] or [`ASAPOp::SummaryTumblingWindowChain`], what -//! [`asap_accuracy::propagation::approximate_inputs`] counts), and pairs of -//! approximate candidates `q` may not read together. Both come from the -//! composition itself, never from an operator table -//! ([`accuracy_positions`]): `q`'s outputs are built over the candidates -//! with `k` approximate and every other candidate exact, their guarantees -//! composed by `asap-accuracy` ([`asap_accuracy::propagation::composed_guarantee`]) -//! and checked by the same check the selected plan passes -//! ([`crate::checks::guarantee_check`], with the deployment's model): -//! - every output exact: `k`'s values do not reach the output (only a row -//! count does), and `k` gets no weight; -//! - every output meets the target: `w(q, k) = 1`, one share of `q`'s -//! budget ([`asap_accuracy::budget`]); `k`'s sizes are those of its -//! `parts`, so a lone denominator sized for the whole target -//! (`ε/(1 − ε) > ε`) fails here; -//! - otherwise [`AccuracyWeight::Unusable`], with the check's cause -//! ([`UnusableReason`]): e.g. a rank error through any arithmetic, or an -//! HLL whose declared distinct bound is outside its contract. -//! -//! For each pair `k1, k2` of candidates usable alone, both sized for at -//! least two shares, whose values meet in one plan of `q` (under two inputs -//! of one candidate, under two references of a scalar root, or one below -//! the other), the outputs are built and checked with both approximate; a -//! pair that fails is excluded (`a[q, k1] + a[q, k2] ≤ 1`). A pair with a -//! member sized for one share is excluded by the shares row already. The -//! rows still bound the shares a plan reads by each candidate's `parts`, -//! which the equal split proves for one product or quotient of two inputs; -//! three or more approximate inputs are not built together. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - -use asap_cost::{CostInputs, NodeUsage, ProducerOutput}; -use asap_types::ir::materialized_view::RefreshWindow; -use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, ScanWindow, -}; -use asap_types::ir::properties::execution_data_state::{DataPrimitive, ExecutionDataState}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; -use asap_types::ir::schema::{FieldDataType, Schema}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use asap_types::physical::Materialization; -use asap_types::stages::{ - output_references, MemoGroupId, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, -}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use crate::SelectionModels; - -use asap_types::stages::{PhysicalCandidateId, ReaderConstraint}; - -/// What the MILP is built from (module documentation). -#[derive(Debug, Clone)] -pub(crate) struct MilpInputs { - /// Indexed by [`PhysicalCandidateId`]: each candidate's data, pruned or not. - pub(crate) candidate_data: Vec, - /// The candidates dominance deleted, ascending. - pub(crate) pruned_candidates: Vec, - /// Indexed by [`PhysicalCandidateId`]: per input, the producer candidates left - /// after pruning (empty for a pruned candidate). - pub(crate) producer_candidates_per_input: Vec>>, - /// Per output reference (`output_references` of the query roots): the - /// candidates left that may produce it, `Out(r)`, ascending. - pub(crate) output_candidates: Vec>, - /// Indexed by [`PhysicalCandidateId`]: where it runs. - pub(crate) kind: Vec, - /// Indexed by [`PhysicalCandidateId`]: the result it gives. - pub(crate) slot: Vec, - /// The execution contexts: one per query (`0..Q`, in workload order), - /// then the ingestion pipelines `I_r` some context reads, ascending by - /// [`IngestionPipeline`], then one per shareable candidate some context - /// may read, ascending by candidate. - pub(crate) contexts: Vec, - /// Per shareable candidate with a context, its index in `contexts`. - pub(crate) context_of_shareable: BTreeMap, - /// Per ingestion-time shareable candidate whose context reads a - /// candidate of an `I_r`: the index of that `I_r` in `contexts`. - pub(crate) ingestion_context_of_shareable: BTreeMap, - /// Per query: `w(q, k)` for each approximate candidate `k` whose values - /// reach its output. - pub(crate) accuracy_weights: Vec>, - /// Per query: the pairs `(k1, k2)`, `k1 < k2`, of candidates usable alone - /// that it may not read together. - pub(crate) accuracy_excluded_pairs: Vec>, -} - -/// One candidate's data for the MILP. -#[derive(Debug, Clone)] -pub(crate) struct CandidateData { - /// Whether it outputs approximate values: a sketch `SummaryAgg` or - /// `SummaryTumblingWindowChain`, each an approximate input of the queries reading it. - pub(crate) approximate: bool, - /// `parts(k)`: its operator node's `accuracy_budget_parts` (Stage 2's - /// `OperatorNodeOfCandidateForMemoGroup`: for a factored node, the - /// largest among its candidates). - pub(crate) accuracy_budget_parts: usize, - /// Its usage under the first context, or why `asap-cost` could not cost - /// it. - pub(crate) usage: Result, - /// The context options that changed its usage, ascending; empty when it - /// is plan-independent. - pub(crate) usage_depends_on: Vec, - /// The inputs whose producer (or that producer's output) changes its - /// usage or its own output, ascending. - pub(crate) inputs_changing_usage: Vec, - /// Its usage and its own output (an index below `output_count`) per - /// choice of a producer and that producer's output for each of - /// `inputs_changing_usage` (in that order); `None` when `asap-cost` could - /// not cost that choice, so no plan can make it. Empty when - /// `inputs_changing_usage` is. - pub(crate) usage_per_producers: Vec<(Vec, Option)>, - /// For a query-time scan whose usage depends on the window its reader - /// reads ([`UsageContext::ScanWindowOfReader`]): its usage per window it - /// can be costed at, with its output there. Empty otherwise. - pub(crate) usage_per_window: Vec<(ScanWindow, CandidateUsage, usize)>, - /// How many distinct outputs (statistics its readers are costed from) - /// it can give, by context. - pub(crate) output_count: usize, - /// For a query-time scan: per reader, the windows it may read the scan - /// at (through time shifts, [`window_read_through`]). Empty for any - /// other candidate. - pub(crate) windows_per_reader: Vec<(PhysicalCandidateId, Vec)>, - /// For a kept pane chain whose usage depends on the panes it retains - /// ([`UsageContext::PaneRetentionOfReachingQueries`]): its usage per - /// retained pane count. Empty otherwise. - pub(crate) usage_per_retained_panes: Vec<(u64, CandidateUsage)>, - /// With `usage_per_retained_panes`: per query that can reach it, the - /// panes it needs retained. - pub(crate) retained_panes_per_query: Vec<(usize, u64)>, - /// Whether some entry of `usage_per_producers` stands for several usages - /// (a producer whose own output depends on its context), of which the - /// costliest is kept. - pub(crate) producer_usage_ambiguous: bool, -} - -impl Default for CandidateData { - fn default() -> Self { - Self { - approximate: false, - accuracy_budget_parts: 1, - usage: Err("not costed".to_string()), - usage_depends_on: Vec::new(), - inputs_changing_usage: Vec::new(), - usage_per_producers: Vec::new(), - usage_per_window: Vec::new(), - output_count: 1, - windows_per_reader: Vec::new(), - usage_per_retained_panes: Vec::new(), - retained_panes_per_query: Vec::new(), - producer_usage_ambiguous: false, - } - } -} - -/// One candidate's usage, from [`asap_cost::node_usage`] (module -/// documentation). -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct CandidateUsage { - /// `u(k)`: CPU operations of one execution. - pub(crate) cpu_ops_per_execution: f64, - /// `u(k)`: bytes one execution scans. - pub(crate) scanned_bytes_per_execution: f64, - /// Milliseconds of one execution on one core. - pub(crate) work_ms_per_execution: f64, - /// `d_k`: what it adds to a reaching query's latency in - /// [`asap_cost::plan_cost`], its work per execution at query time; 0 - /// at ingestion time, whose latency is the plan-level compaction term. - pub(crate) own_latency_ms: f64, - /// Cost model 1: the cost of one execution; `None` under other models. - pub(crate) cost_per_execution: Option, - /// Cost model 1: the cost per second of what it keeps whatever runs it - /// (kept memory and disk); `None` under other models. - pub(crate) standing_cost_per_second: Option, - /// Memory kept between executions, averaged over time. - pub(crate) average_memory_bytes: f64, - pub(crate) peak_memory_bytes: u64, - pub(crate) disk_bytes: u64, - /// Query-time memory held while an evaluation reading it runs (a - /// `MatAtQuery` state, or a not-materialized node's working memory). - pub(crate) held_bytes: u64, - /// Raw data a query-time scan keeps on disk. - pub(crate) raw_data_kept_bytes: u64, - /// An in-memory ingestion-time state's bytes per window. - pub(crate) maintained_window_bytes: u64, - pub(crate) output_rows: u64, - pub(crate) output_bytes: u64, -} - -/// A candidate's usage and its output (an index below -/// [`CandidateData::output_count`]) in one context. -pub(crate) type UsageAndOutput = (CandidateUsage, usize); - -/// One producer choice for an input: a producer candidate, and for one -/// with several outputs ([`CandidateData::output_count`]), which output. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ProducerChoice { - pub(crate) class: PhysicalCandidateId, - pub(crate) output: Option, -} - -/// A context option that changed a candidate's usage. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum UsageContext { - /// Which producer candidate (of different output) an input reads. - ProducerOutput, - /// Which reader reads a query-time scan, so which window it scans. - ScanWindowOfReader, - /// Which queries reach a kept pane chain, so how many panes it keeps. - PaneRetentionOfReachingQueries, - /// A producer candidate on a cycle of candidate links was not costed yet, so - /// its outputs were not among the options. - ProducerOnACycle, - /// `asap-cost` could not cost some contexts (e.g. a scan read with no - /// window from a source without `rows_held`); the usage is the first - /// costed one's. - UncostableContext, -} - -/// A candidate dominance deleted. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct DominatedCandidate { - pub(crate) class: PhysicalCandidateId, - pub(crate) dominated_by: PhysicalCandidateId, - pub(crate) reason: String, -} - -/// `w(q, k)`: one share of `q`'s budget, or unusable. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum AccuracyWeight { - One, - /// `∞`: `k` is unusable for `q`. - Unusable(UnusableReason), -} - -/// Why a position's weight is `∞`: the check of `q`'s outputs with `k` -/// approximate failed. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) enum UnusableReason { - /// Some output has no guarantee - /// ([`RejectionCause::NoAccuracyGuarantee`](crate::report::RejectionCause::NoAccuracyGuarantee)). - NoAccuracyGuarantee, - /// Some output's guarantee misses the target - /// ([`RejectionCause::AccuracyTargetMissed`](crate::report::RejectionCause::AccuracyTargetMissed)). - AccuracyTargetMissed, - /// Not checked, so not usable: a node that does not rebuild over its - /// producers. - NotChecked, -} - -/// Where a candidate runs (#1095 §2.0, "Memo groups and execution -/// contexts"), by its materialization alone, as `asap_cost::plan_cost` -/// counts its executions: -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) enum RunKind { - /// `NotMaterialized` at query time: computed once in each execution - /// context (a query evaluation, or a shared query-time candidate's - /// inputs) that selects it. - PerContext, - /// `NotMaterialized` at ingestion time: runs in the ingestion pipeline - /// `I_r` of its readers' refresh window `r`, once per second of - /// arriving data for all of them. - Ingestion, - /// Materialized (`MatAtIngestion`, `MatAtQuery`, - /// `MatAtQueryKeptAcrossEvaluations`): selected once, its own execution - /// context for its inputs, charged once. - Shareable, -} - -/// The [`RunKind`] of a candidate at `timing` and `materialization`. -pub(crate) fn run_kind(timing: ExecutionTiming, materialization: Materialization) -> RunKind { - match (materialization, timing) { - (Materialization::NotMaterialized, ExecutionTiming::QueryTime) => RunKind::PerContext, - (Materialization::NotMaterialized, ExecutionTiming::IngestionTime) => RunKind::Ingestion, - _ => RunKind::Shareable, - } -} - -/// What "one candidate per memo group per context" (row 2) ranges over: -/// the candidates that give one result. A memo group's implementation -/// roots give the group's result; an implementation's private nodes -/// (`is_implementation_root == false`) give an intermediate result of -/// their own, so each private operator node is its own slot (its -/// candidates differ only in timing and materialization). -/// -/// The memo group graph is not acyclic: a widened scan puts a `Project` in -/// the memo group of the scan it reads, so a group's candidate may read -/// its own group. Such a candidate and its producer cannot share one slot -/// (one context would need both), so a group's roots are split by -/// *self-read depth*: 0 for a root reading no root of its own group (the -/// scan), 1 for one that may (the `Project`). A depth-1 root reading -/// another depth-1 root of its group in one context (a `Project` over a -/// `Project` of the widened scan) is then a read of its own slot, which -/// [`CandidateLinks`] drops for candidates that run in a context. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum ResultSlot { - Group(MemoGroupId, u32), - PrivateNode(OperatorNodeOfCandidateForMemoGroupId), -} - -/// The [`ResultSlot`] of every candidate (see there). -pub(crate) fn result_slots(physical: &PhysicalASAPMemoCandidates) -> Vec { - let count = physical.candidates.len(); - let root = |candidate: usize| { - let operator_node = physical.candidates[candidate].operator_node; - let node = &physical.operator_nodes_of_candidates_for_memo_groups[operator_node.0]; - node.is_implementation_root - .then(|| node.group()) - .ok_or(operator_node) - }; - // Self-read depth: 1 for a root that may read a root of its own group. - let depth: Vec = (0..count) - .map(|candidate| { - let Ok(group) = root(candidate) else { - return 0; - }; - let reads_own_group = physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .flat_map(|producers| producers.iter()) - .any(|producer| producer.0 != candidate && root(producer.0) == Ok(group)); - u32::from(reads_own_group) - }) - .collect(); - (0..count) - .map(|candidate| match root(candidate) { - Ok(group) => ResultSlot::Group(group, depth[candidate]), - Err(operator_node) => ResultSlot::PrivateNode(operator_node), - }) - .collect() -} - -/// Per candidate of a [`PhysicalASAPMemoCandidates`]: its producer -/// candidates per input, ascending and without repeats, and its -/// [`RunKind`]. -struct CandidateLinks { - producers_per_input: Vec>>, - kind: Vec, -} - -impl CandidateLinks { - fn new(physical: &PhysicalASAPMemoCandidates) -> Self { - let producers_per_input: Vec>> = physical - .candidates - .iter() - .map(|candidate| { - candidate - .producer_candidates_per_input - .iter() - .map(|producers| { - let set: BTreeSet<_> = producers.iter().copied().collect(); - set.into_iter().collect() - }) - .collect() - }) - .collect(); - let kind: Vec = physical - .candidates - .iter() - .map(|candidate| run_kind(candidate.timing, candidate.materialization)) - .collect(); - // With one candidate per result slot per context, a candidate that - // runs in a context cannot read its own slot there (only a self-loop - // could): such links are dropped (a widened scan's `Project` lists - // its own memo group). A shareable candidate reads in its own - // context, so it keeps them, but never reads itself. - let slot = result_slots(physical); - let producers_per_input = producers_per_input - .into_iter() - .enumerate() - .map(|(reader, inputs): (usize, Vec>)| { - inputs - .into_iter() - .map(|producers| { - producers - .into_iter() - .filter(|producer| { - producer.0 != reader - && (kind[reader] == RunKind::Shareable - || slot[producer.0] != slot[reader]) - }) - .collect() - }) - .collect() - }) - .collect(); - Self { - producers_per_input, - kind, - } - } - - /// Whether `candidate` is materialized: selected once, its own context. - fn is_shareable(&self, candidate: PhysicalCandidateId) -> bool { - self.kind[candidate.0] == RunKind::Shareable - } -} - -/// Per candidate, its reader constraints, each as an id equal for equal -/// constraints once the constrained candidate is left out, ascending and -/// without repeats. -fn reader_constraint_ids_per_candidate(physical: &PhysicalASAPMemoCandidates) -> Vec> { - let mut ids: HashMap = HashMap::new(); - let mut per_candidate = vec![Vec::new(); physical.candidates.len()]; - for constraint in &physical.reader_constraints { - let mut unowned = constraint.clone(); - *unowned.candidate_mut() = PhysicalCandidateId(usize::MAX); - let next = ids.len() as u32; - per_candidate[constraint.candidate().0].push(*ids.entry(unowned).or_insert(next)); - } - for constraint_ids in &mut per_candidate { - constraint_ids.sort_unstable(); - constraint_ids.dedup(); - } - per_candidate -} - -/// [`MilpInputs`] for `physical`'s candidates; `entries[q]` is query `q`'s -/// workload entry. Accuracy is checked with the built-in model and no -/// evidence ([`prepare_milp_inputs_with`]). -#[cfg(test)] -pub(crate) fn prepare_milp_inputs( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - cost_inputs: &CostInputs, -) -> MilpInputs { - let models = SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs, - }; - prepare_milp_inputs_with(physical, entries, data, &models) -} - -/// [`MilpInputs`] for `physical`'s candidates, its accuracy positions -/// checked with `models` (the deployment's accuracy model and evidence) and -/// its usage costed with `models.cost_inputs`. -pub(crate) fn prepare_milp_inputs_with( - physical: &PhysicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> MilpInputs { - let cost_inputs = models.cost_inputs; - let links = &CandidateLinks::new(physical); - let class_count = physical.candidates.len(); - let all_alive = vec![true; class_count]; - let unpruned_links = producer_candidates_left(links, &all_alive); - let unpruned_outputs = output_candidates(physical, &all_alive); - let unpruned_reach = queries_reaching(physical, &unpruned_links, &unpruned_outputs); - let refresh_windows = refresh_windows(physical, links); - // Every context before pruning, for the windows each scan may be read - // at within one context. - let all_contexts = execution_contexts( - physical, - &links.kind, - &unpruned_links, - &unpruned_outputs, - &refresh_windows, - |_| true, - ) - .contexts; - let candidate_data = candidate_data( - physical, - links, - entries, - &unpruned_reach, - &all_contexts, - &unpruned_outputs, - data, - cost_inputs, - ); - let pruned_candidates = prune_dominated_candidates( - &dominance_profiles(physical, links, &unpruned_outputs), - &candidate_data, - ); - let mut alive = all_alive; - for pruned in &pruned_candidates { - alive[pruned.class.0] = false; - } - let producer_candidates_per_input = producer_candidates_left(links, &alive); - let output_candidates = output_candidates(physical, &alive); - let kind = links.kind.clone(); - let slot = result_slots(physical); - let usable = |class: PhysicalCandidateId| candidate_data[class.0].usage.is_ok(); - let ExecutionContexts { - contexts, - context_of_shareable, - ingestion_context_of_shareable, - } = execution_contexts( - physical, - &kind, - &producer_candidates_per_input, - &output_candidates, - &refresh_windows, - usable, - ); - let AccuracyPositions { - weights: accuracy_weights, - excluded_pairs: accuracy_excluded_pairs, - } = accuracy_positions::accuracy_positions( - physical, - &candidate_data, - &producer_candidates_per_input, - &output_candidates, - entries, - data, - models, - ); - MilpInputs { - candidate_data, - pruned_candidates, - producer_candidates_per_input, - output_candidates, - kind, - slot, - contexts, - context_of_shareable, - ingestion_context_of_shareable, - accuracy_weights, - accuracy_excluded_pairs, - } -} - -/// Per candidate, per input, its producer candidates that are `alive` (none for a -/// candidate that is not). -fn producer_candidates_left( - links: &CandidateLinks, - alive: &[bool], -) -> Vec>> { - links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| match alive[class] { - false => Vec::new(), - true => producers_per_input - .iter() - .map(|producers| { - producers - .iter() - .copied() - .filter(|producer| alive[producer.0]) - .collect() - }) - .collect(), - }) - .collect() -} - -/// `Out(r)` per output reference: the `alive` candidates among its output -/// candidates, ascending. -fn output_candidates( - physical: &PhysicalASAPMemoCandidates, - alive: &[bool], -) -> Vec> { - (0..physical.output_reference_count()) - .map(|reference| { - let set: BTreeSet<_> = physical - .query_output_candidates(reference) - .filter(|candidate| alive[candidate.0]) - .collect(); - set.into_iter().collect() - }) - .collect() -} - -/// Per candidate, the queries whose output candidates reach it through the links -/// `producer_candidates_per_input`, ascending. -fn queries_reaching( - physical: &PhysicalASAPMemoCandidates, - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], -) -> Vec> { - let mut reaching = vec![Vec::new(); producer_candidates_per_input.len()]; - let output_refs = output_references(&physical.query_roots); - for query in 0..physical.query_roots.len() { - let mut seen = vec![false; producer_candidates_per_input.len()]; - let mut stack: Vec = output_refs - .iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .flat_map(|(reference, _)| output_candidates[reference].iter().copied()) - .collect(); - while let Some(class) = stack.pop() { - if std::mem::replace(&mut seen[class.0], true) { - continue; - } - reaching[class.0].push(query); - stack.extend( - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - } - reaching -} - -/// Which execution context (#1095 §2.0): a query evaluation, an ingestion -/// pipeline `I_r`, or a shareable candidate's own inputs. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum ContextKind { - Query(usize), - Ingestion(IngestionPipeline), - Shared(PhysicalCandidateId), -} - -/// Which ingestion pipeline `I_r` (#1095 §2.5 O3(b)): the deployment -/// refreshes the ingestion-time views of one refresh window in one run, so -/// `asap_cost::execution_demand::computations` charges a `NotMaterialized` -/// ingestion-time node once per refresh window of the views reading it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum IngestionPipeline { - /// The views of this refresh window. - Window(RefreshWindow), - /// An ingestion-time shareable candidate whose window is unbounded or - /// empty: refreshed alone. - Alone(PhysicalCandidateId), -} - -/// The [`IngestionPipeline`] of ingestion-time shareable candidate -/// `shareable`, from its refresh window ([`refresh_windows`]). -pub(crate) fn ingestion_pipeline( - shareable: PhysicalCandidateId, - refresh_windows: &HashMap>, -) -> IngestionPipeline { - match refresh_windows.get(&shareable) { - Some(Some(window)) => IngestionPipeline::Window(*window), - _ => IngestionPipeline::Alone(shareable), - } -} - -/// Each ingestion-time shareable candidate's refresh window, as -/// `asap_cost::execution_demand::computations` computes it on the realized -/// plan ([`PhysicalASAPDAG::refresh_windows`]): on the DAG of the candidates -/// below it, each input read from its first producer candidate. That is -/// exact when the producer candidates of each input read one window, which -/// is debug-asserted ([`producer_window_disagreements`]). -fn refresh_windows( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, -) -> HashMap> { - let views: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|&class| { - links.is_shareable(class) - && physical.candidates[class.0].timing == ExecutionTiming::IngestionTime - }) - .collect(); - let dag = dag_below(physical, links, &views); - debug_assert!( - producer_window_disagreements(links, &dag).is_empty(), - "producer candidates of one input read different windows: {:?}", - producer_window_disagreements(links, &dag) - ); - dag.refresh_windows(views.iter().map(|view| view.0)) - .into_iter() - .map(|(id, window)| (PhysicalCandidateId(id), window)) - .collect() -} - -/// The DAG of `tops` and every candidate below them, without edges: each -/// candidate a node ([`catalog_node`]) whose inputs are its first producer -/// candidates. -fn dag_below( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - tops: &[PhysicalCandidateId], -) -> PhysicalASAPDAG { - let mut nodes = Vec::new(); - let mut seen = HashSet::new(); - let mut stack = tops.to_vec(); - while let Some(class) = stack.pop() { - if seen.insert(class) { - nodes.push(catalog_node(physical, links, class)); - stack.extend(links.producers_per_input[class.0].iter().flatten()); - } - } - let nodes: Vec<&PhysicalASAPDAGNode> = nodes.iter().collect(); - nodes_dag(&nodes) -} - -/// The `(candidate, input)` pairs of `dag`'s candidates -/// ([`dag_below`]) whose producer candidates have different view windows -/// ([`PhysicalASAPDAG::view_windows`]). -fn producer_window_disagreements( - links: &CandidateLinks, - dag: &PhysicalASAPDAG, -) -> Vec<(PhysicalCandidateId, usize)> { - let windows = dag.view_windows(dag.nodes.iter().map(|node| node.id)); - let mut disagreements = Vec::new(); - for node in &dag.nodes { - for (input, producers) in links.producers_per_input[node.id].iter().enumerate() { - let mut read = producers.iter().map(|producer| windows[&producer.0]); - if let Some(first) = read.next() { - if read.any(|window| window != first) { - disagreements.push((PhysicalCandidateId(node.id), input)); - } - } - } - } - disagreements -} - -/// [`producer_window_disagreements`] over every candidate of `physical`. -#[cfg(test)] -pub(crate) fn producer_window_disagreements_of_all( - physical: &PhysicalASAPMemoCandidates, -) -> Vec<(PhysicalCandidateId, usize)> { - let links = CandidateLinks::new(physical); - let all: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .collect(); - producer_window_disagreements(&links, &dag_below(physical, &links, &all)) -} - -/// One execution context and the candidates that may give a result in it. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ExecutionContext { - pub(crate) kind: ContextKind, - /// Whether it runs at query time (a query, or a query-time shareable - /// candidate's inputs): its `PerContext` candidates are computed in it. - pub(crate) at_query_time: bool, - /// The candidates that may give a result slot in it, ascending: each - /// computed in it ([`ExecutionContext::computes`]) or read across its - /// boundary (a shareable candidate, or a candidate of an `I_r`). - pub(crate) options: Vec, -} - -impl ExecutionContext { - /// Whether a candidate of `kind` is computed in this context (otherwise - /// it is read across the boundary). - pub(crate) fn computes(&self, kind: RunKind) -> bool { - match kind { - RunKind::PerContext => self.at_query_time, - RunKind::Ingestion => matches!(self.kind, ContextKind::Ingestion(_)), - RunKind::Shareable => false, - } - } -} - -impl MilpInputs { - /// The context where context `context` reads a candidate of an `I_r`: - /// itself for an `I_r`, `I_{r(m)}` for ingestion-time shareable `m`'s - /// context; `None` for any other context (it reads none). - pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { - match self.contexts[context].kind { - ContextKind::Ingestion(_) => Some(context), - ContextKind::Shared(m) => self.ingestion_context_of_shareable.get(&m).copied(), - ContextKind::Query(_) => None, - } - } -} - -/// [`execution_contexts`]' result (see [`MilpInputs`] for each field). -pub(crate) struct ExecutionContexts { - pub(crate) contexts: Vec, - pub(crate) context_of_shareable: BTreeMap, - pub(crate) ingestion_context_of_shareable: BTreeMap, -} - -/// The execution contexts and their options (module documentation): from -/// each query's output candidates, each shareable candidate's producers -/// and, per ingestion pipeline `I_r`, every candidate of `I_r` an -/// ingestion-time shareable candidate of refresh window `r` -/// (`refresh_windows`) reads across its boundary, expanded through the -/// inputs of the candidates each context computes. Candidates `usable` -/// rejects (no usage) are left out. A query-time `PerContext` candidate -/// that only an ingestion-time context could read is left out too: it -/// cannot run there. -pub(crate) fn execution_contexts( - physical: &PhysicalASAPMemoCandidates, - kind: &[RunKind], - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - refresh_windows: &HashMap>, - usable: impl Fn(PhysicalCandidateId) -> bool, -) -> ExecutionContexts { - let query_count = physical.query_roots.len(); - let output_refs = output_references(&physical.query_roots); - // Expands `seeds` in a context: the options and, through the candidates - // it computes, their producers. - let expand = |context: &ExecutionContext, seeds: Vec| { - let mut seen: BTreeSet = BTreeSet::new(); - let mut stack = seeds; - while let Some(class) = stack.pop() { - if !usable(class) || seen.contains(&class) { - continue; - } - let class_kind = kind[class.0]; - if class_kind == RunKind::PerContext && !context.at_query_time { - continue; - } - seen.insert(class); - if context.computes(class_kind) { - stack.extend( - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - } - seen.into_iter().collect::>() - }; - let mut contexts: Vec = Vec::new(); - for query in 0..query_count { - let mut context = ExecutionContext { - kind: ContextKind::Query(query), - at_query_time: true, - options: Vec::new(), - }; - let seeds = output_refs - .iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .flat_map(|(reference, _)| output_candidates[reference].iter().copied()) - .collect(); - context.options = expand(&context, seeds); - contexts.push(context); - } - // Shareable contexts, discovered from the options, then each `I_r` from - // every candidate of `I_r` read across a boundary; repeated until - // nothing new. - let mut context_of_shareable: BTreeMap = BTreeMap::new(); - let mut shared: Vec = Vec::new(); - // Per `I_r`: its context and its seeds. - let mut ingestion: BTreeMap< - IngestionPipeline, - (ExecutionContext, BTreeSet), - > = BTreeMap::new(); - let mut scanned = 0; - loop { - let mut pending: Vec = Vec::new(); - let mut seeds: Vec<(IngestionPipeline, PhysicalCandidateId)> = Vec::new(); - let unscanned = contexts.iter().chain(shared.iter()).skip(scanned); - for context in unscanned.chain(ingestion.values().map(|(context, _)| context)) { - // Only an ingestion-time shareable candidate reads a candidate - // of an `I_r` (Stage 2's timing rule). - let pipeline = match context.kind { - ContextKind::Shared(m) if !context.at_query_time => { - Some(ingestion_pipeline(m, refresh_windows)) - } - _ => None, - }; - for &class in &context.options { - match kind[class.0] { - RunKind::Shareable => { - if !context_of_shareable.contains_key(&class) && !pending.contains(&class) { - pending.push(class); - } - } - RunKind::Ingestion => { - if let Some(pipeline) = pipeline { - seeds.push((pipeline, class)); - } - } - RunKind::PerContext => {} - } - } - } - scanned = contexts.len() + shared.len(); - let mut grown: BTreeSet = BTreeSet::new(); - for (pipeline, class) in seeds { - let (_, known) = ingestion.entry(pipeline).or_insert_with(|| { - let context = ExecutionContext { - kind: ContextKind::Ingestion(pipeline), - at_query_time: false, - options: Vec::new(), - }; - (context, BTreeSet::new()) - }); - if known.insert(class) { - grown.insert(pipeline); - } - } - if pending.is_empty() && grown.is_empty() { - break; - } - pending.sort_unstable(); - for class in pending { - let at_query_time = physical.candidates[class.0].timing == ExecutionTiming::QueryTime; - let mut context = ExecutionContext { - kind: ContextKind::Shared(class), - at_query_time, - options: Vec::new(), - }; - context.options = expand( - &context, - producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied() - .collect(), - ); - context_of_shareable.insert(class, usize::MAX); - shared.push(context); - } - for pipeline in grown { - let (context, known) = ingestion.get_mut(&pipeline).expect("seeded above"); - context.options = expand(context, known.iter().copied().collect()); - } - } - shared.sort_by_key(|context| context.kind); - let mut context_of_pipeline: BTreeMap = BTreeMap::new(); - for (pipeline, (context, _)) in ingestion { - context_of_pipeline.insert(pipeline, contexts.len()); - contexts.push(context); - } - let mut ingestion_context_of_shareable = BTreeMap::new(); - for context in shared { - if let ContextKind::Shared(class) = context.kind { - context_of_shareable.insert(class, contexts.len()); - if !context.at_query_time { - let pipeline = ingestion_pipeline(class, refresh_windows); - if let Some(&at) = context_of_pipeline.get(&pipeline) { - ingestion_context_of_shareable.insert(class, at); - } - } - } - contexts.push(context); - } - ExecutionContexts { - contexts, - context_of_shareable, - ingestion_context_of_shareable, - } -} - -// --------------------------------------------------------------------------- -// Per-candidate data -// --------------------------------------------------------------------------- - -/// Whether `node` outputs approximate values ([`CandidateData::approximate`]). -fn outputs_approximate_values(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), - .. - } - ) - ) -} - -/// The physical-DAG node of `candidate`, with node id `candidate` -/// and each input's child id its first producer candidate (`usize::MAX` when it -/// has none), as Stage 2's realizer would export it before retention. -fn catalog_node( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - class: PhysicalCandidateId, -) -> PhysicalASAPDAGNode { - let candidate = &physical.candidates[class.0]; - let node = physical.physical_node(class); - let mut input = 0; - let payload = node.operator.map_children(|_| { - let child = links.producers_per_input[class.0] - .get(input) - .and_then(|producers| producers.first()) - .map_or(usize::MAX, |producer| producer.0); - input += 1; - child - }); - PhysicalASAPDAGNode { - id: class.0, - payload, - output_state: ExecutionDataState { - timing: candidate.timing, - primitive: if node.asap().and_then(ASAPOp::produced_state).is_some() { - DataPrimitive::SummaryState - } else { - DataPrimitive::Raw - }, - }, - materialization: candidate.materialization, - output_schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - retention_ms: None, - } -} - -/// A DAG of `nodes` alone, without edges: what -/// [`asap_cost::tumbling_window_retention_ms`] reads (payload children and nodes). -fn nodes_dag(nodes: &[&PhysicalASAPDAGNode]) -> PhysicalASAPDAG { - PhysicalASAPDAG { - nodes: nodes.iter().map(|&node| node.clone()).collect(), - edges: Vec::new(), - roots: Vec::new(), - scalar_roots: Vec::new(), - } -} - -/// The window a scan read through `path` (time shifts, then the reader that -/// ends it) covers, as [`PhysicalASAPDAG::scan_window`] computes it for a -/// plan where `path` is the scan's only reader: the shifts' offsets add up, -/// and a `TimeRange` reads its range, a kept pane chain one pane, any other -/// pane chain its whole window; any other reader reads `Unbounded`. -fn window_read_through(path: &[&PhysicalASAPDAGNode]) -> ScanWindow { - let mut end_offset_ms: i64 = 0; - for node in path { - let width_ms: u128 = match &node.payload { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if shift.at.is_none() => { - end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); - continue; - } - Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) => range.as_millis(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if matches!( - node.materialization, - Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) => - { - u128::from(*tumbling_window_width_ms) - } - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => u128::from(*tumbling_window_width_ms) * u128::from(*tumbling_window_count), - _ => return ScanWindow::Unbounded, - }; - return ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }; - } - ScanWindow::Unbounded -} - -/// The windows query-time scan `scan` may be read at: each reader's -/// windows (one per path through time shifts) and, per execution context, -/// the unions of the window kinds its readers there read (one scan run read -/// by several readers reads the union, `PhysicalASAPDAG::scan_window`): -/// every union of the kinds. -/// A read as an output or as a shareable candidate's own input is -/// `Unbounded`. Distinct; `Unbounded` alone when nothing reads it. -fn scan_window_options( - scan: PhysicalCandidateId, - windows_per_reader: &[(PhysicalCandidateId, Vec)], - contexts: &[ExecutionContext], - kind: &[RunKind], - producers_per_input: &[Vec>], - output_candidates: &[Vec], -) -> Vec { - use crate::milp::cost_terms::window_union; - let mut windows: Vec = Vec::new(); - let add = |window: ScanWindow, windows: &mut Vec| { - if !windows.contains(&window) { - windows.push(window); - } - }; - for (_, read) in windows_per_reader { - for &window in read { - add(window, &mut windows); - } - } - let window_of = |reader: PhysicalCandidateId| { - windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - .map_or(ScanWindow::Unbounded, |(_, read)| { - window_union(read.iter().copied()) - }) - }; - let shareable = kind[scan.0] == RunKind::Shareable; - let mut all_kinds: Vec = Vec::new(); - for context in contexts { - if !context.options.contains(&scan) { - continue; - } - let mut kinds: Vec = Vec::new(); - for &reader in &context.options { - if context.computes(kind[reader.0]) - && producers_per_input[reader.0] - .iter() - .any(|producers| producers.contains(&scan)) - { - let window = window_of(reader); - if !kinds.contains(&window) { - kinds.push(window); - } - } - } - // A shareable candidate reading the scan as its own input reads it - // at its own window; a read as an output reads it whole. - if let ContextKind::Shared(own) = context.kind { - if producers_per_input[own.0] - .iter() - .any(|producers| producers.contains(&scan)) - { - let window = window_of(own); - if !kinds.contains(&window) { - kinds.push(window); - } - } - } - let read_whole = match context.kind { - ContextKind::Query(_) => output_candidates.iter().any(|out| out.contains(&scan)), - ContextKind::Shared(_) | ContextKind::Ingestion(_) => false, - }; - if read_whole && !kinds.contains(&ScanWindow::Unbounded) { - kinds.push(ScanWindow::Unbounded); - } - if shareable { - for window in kinds { - if !all_kinds.contains(&window) { - all_kinds.push(window); - } - } - continue; - } - add_unions(&kinds, &mut windows); - } - if shareable { - add_unions(&all_kinds, &mut windows); - } - if windows.is_empty() { - windows.push(ScanWindow::Unbounded); - } - windows -} - -/// Whether window `outer` contains window `inner`. -fn window_contains(outer: ScanWindow, inner: ScanWindow) -> bool { - match (outer, inner) { - (ScanWindow::Unbounded, _) => true, - (ScanWindow::Within { .. }, ScanWindow::Unbounded) => false, - ( - ScanWindow::Within { - end_offset_ms: outer_end, - width_ms: outer_width, - }, - ScanWindow::Within { - end_offset_ms: inner_end, - width_ms: inner_width, - }, - ) => { - outer_end <= inner_end - && outer_end.saturating_add(outer_width) >= inner_end.saturating_add(inner_width) - } - } -} - -/// Adds to `windows` every union of a nonempty subset of `kinds`. -fn add_unions(kinds: &[ScanWindow], windows: &mut Vec) { - use crate::milp::cost_terms::scan_window_read_at as window_union; - let mut push = |window: ScanWindow| { - if !windows.contains(&window) { - windows.push(window); - } - }; - // Every union of the kinds is enumerated (#1185: no cap); a shift past - // the word size would silently enumerate nothing. - assert!( - kinds.len() < usize::BITS as usize, - "a scan run with {} window kinds has too many unions to enumerate", - kinds.len() - ); - for set in 1usize..(1 << kinds.len()) { - push(window_union( - (0..kinds.len()) - .filter(|bit| set & (1 << bit) != 0) - .map(|bit| kinds[bit]), - )); - } -} - -/// Per class reading the query-time scan `scan`, the windows it may read it -/// at (through time shifts above it), distinct. -fn scan_windows_per_reader( - scan: PhysicalCandidateId, - nodes: &[PhysicalASAPDAGNode], - readers: &[Vec<(PhysicalCandidateId, usize)>], -) -> Vec<(PhysicalCandidateId, Vec)> { - /// Time shifts nested deeper than this are not followed. - const MAX_SHIFTS: usize = 4; - let mut per_reader: Vec<(PhysicalCandidateId, Vec)> = Vec::new(); - let mut paths: Vec> = vec![vec![scan]]; - while let Some(path) = paths.pop() { - let last = *path.last().expect("never empty"); - let follow = path.len() == 1 - || (matches!( - nodes[last.0].payload, - Operator::NonASAP(NonASAPOp::TimeShift { .. }) - ) && path.len() <= MAX_SHIFTS - && !readers[last.0].is_empty()); - if !follow { - // The path ends at a reader: what the scan reads under it. - let through: Vec<_> = path[1..].iter().map(|class| &nodes[class.0]).collect(); - let window = window_read_through(&through); - let Some(&reader) = path.get(1) else { - continue; - }; - match per_reader.iter_mut().find(|(known, _)| *known == reader) { - Some((_, windows)) if windows.contains(&window) => {} - Some((_, windows)) => windows.push(window), - None => per_reader.push((reader, vec![window])), - } - continue; - } - for &(reader, _) in readers[last.0].iter().rev() { - if !path.contains(&reader) { - let mut longer = path.clone(); - longer.push(reader); - paths.push(longer); - } - } - } - per_reader.sort_by_key(|(reader, _)| *reader); - per_reader -} - -/// The panes kept pane chain `chain` may keep, one per query that can reach -/// it (its range-evaluation span), in `reaching` order; its pane count for -/// any other node. -fn retained_pane_options( - chain: PhysicalCandidateId, - nodes: &[PhysicalASAPDAGNode], - first_producer: &[Option], - reaching: &[usize], - entries: &[QueryWorkloadEntry], -) -> Vec { - let node = &nodes[chain.0]; - let kept = matches!( - (&node.payload, node.materialization), - ( - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - Materialization::MatAtIngestion(_) - | Materialization::MatAtQueryKeptAcrossEvaluations(_) - ) - ); - if !kept { - return vec![asap_cost::tumbling_windows_of(node)]; - } - // The chain over the time shifts below it, which set its input offset. - let mut below = Vec::new(); - let mut current = first_producer[chain.0]; - while let Some(producer) = current { - if !matches!( - nodes[producer.0].payload, - Operator::NonASAP(NonASAPOp::TimeShift { .. }) - ) || below.contains(&producer) - { - break; - } - below.push(producer); - current = first_producer[producer.0]; - } - let mut dag_nodes: Vec<&PhysicalASAPDAGNode> = - below.iter().rev().map(|class| &nodes[class.0]).collect(); - dag_nodes.push(node); - let dag = nodes_dag(&dag_nodes); - let mut options = Vec::new(); - let reach = vec![HashSet::from([chain.0])]; - let spans: Vec<_> = match reaching.is_empty() { - true => vec![None], - false => reaching - .iter() - .map(|&query| entries.get(query).and_then(|entry| entry.range_evaluation)) - .collect(), - }; - for range in spans { - let retention = asap_cost::tumbling_window_retention_ms(&dag, &reach, &[range]); - let panes = asap_cost::retained_tumbling_windows(node, &retention); - options.push(panes); - } - options -} - -/// Whether two producer outputs cost their readers the same. -fn same_producer_output(left: &ProducerOutput<'_>, right: &ProducerOutput<'_>) -> bool { - left.output.rows == right.output.rows - && left.output.bytes == right.output.bytes - && left.sources == right.sources - && left.family == right.family - && left.unchanged_columns == right.unchanged_columns - && (std::ptr::eq(left.payload, right.payload) - || left.payload.map_children(|_| 0usize) == right.payload.map_children(|_| 0usize)) - && (std::ptr::eq(left.output_schema, right.output_schema) - || left.output_schema == right.output_schema) -} - -/// `usage` as a reader of `node` sees it. -fn producer_output<'a>(node: &'a PhysicalASAPDAGNode, usage: &NodeUsage<'a>) -> ProducerOutput<'a> { - ProducerOutput { - payload: &node.payload, - output_schema: &node.output_schema, - output: usage.output, - sources: usage.sources.clone(), - family: usage.family, - unchanged_columns: usage.unchanged_columns.clone(), - } -} - -/// Per candidate, every `(reader candidate, input)` that may read it, ascending. -fn readers_of( - producer_candidates_per_input: &[Vec>], -) -> Vec> { - let mut readers = vec![Vec::new(); producer_candidates_per_input.len()]; - for (reader, inputs) in producer_candidates_per_input.iter().enumerate() { - for (input, producers) in inputs.iter().enumerate() { - for producer in producers { - readers[producer.0].push((PhysicalCandidateId(reader), input)); - } - } - } - readers -} - -/// Each candidate's [`CandidateData`] (module documentation, "Usage per execution -/// and its context"). `reaching[k]` are the queries that can reach candidate -/// `k` before pruning. -#[allow(clippy::too_many_arguments)] -fn candidate_data( - physical: &PhysicalASAPMemoCandidates, - links: &CandidateLinks, - entries: &[QueryWorkloadEntry], - reaching: &[Vec], - contexts: &[ExecutionContext], - output_candidates: &[Vec], - data: &DataWorkload, - cost_inputs: &CostInputs, -) -> Vec { - let count = physical.candidates.len(); - let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let nodes: Vec = (0..count) - .map(|class| catalog_node(physical, links, PhysicalCandidateId(class))) - .collect(); - let node_by_id: asap_cost::NodeMap<'_> = nodes.iter().map(|node| (node.id, node)).collect(); - let first_producer: Vec> = links - .producers_per_input - .iter() - .map(|producers_per_input| { - producers_per_input - .first() - .and_then(|producers| producers.first().copied()) - }) - .collect(); - let first_producers_of: HashMap> = links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| { - ( - class, - producers_per_input - .iter() - .filter_map(|producers| producers.first().map(|producer| producer.0)) - .collect(), - ) - }) - .collect(); - let readers = readers_of(&links.producers_per_input); - // Per candidate, the distinct outputs it may give its readers; `None` until - // it is costed. - let mut outputs: Vec>, String>>> = vec![None; count]; - let mut result: Vec = (0..count) - .map(|class| { - let node = physical.physical_node(PhysicalCandidateId(class)); - let operator_node = physical.candidates[class].operator_node; - let accuracy_budget_parts = physical.operator_nodes_of_candidates_for_memo_groups - [operator_node.0] - .accuracy_budget_parts; - CandidateData { - approximate: outputs_approximate_values(node), - accuracy_budget_parts, - usage: Err("not costed".to_string()), - ..CandidateData::default() - } - }) - .collect(); - // Producers first (Kahn's order). Candidate links can close a cycle - // (e.g. a widened scan's `Project` that reads its own memo group): when no candidate - // is ready, the smallest one each of whose inputs has a costed producer - // is costed from those producers alone, and marked - // [`UsageContext::ProducerOnACycle`]. - let producer_sets: Vec> = links - .producers_per_input - .iter() - .map(|producers_per_input| producers_per_input.iter().flatten().map(|p| p.0).collect()) - .collect(); - let mut waiting: Vec = producer_sets.iter().map(BTreeSet::len).collect(); - let mut ready: std::collections::BinaryHeap> = (0..count) - .filter(|&class| waiting[class] == 0) - .map(std::cmp::Reverse) - .collect(); - let mut costed_classes = vec![false; count]; - loop { - let class = match ready.pop() { - Some(std::cmp::Reverse(class)) => class, - None => match (0..count).find(|&class| { - !costed_classes[class] - && links.producers_per_input[class] - .iter() - .all(|producers| producers.iter().any(|p| costed_classes[p.0])) - }) { - Some(class) => class, - None => break, - }, - }; - if std::mem::replace(&mut costed_classes[class], true) { - continue; - } - let mut readers_of_class: BTreeSet = BTreeSet::new(); - for &(reader, _) in &readers[class] { - readers_of_class.insert(reader.0); - } - for reader in readers_of_class { - waiting[reader] -= 1; - if waiting[reader] == 0 && !costed_classes[reader] { - ready.push(std::cmp::Reverse(reader)); - } - } - let class = PhysicalCandidateId(class); - let node = &nodes[class.0]; - let inputs = &links.producers_per_input[class.0]; - // Per input, the distinct outputs of its costed producer candidates. - let mut producer_options: Vec>> = Vec::with_capacity(inputs.len()); - let mut failed = None; - let mut on_a_cycle = false; - // A scan's outputs at windows that do not contain this reader's own - // window are never what it reads (the scan reads at least the - // union of its readers' windows). - let reader = class; - let reads_output = - |result: &[CandidateData], producer: PhysicalCandidateId, at: usize| -> bool { - let windowed = &result[producer.0].usage_per_window; - if windowed.is_empty() { - return true; - } - let Some((_, own)) = result[producer.0] - .windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - else { - return true; - }; - let own = crate::milp::cost_terms::window_union(own.iter().copied()); - // An unbounded read is priced at the other readers' widest - // window (`plan_cost`), whatever it is. - if own == ScanWindow::Unbounded { - return true; - } - windowed - .iter() - .any(|&(window, _, output)| output == at && window_contains(window, own)) - }; - for producers in inputs { - let mut options: Vec> = Vec::new(); - for producer in producers { - match &outputs[producer.0] { - Some(Ok(produced)) => { - for (at, output) in produced.iter().enumerate() { - if !reads_output(&result, *producer, at) { - continue; - } - if !options - .iter() - .any(|known| same_producer_output(known, output)) - { - options.push(output.clone()); - } - } - } - // An uncostable producer is one no plan uses: the - // others still serve this input. - Some(Err(_)) => {} - None => on_a_cycle = true, - } - } - if options.is_empty() { - failed.get_or_insert_with(|| { - format!( - "input {} has no producer with a usage (producers {:?})", - producer_options.len(), - producers.iter().map(|p| p.0).collect::>() - ) - }); - } - producer_options.push(options); - } - if let Some(reason) = failed { - outputs[class.0] = Some(Err(reason.clone())); - result[class.0].usage = Err(reason); - continue; - } - let windows = match (&node.payload, node.output_state.timing) { - (Operator::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - result[class.0].windows_per_reader = - scan_windows_per_reader(class, &nodes, &readers); - scan_window_options( - class, - &result[class.0].windows_per_reader, - contexts, - &links.kind, - &links.producers_per_input, - output_candidates, - ) - } - _ => vec![ScanWindow::Unbounded], - }; - let retained_per_query = - retained_pane_options(class, &nodes, &first_producer, &reaching[class.0], entries); - let mut retained: Vec = Vec::new(); - for &panes in &retained_per_query { - if !retained.contains(&panes) { - retained.push(panes); - } - } - if retained.len() > 1 { - result[class.0].retained_panes_per_query = reaching[class.0] - .iter() - .copied() - .zip(retained_per_query.iter().copied()) - .collect(); - } - let ingestion_window_ms = (node.output_state.timing == ExecutionTiming::IngestionTime) - .then(|| asap_cost::maintained_window_ms(class.0, &node_by_id, &first_producers_of)) - .flatten(); - // Every combination of the options. - let combinations = producer_options - .iter() - .map(Vec::len) - .chain([windows.len(), retained.len()]) - .try_fold(1usize, |total, options| total.checked_mul(options)) - .expect("the number of context combinations fits in usize"); - // Each costed combination: its usage, its output, and its picks - // (the option per input, the window and the retained panes). - let mut costed: Vec<(CandidateUsage, ProducerOutput<'_>, Picks)> = Vec::new(); - let mut error = None; - for index in 0..combinations { - let mut rest = index; - let mut pick = |options: usize| { - let picked = rest % options; - rest /= options; - picked - }; - let retained_pick = pick(retained.len()); - let retained_tumbling_windows = retained[retained_pick]; - let window_pick = pick(windows.len()); - let window = windows[window_pick]; - let mut option_picks: Vec = producer_options - .iter() - .rev() - .map(|options| pick(options.len())) - .collect(); - option_picks.reverse(); - let producers: Vec> = producer_options - .iter() - .zip(&option_picks) - .map(|(options, &picked)| options[picked].clone()) - .collect(); - let usage_at = |per_second: f64| { - asap_cost::node_usage( - node, - &producers, - (1, per_second), - window, - retained_tumbling_windows, - ingestion_window_ms, - data, - cost_inputs, - formula_scale, - ) - }; - match (usage_at(1.0), usage_at(0.0)) { - (Ok(executed), Ok(standing)) => { - let usage = candidate_usage(&executed, &standing, node.output_state.timing); - costed.push(( - usage, - producer_output(node, &executed), - Picks { - options: option_picks, - window: window_pick, - retained: retained_pick, - }, - )); - } - (Err(e), _) | (_, Err(e)) => { - error.get_or_insert(e.to_string()); - } - } - } - // A context `asap-cost` cannot cost is one no plan can use. - let uncostable_context = error.is_some(); - if costed.is_empty() { - let error = error.unwrap_or_else(|| "no context".into()); - outputs[class.0] = Some(Err(error.clone())); - result[class.0].usage = Err(error); - continue; - } - let first = costed[0].0; - let mut depends_on = Vec::new(); - // Its distinct outputs, and the one each costed combination gives. - let mut produced: Vec> = Vec::new(); - let mut output_of_combination = Vec::with_capacity(costed.len()); - for (_, output, _) in &costed { - let at = match produced - .iter() - .position(|known| same_producer_output(known, output)) - { - Some(at) => at, - None => { - produced.push(output.clone()); - produced.len() - 1 - } - }; - output_of_combination.push(at); - } - // What a combination gives its plan: its usage and its output. - let given: Vec<(CandidateUsage, usize)> = costed - .iter() - .zip(&output_of_combination) - .map(|((usage, _, _), &output)| (*usage, output)) - .collect(); - // The inputs whose option changes the usage or the output, and - // whether the window does: two costed combinations differing only - // there give different usages or outputs. - let changes = |differs: &dyn Fn(&Picks, &Picks) -> bool| { - costed.iter().zip(&given).any(|((_, _, picks), gives)| { - costed - .iter() - .zip(&given) - .any(|((_, _, others), other)| differs(picks, others) && gives != other) - }) - }; - let inputs_changing_usage: Vec = (0..producer_options.len()) - .filter(|&input| { - changes(&|a: &Picks, b: &Picks| { - a.window == b.window - && a.retained == b.retained - && a.options[input] != b.options[input] - && (0..a.options.len()) - .all(|other| other == input || a.options[other] == b.options[other]) - }) - }) - .collect(); - let window_changes_usage = changes(&|a: &Picks, b: &Picks| { - a.window != b.window && a.retained == b.retained && a.options == b.options - }); - if !inputs_changing_usage.is_empty() { - // Per changing input, each producer and the options it gives: - // one choice per output of a producer with several. - let options_of_producers: Vec)>> = - inputs_changing_usage - .iter() - .map(|&input| { - let mut choices = Vec::new(); - for &producer in &inputs[input] { - let Some(Ok(produced)) = &outputs[producer.0] else { - continue; - }; - let option_of = |output: &ProducerOutput<'_>| { - producer_options[input] - .iter() - .position(|known| same_producer_output(known, output)) - }; - if produced.len() == 1 { - choices.push(( - ProducerChoice { - class: producer, - output: None, - }, - produced.iter().filter_map(option_of).collect(), - )); - continue; - } - for (at, output) in produced.iter().enumerate() { - if !reads_output(&result, producer, at) { - continue; - } - choices.push(( - ProducerChoice { - class: producer, - output: Some(at), - }, - option_of(output).into_iter().collect(), - )); - } - } - choices - }) - .collect(); - let choices = options_of_producers - .iter() - .try_fold(1usize, |total, producers| { - total.checked_mul(producers.len()) - }) - .expect("the number of producer choices fits in usize"); - for index in 0..choices { - let mut rest = index; - let chosen: Vec<&(ProducerChoice, Vec)> = options_of_producers - .iter() - .map(|producers| { - let picked = &producers[rest % producers.len()]; - rest /= producers.len(); - picked - }) - .collect(); - let matching: Vec<&(CandidateUsage, usize)> = costed - .iter() - .zip(&given) - .filter(|((_, _, picks), _)| { - inputs_changing_usage - .iter() - .zip(&chosen) - .all(|(&input, (_, options))| options.contains(&picks.options[input])) - }) - .map(|(_, gives)| gives) - .collect(); - let costliest = matching.iter().copied().max_by(|a, b| { - let total = |usage: &CandidateUsage| { - usage.cost_per_execution.unwrap_or(0.0) - + usage.standing_cost_per_second.unwrap_or(0.0) - + usage.work_ms_per_execution - }; - total(&a.0).total_cmp(&total(&b.0)) - }); - if matching.iter().any(|gives| Some(*gives) != costliest) { - result[class.0].producer_usage_ambiguous = true; - } - result[class.0].usage_per_producers.push(( - chosen.iter().map(|(choice, _)| *choice).collect(), - costliest.copied(), - )); - } - result[class.0].inputs_changing_usage = inputs_changing_usage; - } - if retained.len() > 1 { - let retained_changes_usage = changes(&|a: &Picks, b: &Picks| { - a.retained != b.retained && a.window == b.window && a.options == b.options - }); - if retained_changes_usage { - for (pick, &panes) in retained.iter().enumerate() { - if let Some(at) = costed - .iter() - .position(|(_, _, picks)| picks.retained == pick) - { - result[class.0] - .usage_per_retained_panes - .push((panes, given[at].0)); - } - } - } else { - result[class.0].retained_panes_per_query.clear(); - } - } - // A query-time scan keeps its usage per window even when every - // costed window gives the same: a window `asap-cost` cannot cost - // (an unbounded read with no `rows_held`) must stay unusable. - let query_time_scan = matches!( - (&node.payload, node.output_state.timing), - ( - Operator::NonASAP(NonASAPOp::Scan { .. }), - ExecutionTiming::QueryTime - ) - ); - if window_changes_usage || query_time_scan { - for (pick, window) in windows.iter().enumerate() { - if let Some(at) = costed.iter().position(|(_, _, picks)| picks.window == pick) { - let (usage, output) = given[at]; - result[class.0] - .usage_per_window - .push((*window, usage, output)); - } - } - } - if given.iter().any(|gives| gives.0 != first) { - if producer_options.iter().any(|options| options.len() > 1) { - depends_on.push(UsageContext::ProducerOutput); - } - if windows.len() > 1 { - depends_on.push(UsageContext::ScanWindowOfReader); - } - if retained.len() > 1 { - depends_on.push(UsageContext::PaneRetentionOfReachingQueries); - } - } - if on_a_cycle { - depends_on.push(UsageContext::ProducerOnACycle); - } - if uncostable_context { - depends_on.push(UsageContext::UncostableContext); - } - result[class.0].output_count = produced.len(); - outputs[class.0] = Some(Ok(produced)); - result[class.0].usage = Ok(first); - result[class.0].usage_depends_on = depends_on; - } - for (class, costed) in costed_classes.iter().enumerate() { - if !costed { - result[class].usage = Err("its producer links form a cycle".into()); - } - } - result -} - -/// One costed combination of a class's context options, by index: the -/// option per input, the window and the retained panes. -#[derive(Debug, Clone, PartialEq, Eq)] -struct Picks { - options: Vec, - window: usize, - retained: usize, -} - -/// The numbers of one execution (`executed`, at one execution per second) -/// and of none (`standing`) of a node at `timing`. -fn candidate_usage( - executed: &NodeUsage<'_>, - standing: &NodeUsage<'_>, - timing: ExecutionTiming, -) -> CandidateUsage { - let standing_cost = standing.cost.cost_per_second; - CandidateUsage { - cpu_ops_per_execution: executed.cpu_ops_per_second, - scanned_bytes_per_execution: executed.scanned_bytes_per_second, - work_ms_per_execution: executed.work_ms_per_execution, - own_latency_ms: match timing { - ExecutionTiming::QueryTime => executed.work_ms_per_execution, - ExecutionTiming::IngestionTime => 0.0, - }, - cost_per_execution: executed - .cost - .cost_per_second - .zip(standing_cost) - .map(|(total, standing)| total - standing), - standing_cost_per_second: standing_cost, - average_memory_bytes: executed.average_memory_bytes, - peak_memory_bytes: executed.peak_memory_bytes, - disk_bytes: executed.disk_bytes, - held_bytes: executed.held.map_or(0, |(bytes, _, _)| bytes), - raw_data_kept_bytes: executed.raw_scan.map_or(0, |(_, bytes)| bytes), - maintained_window_bytes: executed.maintained.unwrap_or(0), - output_rows: executed.output.rows, - output_bytes: executed.output.bytes, - } -} - -// --------------------------------------------------------------------------- -// Dominance -// --------------------------------------------------------------------------- - -/// What one candidate is compared on (module documentation, "Dominance"). -#[derive(Debug, Clone)] -pub(crate) struct DominanceProfile<'a> { - /// Must be equal for one candidate to dominate another. - pub(crate) same: DominanceKey<'a>, - pub(crate) guarantee: Option<&'a AccuracyGuarantee>, -} - -/// The parts of a [`DominanceProfile`] that must be equal. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct DominanceKey<'a> { - pub(crate) group: MemoGroupId, - pub(crate) timing: ExecutionTiming, - pub(crate) materialization: Materialization, - pub(crate) shareable: bool, - pub(crate) producer_candidates_per_input: &'a [Vec], - pub(crate) readers: Vec<(PhysicalCandidateId, usize)>, - pub(crate) output_references: Vec, - pub(crate) reader_constraint_ids: Vec, - pub(crate) may_keep_across_evaluations_per_sole_reader_query: &'a [bool], - pub(crate) output_schema: &'a Schema, - pub(crate) operator_kind: OperatorKindForReaders, -} - -/// The parts of a node's operator its readers' costs read: its kind, and a -/// pane chain's panes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum OperatorKindForReaders { - SummaryTumblingWindowChain { - tumbling_window_width_ms: u64, - tumbling_window_count: u32, - }, - Other(std::mem::Discriminant, &'static str), -} - -/// Each candidate's [`DominanceProfile`]. `output_candidates` are `Out(r)` before -/// pruning. -fn dominance_profiles<'a>( - physical: &'a PhysicalASAPMemoCandidates, - links: &'a CandidateLinks, - output_candidates: &[Vec], -) -> Vec> { - let constraint_ids = reader_constraint_ids_per_candidate(physical); - let mut readers = readers_of(&links.producers_per_input); - for list in &mut readers { - list.sort_unstable(); - } - let mut output_refs_of = vec![Vec::new(); physical.candidates.len()]; - for (reference, out) in output_candidates.iter().enumerate() { - for class in out { - output_refs_of[class.0].push(reference); - } - } - links - .producers_per_input - .iter() - .enumerate() - .map(|(class, producers_per_input)| { - let copy = PhysicalCandidateId(class); - let candidate = &physical.candidates[copy.0]; - let node = physical.physical_node(copy); - DominanceProfile { - same: DominanceKey { - group: physical.memo_group_of(copy), - timing: candidate.timing, - materialization: candidate.materialization, - shareable: links.is_shareable(copy), - producer_candidates_per_input: producers_per_input, - readers: std::mem::take(&mut readers[class]), - output_references: std::mem::take(&mut output_refs_of[class]), - reader_constraint_ids: constraint_ids[copy.0].clone(), - may_keep_across_evaluations_per_sole_reader_query: &candidate - .may_keep_across_evaluations_per_sole_reader_query, - output_schema: &node.schema, - operator_kind: match &node.operator { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => OperatorKindForReaders::SummaryTumblingWindowChain { - tumbling_window_width_ms: *tumbling_window_width_ms, - tumbling_window_count: *tumbling_window_count, - }, - operator => OperatorKindForReaders::Other( - std::mem::discriminant(operator), - node.operator.kind_name(), - ), - }, - }, - guarantee: node.guarantee.as_ref(), - } - }) - .collect() -} - -/// Whether `stronger` is at least as strong as `weaker`: equal, or `stronger` -/// exact, or both in one metric and scope with constant bounds and failure -/// probabilities no larger. Anything else (an unknown statistic, different -/// metrics) is not comparable. -pub(crate) fn guarantee_at_least_as_strong( - stronger: Option<&AccuracyGuarantee>, - weaker: Option<&AccuracyGuarantee>, -) -> bool { - match (stronger, weaker) { - (None, None) => true, - (Some(stronger), Some(weaker)) => { - if stronger.spec == weaker.spec || stronger.is_exact() { - return true; - } - stronger.spec.accuracy_metric == weaker.spec.accuracy_metric - && stronger.spec.accuracy_scope == weaker.spec.accuracy_scope - && matches!( - (stronger.spec.error.evaluate(), weaker.spec.error.evaluate()), - (Some(s), Some(w)) if s <= w - ) - && matches!( - ( - stronger.spec.failure_probability.evaluate(), - weaker.spec.failure_probability.evaluate() - ), - (Some(s), Some(w)) if s <= w - ) - } - _ => false, - } -} - -/// Every number of `usage` that a candidate it dominates must not beat, in a -/// fixed order. -fn compared_numbers(usage: &CandidateUsage) -> [f64; 12] { - [ - usage.cpu_ops_per_execution, - usage.scanned_bytes_per_execution, - usage.work_ms_per_execution, - usage.own_latency_ms, - usage.average_memory_bytes, - usage.peak_memory_bytes as f64, - usage.disk_bytes as f64, - usage.held_bytes as f64, - usage.raw_data_kept_bytes as f64, - usage.maintained_window_bytes as f64, - usage.output_rows as f64, - usage.output_bytes as f64, - ] -} - -/// Whether candidate `by` dominates candidate `candidate` (module documentation). -pub(crate) fn dominates( - by: (&DominanceProfile<'_>, &CandidateData), - class: (&DominanceProfile<'_>, &CandidateData), -) -> bool { - let ((by_profile, by_data), (profile, data)) = (by, class); - let (Ok(by_usage), Ok(usage)) = (&by_data.usage, &data.usage) else { - return false; - }; - by_profile.same == profile.same - && by_data.usage_depends_on.is_empty() - && data.usage_depends_on.is_empty() - && by_data.approximate == data.approximate - && by_data.accuracy_budget_parts == data.accuracy_budget_parts - && guarantee_at_least_as_strong(by_profile.guarantee, profile.guarantee) - && compared_numbers(by_usage) - .iter() - .zip(compared_numbers(usage)) - .all(|(by, class)| *by <= class) -} - -/// The candidates another candidate dominates (module documentation), each with -/// the smallest dominating candidate that is kept. Among candidates that dominate -/// each other, the smallest id is kept. -pub(crate) fn prune_dominated_candidates( - profiles: &[DominanceProfile<'_>], - candidate_data: &[CandidateData], -) -> Vec { - // Only candidates with equal keys can dominate each other. - let mut buckets: Vec> = Vec::new(); - let mut bucket_of_key: HashMap<(MemoGroupId, ExecutionTiming, Materialization), Vec> = - HashMap::new(); - for (class, profile) in profiles.iter().enumerate() { - let candidates = bucket_of_key - .entry(( - profile.same.group, - profile.same.timing, - profile.same.materialization, - )) - .or_default(); - match candidates - .iter() - .find(|&&bucket| profiles[buckets[bucket][0]].same == profile.same) - { - Some(&bucket) => buckets[bucket].push(class), - None => { - candidates.push(buckets.len()); - buckets.push(vec![class]); - } - } - } - let mut pruned = Vec::new(); - for bucket in buckets.iter().filter(|bucket| bucket.len() > 1) { - for &class in bucket { - let dominated_by = bucket.iter().copied().find(|&other| { - other != class - && dominates( - (&profiles[other], &candidate_data[other]), - (&profiles[class], &candidate_data[class]), - ) - && (other < class - || !dominates( - (&profiles[class], &candidate_data[class]), - (&profiles[other], &candidate_data[other]), - )) - }); - if let Some(other) = dominated_by { - pruned.push(DominatedCandidate { - class: PhysicalCandidateId(class), - dominated_by: PhysicalCandidateId(other), - reason: format!( - "candidate {other} has the same memo group, producers, readers and \ - (approx, parts), a guarantee at least as strong, and no larger \ - usage, latency, output or held state" - ), - }); - } - } - } - pruned.sort_by_key(|pruned| pruned.class); - pruned -} - -mod accuracy_positions; - -use accuracy_positions::AccuracyPositions; - -#[cfg(test)] -mod tests; diff --git a/crates/plan-selection/src/groups/accuracy_positions.rs b/crates/plan-selection/src/groups/accuracy_positions.rs deleted file mode 100644 index fa81af61f..000000000 --- a/crates/plan-selection/src/groups/accuracy_positions.rs +++ /dev/null @@ -1,777 +0,0 @@ -//! Accuracy weights and pair exclusions per query (module documentation of -//! [`super`], "Accuracy weights"), each from `asap-accuracy`'s composition -//! and Stage 3's own check ([`guarantee_check`]) of an output built over -//! the candidates. -//! -//! For query `q` and a set `S` of approximate candidates (one candidate, or -//! a pair), the outputs `q` can have with exactly `S` approximate are built -//! from the candidates' operator nodes: a candidate in `S` is its own node; -//! a candidate with no member of `S` below it stands in as its node with an -//! exact guarantee; any other candidate is its node rebuilt over each -//! combination of its producers' outputs, with its guarantee composed again -//! ([`composed_guarantee`]), as the realizer recomposes a selected plan. -//! Outputs over which readers compose the same guarantee are kept once -//! ([`reader_key`]). Each output is then checked as -//! the selected plan is: [`guarantee_check`] with the deployment's model. -//! `S` is usable for `q` only when every output passes: a candidate that -//! fails through one reader is unusable through every reader (the largest -//! weight counts, as before). The candidates of one operator node (other -//! timings or materializations) build the same node, so they are checked -//! once, together. What a check builds is kept for the later checks -//! ([`BuiltBefore`]). -use std::cell::OnceCell; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use asap_accuracy::propagation::{composed_guarantee, root_guarantee, with_guarantees}; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::{OperatorNode, OperatorResultKind, QueryRoot}; -use asap_types::stages::{output_references, PhysicalASAPMemoCandidates, PhysicalCandidateId}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -use super::{AccuracyWeight, CandidateData, UnusableReason}; -use crate::checks::guarantee_check; -use crate::report::RejectionCause; -use crate::SelectionModels; - -/// What the accuracy rows (#1095 §2.0 row 9) are built from. -#[derive(Debug, Clone, Default)] -pub(crate) struct AccuracyPositions { - /// Per query: `w(q, k)` for each approximate candidate `k` whose values - /// reach `q`'s output. - pub(crate) weights: Vec>, - /// Per query: the pairs `(k1, k2)`, `k1 < k2`, each usable alone, whose - /// outputs miss `q`'s target when both are approximate: - /// `a[q, k1] + a[q, k2] ≤ 1`. - pub(crate) excluded_pairs: Vec>, -} - -/// The outcome of one check of a query with a set of approximate candidates. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Outcome { - /// Every output is exact: no value of the set reaches the output. - Exact, - /// Every output meets the target. - Meets, - Fails(UnusableReason), -} - -/// [`AccuracyPositions`] of `physical`'s queries over the candidate links -/// left (`producer_candidates_per_input`, `output_candidates`). -pub(crate) fn accuracy_positions( - physical: &PhysicalASAPMemoCandidates, - candidate_data: &[CandidateData], - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: &SelectionModels<'_>, -) -> AccuracyPositions { - let below = approximate_below(candidate_data, producer_candidates_per_input); - let output_refs = output_references(&physical.query_roots); - let exact_stand_ins: Vec>> = (0..physical.candidates.len()) - .map(|_| OnceCell::new()) - .collect(); - let mut built_before = BuiltBefore::default(); - let mut positions = AccuracyPositions::default(); - for (query, root) in physical.query_roots.iter().enumerate() { - let references: Vec = (0..output_refs.len()) - .filter(|&reference| output_refs[reference].0 == query) - .collect(); - let reachable: BTreeSet = references - .iter() - .flat_map(|&reference| &output_candidates[reference]) - .flat_map(|class| below[class.0].iter().copied()) - .collect(); - let Some(entry) = entries.get(query) else { - positions.weights.push(BTreeMap::new()); - positions.excluded_pairs.push(Vec::new()); - continue; - }; - let checker = Checker { - physical, - exact_stand_ins: &exact_stand_ins, - producer_candidates_per_input, - output_candidates, - below: &below, - query, - root, - references: &references, - entry, - data, - models, - }; - // Candidates of one operator node (other timings or - // materializations) build the same node: checked once, together. - let mut groups: BTreeMap> = BTreeMap::new(); - for &candidate in &reachable { - groups - .entry(physical.candidates[candidate].operator_node.0) - .or_default() - .push(candidate); - } - let mut weights = BTreeMap::new(); - let mut usable: Vec<&[usize]> = Vec::new(); - for group in groups.values() { - let weight = match checker.check(group, &mut built_before) { - Outcome::Exact => continue, - Outcome::Meets => { - usable.push(group); - AccuracyWeight::One - } - Outcome::Fails(reason) => AccuracyWeight::Unusable(reason), - }; - for &candidate in group { - weights.insert(PhysicalCandidateId(candidate), weight); - } - } - let usable_candidates: Vec = usable - .iter() - .flat_map(|group| group.iter().copied()) - .collect(); - let group_of: HashMap = usable - .iter() - .enumerate() - .flat_map(|(index, group)| group.iter().map(move |&candidate| (candidate, index))) - .collect(); - let group_pairs: BTreeSet<(usize, usize)> = pairs_that_meet( - &usable_candidates, - &references, - physical, - producer_candidates_per_input, - output_candidates, - &below, - ) - .into_iter() - .map(|(first, second)| (group_of[&first], group_of[&second])) - .filter(|(first, second)| first != second) - .map(|(first, second)| (first.min(second), first.max(second))) - // A pair with a member sized for fewer than two shares is never - // read together: the shares row already excludes it. - .filter(|&(first, second)| { - [first, second] - .iter() - .all(|&group| candidate_data[usable[group][0]].accuracy_budget_parts >= 2) - }) - .collect(); - let mut excluded = Vec::new(); - for (first, second) in group_pairs { - let both: Vec = usable[first] - .iter() - .chain(usable[second]) - .copied() - .collect(); - if let Outcome::Fails(_) = checker.check(&both, &mut built_before) { - for &one in usable[first] { - for &other in usable[second] { - excluded.push(( - PhysicalCandidateId(one.min(other)), - PhysicalCandidateId(one.max(other)), - )); - } - } - } - } - excluded.sort_unstable(); - positions.weights.push(weights); - positions.excluded_pairs.push(excluded); - } - positions -} - -/// Per candidate, the approximate candidates at or below it through the -/// links, ascending. A fixpoint, so cycles of candidate links are covered. -fn approximate_below( - candidate_data: &[CandidateData], - producer_candidates_per_input: &[Vec>], -) -> Vec> { - let mut below: Vec> = candidate_data - .iter() - .enumerate() - .map(|(class, data)| match data.approximate { - true => BTreeSet::from([class]), - false => BTreeSet::new(), - }) - .collect(); - loop { - let mut changed = false; - for class in 0..below.len() { - let mut grown = below[class].clone(); - for producer in producer_candidates_per_input[class].iter().flatten() { - grown.extend(below[producer.0].iter().copied()); - } - if grown.len() != below[class].len() { - below[class] = grown; - changed = true; - } - } - if !changed { - return below; - } - } -} - -/// The pairs of `usable` (ascending, `first < second`) one plan of the -/// query can read together where their values meet: under two different -/// inputs of one candidate, under two output references of a scalar root, -/// or one below the other. -fn pairs_that_meet( - usable: &[usize], - references: &[usize], - physical: &PhysicalASAPMemoCandidates, - producer_candidates_per_input: &[Vec>], - output_candidates: &[Vec], - below: &[BTreeSet], -) -> BTreeSet<(usize, usize)> { - let usable: BTreeSet = usable.iter().copied().collect(); - let mut pairs = BTreeSet::new(); - let mut add_across = |sides: &[BTreeSet]| { - for (index, left) in sides.iter().enumerate() { - for right in &sides[index + 1..] { - for &a in left { - for &b in right { - if a != b { - pairs.insert((a.min(b), a.max(b))); - } - } - } - } - } - }; - let side = |producers: &[PhysicalCandidateId]| -> BTreeSet { - producers - .iter() - .flat_map(|producer| below[producer.0].iter().copied()) - .filter(|candidate| usable.contains(candidate)) - .collect() - }; - for producers_per_input in producer_candidates_per_input - .iter() - .take(physical.candidates.len()) - { - if producers_per_input.len() < 2 { - continue; - } - let sides: Vec> = producers_per_input - .iter() - .map(|producers| side(producers)) - .collect(); - add_across(&sides); - } - let sides: Vec> = references - .iter() - .map(|&reference| side(&output_candidates[reference])) - .collect(); - add_across(&sides); - for &upper in &usable { - for &lower in &below[upper] { - if lower != upper && usable.contains(&lower) { - pairs.insert((lower.min(upper), lower.max(upper))); - } - } - } - pairs -} - -/// One query's checks. -struct Checker<'a> { - physical: &'a PhysicalASAPMemoCandidates, - /// Per candidate, its node with an exact guarantee, built once. - exact_stand_ins: &'a [OnceCell>], - producer_candidates_per_input: &'a [Vec>], - output_candidates: &'a [Vec], - below: &'a [BTreeSet], - query: usize, - root: &'a QueryRoot, - references: &'a [usize], - entry: &'a QueryWorkloadEntry, - data: &'a DataWorkload, - models: &'a SelectionModels<'a>, -} - -/// The outputs one candidate is built as, or why they are not checked. -type Built = Result>, UnusableReason>; - -/// What the checks built, kept across checks and queries. A candidate's -/// outputs depend only on the members of the approximate set at or below -/// it, so a pair check rebuilds only the candidates above both members -/// and reuses the rest from the single checks. Outputs built across a -/// cycle of links depend on where the walk entered the cycle and are not -/// kept. -#[derive(Default)] -struct BuiltBefore { - /// Per (candidate, approximate members at or below it, ascending): its - /// outputs. - outputs: HashMap<(usize, Vec), Built>, - /// The [`reader_key`]s of every node built so far. - keys: Keys, -} - -#[cfg(test)] -thread_local! { - /// Set by a test to check with no kept builds: every check builds - /// from scratch ([`BuiltBefore`] cleared) and builds every output - /// candidate (no exact stand-in skipped). - pub(crate) static FRESH_BUILDS: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -/// One check's walk. -#[derive(Default)] -struct Walk { - /// Per candidate built in this check. - built: HashMap, - /// The candidates being built. - visiting: Vec, - /// How many reads of a candidate still being built the walk has made. - cycle_reads: usize, -} - -/// A candidate's outputs in one check, and whether they were built across -/// a cycle of links (then they are not kept in [`BuiltBefore`]). -struct BuiltInWalk { - outputs: Built, - across_cycle: bool, -} - -impl Checker<'_> { - /// Checks the query's outputs with exactly `approximate` approximate. - /// `before` holds what earlier checks built, for every query. - fn check(&self, approximate: &[usize], before: &mut BuiltBefore) -> Outcome { - #[cfg(test)] - let fresh = FRESH_BUILDS.with(std::cell::Cell::get); - #[cfg(not(test))] - let fresh = false; - if fresh { - *before = BuiltBefore::default(); - } - let mut walk = Walk::default(); - let mut per_reference: Vec>> = Vec::new(); - for &reference in self.references { - let mut outputs: Vec> = Vec::new(); - // Exact stand-ins that are not summary states are the same - // for readers ([`reader_key`]): only the first is kept. - let mut exact_value_stand_in = false; - for &class in &self.output_candidates[reference] { - let reaches = approximate - .iter() - .any(|member| self.below[class.0].contains(member)); - if !fresh - && !reaches - && self.physical.physical_node(class).result_kind != OperatorResultKind::State - && std::mem::replace(&mut exact_value_stand_in, true) - { - continue; - } - match self.built(class.0, approximate, &mut walk, before) { - Ok(built) => outputs.extend(built), - Err(reason) => return Outcome::Fails(reason), - } - } - per_reference.push(distinct_for_readers(outputs, &mut before.keys)); - } - let roots = self.roots(&per_reference); - let mut outcome = Outcome::Exact; - for root in roots { - if root_guarantee(&root).is_some_and(|g| g.is_exact()) { - continue; - } - match guarantee_check( - self.query + 1, - &root, - &self.entry.requirements.accuracy, - self.data, - self.models, - ) { - Ok(_) => outcome = Outcome::Meets, - Err(failure) => { - return Outcome::Fails(match failure.cause { - RejectionCause::NoAccuracyGuarantee => UnusableReason::NoAccuracyGuarantee, - _ => UnusableReason::AccuracyTargetMissed, - }) - } - } - } - outcome - } - - /// The query roots over each combination of `per_reference`'s outputs. - fn roots(&self, per_reference: &[Vec>]) -> Vec { - combinations(per_reference) - .into_iter() - .map(|chosen| match self.root { - QueryRoot::Operator(_) => QueryRoot::Operator(chosen[0].clone()), - QueryRoot::Scalar(expr) => { - let mut next = chosen.into_iter(); - QueryRoot::Scalar( - expr.map_operator_refs(&mut |_| { - next.next().expect("one output per reference") - }), - ) - } - }) - .collect() - } - - /// `class`'s node with an exact guarantee, built once. - fn exact_stand_in(&self, class: usize) -> Rc { - self.exact_stand_ins[class] - .get_or_init(|| { - let node = self.physical.physical_node(PhysicalCandidateId(class)); - Rc::new( - (**node) - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact("an exact stand-in"))), - ) - }) - .clone() - } - - /// The outputs `class` is built as with `approximate` approximate - /// (module documentation). `walk.visiting` holds the candidates being - /// built, so a cycle of links builds nothing. - fn built( - &self, - class: usize, - approximate: &[usize], - walk: &mut Walk, - before: &mut BuiltBefore, - ) -> Built { - if let Some(done) = walk.built.get(&class) { - walk.cycle_reads += usize::from(done.across_cycle); - return done.outputs.clone(); - } - if walk.visiting.contains(&class) { - walk.cycle_reads += 1; - return Ok(Vec::new()); - } - let reaches = |candidate: usize| { - approximate - .iter() - .any(|member| self.below[candidate].contains(member)) - }; - if !reaches(class) { - return Ok(vec![self.exact_stand_in(class)]); - } - let mut members: Vec = approximate - .iter() - .copied() - .filter(|member| self.below[class].contains(member)) - .collect(); - members.sort_unstable(); - let key = (class, members); - if let Some(done) = before.outputs.get(&key) { - walk.built.insert( - class, - BuiltInWalk { - outputs: done.clone(), - across_cycle: false, - }, - ); - return done.clone(); - } - let cycle_reads = walk.cycle_reads; - let node = self.physical.physical_node(PhysicalCandidateId(class)); - let producers = &self.producer_candidates_per_input[class]; - let below_inputs = producers - .iter() - .flatten() - .any(|producer| reaches(producer.0)); - let built = if !below_inputs { - // `class` is in `approximate`, over exact inputs: its node with - // every guarantee below it composed, as the realizer does. - with_guarantees(node, &mut HashMap::new()) - .map(|node| vec![node]) - .map_err(|_| UnusableReason::NotChecked) - } else { - walk.visiting.push(class); - let mut per_input = Vec::with_capacity(producers.len()); - let mut failed = None; - for input in producers { - let mut outputs = Vec::new(); - for producer in input.iter().filter(|producer| reaches(producer.0)) { - match self.built(producer.0, approximate, walk, before) { - Ok(built) => outputs.extend(built), - Err(reason) => failed = Some(reason), - } - } - // One exact stand-in for the producers with no member below: - // one with no approximate candidate below, when there is one. - let exact = input - .iter() - .filter(|producer| !reaches(producer.0)) - .min_by_key(|producer| !self.below[producer.0].is_empty()); - if let Some(exact) = exact { - outputs.push(self.exact_stand_in(exact.0)); - } - per_input.push(distinct_for_readers(outputs, &mut before.keys)); - } - walk.visiting.pop(); - match failed { - Some(reason) => Err(reason), - None => rebuilt(node, &per_input, &mut before.keys), - } - }; - let across_cycle = walk.cycle_reads != cycle_reads; - if !across_cycle { - before.outputs.insert(key, built.clone()); - } - walk.built.insert( - class, - BuiltInWalk { - outputs: built.clone(), - across_cycle, - }, - ); - built - } -} - -/// `node` rebuilt over each combination of `per_input`'s outputs, with its -/// timing and materialization and its guarantee composed again; outputs the -/// same for their readers kept once. -fn rebuilt(node: &Rc, per_input: &[Vec>], keys: &mut Keys) -> Built { - let combinations = combinations(per_input); - let mut outputs = Vec::with_capacity(combinations.len()); - for chosen in combinations { - let mut next = chosen.into_iter(); - let rebuilt = node - .with_new_children(|_| next.next().expect("one output per input")) - .map_err(|_| UnusableReason::NotChecked)?; - let guarantee = composed_guarantee(&rebuilt); - outputs.push(Rc::new( - rebuilt - .with_guarantee(guarantee) - .with_timing(node.timing) - .with_materialization(node.materialization), - )); - } - Ok(distinct_for_readers(outputs, keys)) -} - -/// Every choice of one entry per list, in order. One empty list gives none. -fn combinations(lists: &[Vec]) -> Vec> { - let mut combinations: Vec> = vec![Vec::new()]; - for list in lists { - combinations = combinations - .into_iter() - .flat_map(|prefix| { - list.iter().map(move |item| { - let mut next = prefix.clone(); - next.push(item.clone()); - next - }) - }) - .collect(); - } - combinations -} - -/// `outputs` with outputs the same for their readers kept once (the -/// first): equal [`reader_key`]s. -fn distinct_for_readers(outputs: Vec>, keys: &mut Keys) -> Vec> { - let mut seen = HashSet::new(); - outputs - .into_iter() - .filter(|output| seen.insert(reader_key(output, keys))) - .collect() -} - -/// Per node (by address): its [`reader_key`], and whether binding -/// statistics leaves its guarantee as it is; the node is held so its -/// address is not reused during the check. -type Keys = HashMap<*const OperatorNode, (u64, bool, Rc)>; - -/// A key equal for outputs over which readers compose the same guarantee: -/// an exact node as one key (an exact summary state, a stand-in, by its -/// address); a node with a guarantee (not a summary state) by its bound and failure -/// probability (binding keeps a node's guarantee and ignores its inputs; -/// provenance is not read); a node with none that binding cannot give one -/// as one key; any other node (a summary state, whose readers read its -/// inputs' guarantees, or a node with none that binding may still give -/// one) by its operator kind and its inputs' keys. -fn reader_key(node: &Rc, keys: &mut Keys) -> u64 { - key_and_settled(node, keys).0 -} - -/// [`reader_key`], and whether binding statistics leaves `node`'s -/// guarantee as it is: a guarantee with no unknown statistic, or none over -/// inputs that are all settled. -fn key_and_settled(node: &Rc, keys: &mut Keys) -> (u64, bool) { - if let Some(&(key, settled, _)) = keys.get(&Rc::as_ptr(node)) { - return (key, settled); - } - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - // Exact whatever its inputs (an exact stand-in among them). - if node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - { - "exact".hash(&mut hasher); - if node.result_kind == OperatorResultKind::State { - Rc::as_ptr(node).hash(&mut hasher); - } - let known = (hasher.finish(), true); - keys.insert(Rc::as_ptr(node), (known.0, known.1, node.clone())); - return known; - } - let spec = node.guarantee.as_ref().map(|guarantee| &guarantee.spec); - format!("{spec:?}").hash(&mut hasher); - let state = node.result_kind == OperatorResultKind::State; - let known = match &node.guarantee { - Some(guarantee) if !state => ( - hasher.finish(), - guarantee.spec.error.evaluate().is_some() - && guarantee.spec.failure_probability.evaluate().is_some(), - ), - _ => { - let children: Vec<(u64, bool)> = node - .children() - .into_iter() - .map(|child| key_and_settled(child, keys)) - .collect(); - let settled = node.guarantee.is_none() && children.iter().all(|(_, settled)| *settled); - if !(settled && !state) { - node.operator.kind_name().hash(&mut hasher); - state.hash(&mut hasher); - for (key, _) in &children { - key.hash(&mut hasher); - } - } - (hasher.finish(), settled) - } - }; - keys.insert(Rc::as_ptr(node), (known.0, known.1, node.clone())); - known -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::groups::tests::{candidate, catalog, reader_of, relative, root, values_node}; - use asap_logical_asap_optimization::test_support::entry_with; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - /// The depth of each output of `built`. - fn depths(built: &Built) -> Vec { - fn depth(node: &OperatorNode) -> usize { - 1 + node - .children() - .into_iter() - .map(|child| depth(child)) - .max() - .unwrap_or(0) - } - built - .as_ref() - .expect("built") - .iter() - .map(|node| depth(node)) - .collect() - } - - // A two-candidate link cycle above an approximate member: `A` reads - // the leaf or `B`; `B` reads `A` or a second leaf. Walking from the - // top enters the cycle at `A`, so `B` is built there with `A` cut off - // (a cycle read), over the second leaf only. Such outputs are not - // kept: a later walk that starts at `B` builds it in full, over `A`, - // as with no kept builds, for one member and for a pair. - #[test] - fn outputs_built_across_a_cycle_are_not_kept() { - let approximate_leaf = |value: f64, error: f64| { - Rc::new( - (*values_node(value)) - .clone() - .with_guarantee(Some(relative(error, 0.01))), - ) - }; - let leaf = approximate_leaf(1.0, 0.01); - let other_leaf = approximate_leaf(2.0, 0.02); - let a = reader_of(&leaf); - let b = reader_of(&leaf); - let top = reader_of(&a); - let physical = catalog( - vec![(leaf, 0), (a, 1), (b, 2), (top, 3), (other_leaf, 4)], - vec![ - candidate(0, &[], false), - candidate(1, &[&[0, 2]], false), - candidate(2, &[&[1, 4]], false), - candidate(3, &[&[1]], true), - candidate(4, &[], false), - ], - vec![root(3)], - ); - let producers: Vec>> = physical - .candidates - .iter() - .map(|candidate| { - candidate - .producer_candidates_per_input - .iter() - .map(|producers| producers.iter().copied().collect()) - .collect() - }) - .collect(); - // Leaves 0 and 4 stand for approximate candidates. - let below: Vec> = vec![ - BTreeSet::from([0]), - BTreeSet::from([0, 4]), - BTreeSet::from([0, 4]), - BTreeSet::from([0, 4]), - BTreeSet::from([4]), - ]; - let exact_stand_ins: Vec>> = (0..physical.candidates.len()) - .map(|_| OnceCell::new()) - .collect(); - let entry = entry_with( - AccuracyTarget::max_error(0.01).failure_probability(0.01), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))), - ); - let data = DataWorkload::default(); - let cost_inputs = asap_cost::CostInputs::default(); - let models = SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs: &cost_inputs, - }; - let output_candidates = vec![vec![PhysicalCandidateId(3)]]; - let checker = Checker { - physical: &physical, - exact_stand_ins: &exact_stand_ins, - producer_candidates_per_input: &producers, - output_candidates: &output_candidates, - below: &below, - query: 0, - root: &physical.query_roots[0], - references: &[0], - entry: &entry, - data: &data, - models: &models, - }; - for approximate in [&[0][..], &[0, 4]] { - let mut kept = BuiltBefore::default(); - // From the top: `B` is built with `A` cut off, over the second - // leaf alone (depth 2). - let mut walk = Walk::default(); - checker - .built(3, approximate, &mut walk, &mut kept) - .expect("built"); - assert!(walk.cycle_reads > 0); - assert!(walk.built[&2].across_cycle); - assert_eq!(depths(&walk.built[&2].outputs)[0], 2, "{approximate:?}"); - // From `B`: in full, its first output reads `A` over the leaf - // (depth 3), with kept builds as with none. - let from_b = checker.built(2, approximate, &mut Walk::default(), &mut kept); - let fresh = checker.built( - 2, - approximate, - &mut Walk::default(), - &mut BuiltBefore::default(), - ); - assert_eq!(depths(&from_b), depths(&fresh), "{approximate:?}"); - assert_eq!(depths(&fresh)[0], 3, "{approximate:?}"); - } - } -} diff --git a/crates/plan-selection/src/groups/tests.rs b/crates/plan-selection/src/groups/tests.rs deleted file mode 100644 index f9c2ce4ee..000000000 --- a/crates/plan-selection/src/groups/tests.rs +++ /dev/null @@ -1,1085 +0,0 @@ -use std::rc::Rc; - -use super::*; -use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; -use asap_types::ir::scalar::expr_ir::ScalarValue; -use asap_types::ir::schema::SketchAlgorithm; -use asap_types::ir::schema::{DataType, Field}; -use asap_types::ir::{QueryRoot, ScalarExpr}; -use asap_types::physical::Medium; -use asap_types::stages::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalCandidate, ProducerCandidates, -}; - -// --------------------------------------------------------------------------- -// Hand-built catalogs -// --------------------------------------------------------------------------- - -/// A one-row `VALUES` node of `value`. -pub(super) fn values_node(value: f64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(value))]], - schema: Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None), - })) - .unwrap() -} - -/// A `VALUES` node of `rows` one-column rows. -fn rows_node(rows: usize) -> Rc { - let schema = Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0))]; rows], - schema, - })) - .unwrap() -} - -/// A stateless one-input reader of `child`. -pub(super) fn reader_of(child: &Rc) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Dedup { - cols: Vec::new(), - child: Rc::clone(child), - })) - .unwrap() -} - -/// A catalog of `nodes`, node `i` in memo group `nodes[i].1` (each node its -/// own candidate of that group, all implementation roots), with -/// `candidates` and one query root per entry of `roots`. -pub(super) fn catalog( - nodes: Vec<(Rc, u32)>, - candidates: Vec, - roots: Vec>, -) -> PhysicalASAPMemoCandidates { - let mut index_in_group: HashMap = HashMap::new(); - PhysicalASAPMemoCandidates::new( - nodes - .into_iter() - .map(|(physical_node, group)| { - let index = index_in_group.entry(group).or_default(); - *index += 1; - OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group), - index: *index - 1, - }], - is_implementation_root: true, - accuracy_budget_parts: 1, - physical_node, - } - }) - .collect(), - candidates, - roots, - Vec::new(), - ) -} - -/// One candidate of operator node `node` at query time, not materialized, -/// reading `producers` per input, that may produce a query's output when -/// `output`. -pub(super) fn candidate(node: usize, producers: &[&[usize]], output: bool) -> PhysicalCandidate { - PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(node), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - producer_candidates_per_input: producers - .iter() - .map(|input| { - input - .iter() - .map(|&id| PhysicalCandidateId(id)) - .collect::() - }) - .collect(), - may_produce_query_output: output, - may_keep_across_evaluations_per_sole_reader_query: Vec::new(), - } -} - -pub(super) fn root(group: u32) -> QueryRoot { - QueryRoot::Operator(MemoGroupId(group)) -} - -fn prepared(physical: &PhysicalASAPMemoCandidates) -> MilpInputs { - prepare_milp_inputs( - physical, - &[], - &DataWorkload::default(), - &CostInputs::default(), - ) -} - -fn id(candidate: usize) -> PhysicalCandidateId { - PhysicalCandidateId(candidate) -} - -// --------------------------------------------------------------------------- -// Dominance -// --------------------------------------------------------------------------- - -fn usage() -> CandidateUsage { - CandidateUsage { - cpu_ops_per_execution: 10.0, - scanned_bytes_per_execution: 10.0, - work_ms_per_execution: 10.0, - own_latency_ms: 10.0, - cost_per_execution: Some(1.0), - standing_cost_per_second: Some(1.0), - average_memory_bytes: 10.0, - peak_memory_bytes: 10, - disk_bytes: 10, - held_bytes: 10, - raw_data_kept_bytes: 10, - maintained_window_bytes: 10, - output_rows: 10, - output_bytes: 10, - } -} - -fn data_with(usage: CandidateUsage) -> CandidateData { - CandidateData { - approximate: true, - accuracy_budget_parts: 1, - usage: Ok(usage), - ..CandidateData::default() - } -} - -pub(super) fn relative(error: f64, failure_probability: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: error }, - ProbabilityExpr::Constant { - value: failure_probability, - }, - Vec::new(), - ) -} - -// Candidate dominance needs every condition: a candidate with equal keys, a -// guarantee at least as strong and no larger number dominates; changing any one condition stops it. -#[test] -fn dominance_needs_every_condition() { - let producers = vec![vec![PhysicalCandidateId(7)]]; - let other_producers = vec![vec![PhysicalCandidateId(8)]]; - let keep = vec![false]; - let other_keep = vec![true]; - let schema = rows_node(1).schema.clone(); - let other_schema = Schema::lifted(vec![Field::plain("v", DataType::Int64, false)], None); - let strong = relative(0.01, 0.01); - let weak = relative(0.02, 0.01); - let values_kind = - OperatorKindForReaders::Other(std::mem::discriminant(&rows_node(1).operator), "Values"); - let key = DominanceKey { - group: MemoGroupId(3), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - shareable: false, - producer_candidates_per_input: &producers, - readers: vec![(PhysicalCandidateId(9), 0)], - output_references: vec![0], - reader_constraint_ids: vec![1], - may_keep_across_evaluations_per_sole_reader_query: &keep, - output_schema: &schema, - operator_kind: values_kind, - }; - let profile = |guarantee| DominanceProfile { - same: key.clone(), - guarantee, - }; - let better = profile(Some(&strong)); - let worse = profile(Some(&weak)); - let mut worse_usage = usage(); - worse_usage.cpu_ops_per_execution = 20.0; - let (better_data, worse_data) = (data_with(usage()), data_with(worse_usage)); - assert!(dominates((&better, &better_data), (&worse, &worse_data))); - assert!(!dominates((&worse, &worse_data), (&better, &better_data))); - - // Each key field. - let changed_keys = [ - ( - "group", - DominanceKey { - group: MemoGroupId(4), - ..key.clone() - }, - ), - ( - "timing", - DominanceKey { - timing: ExecutionTiming::IngestionTime, - ..key.clone() - }, - ), - ( - "materialization", - DominanceKey { - materialization: Materialization::MatAtQuery(Medium::Memory), - ..key.clone() - }, - ), - ( - "shareable", - DominanceKey { - shareable: true, - ..key.clone() - }, - ), - ( - "producers", - DominanceKey { - producer_candidates_per_input: &other_producers, - ..key.clone() - }, - ), - ( - "readers", - DominanceKey { - readers: vec![(PhysicalCandidateId(10), 0)], - ..key.clone() - }, - ), - ( - "outputs", - DominanceKey { - output_references: vec![1], - ..key.clone() - }, - ), - ( - "reader constraints", - DominanceKey { - reader_constraint_ids: vec![2], - ..key.clone() - }, - ), - ( - "keep flags", - DominanceKey { - may_keep_across_evaluations_per_sole_reader_query: &other_keep, - ..key.clone() - }, - ), - ( - "schema", - DominanceKey { - output_schema: &other_schema, - ..key.clone() - }, - ), - ( - "operator kind", - DominanceKey { - operator_kind: OperatorKindForReaders::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, - }, - ..key.clone() - }, - ), - ]; - for (name, changed) in changed_keys { - let by = DominanceProfile { - same: changed, - ..better.clone() - }; - assert!( - !dominates((&by, &better_data), (&worse, &worse_data)), - "{name}" - ); - } - // (approx, parts), guarantee. - let mut data = better_data.clone(); - data.approximate = false; - assert!( - !dominates((&better, &data), (&worse, &worse_data)), - "approx" - ); - let mut data = better_data.clone(); - data.accuracy_budget_parts = 2; - assert!(!dominates((&better, &data), (&worse, &worse_data)), "parts"); - let by = profile(Some(&weak)); - let class = profile(Some(&strong)); - assert!( - !dominates((&by, &better_data), (&class, &worse_data)), - "guarantee" - ); - // Each number. - for number in 0..compared_numbers(&usage()).len() { - let mut larger = usage(); - match number { - 0 => larger.cpu_ops_per_execution = 30.0, - 1 => larger.scanned_bytes_per_execution = 30.0, - 2 => larger.work_ms_per_execution = 30.0, - 3 => larger.own_latency_ms = 30.0, - 4 => larger.average_memory_bytes = 30.0, - 5 => larger.peak_memory_bytes = 30, - 6 => larger.disk_bytes = 30, - 7 => larger.held_bytes = 30, - 8 => larger.raw_data_kept_bytes = 30, - 9 => larger.maintained_window_bytes = 30, - 10 => larger.output_rows = 30, - _ => larger.output_bytes = 30, - } - assert!( - !dominates((&better, &data_with(larger)), (&worse, &worse_data)), - "number {number}" - ); - } - // A plan-dependent or uncosted usage is never compared. - let mut data = better_data.clone(); - data.usage_depends_on = vec![UsageContext::ProducerOutput]; - assert!( - !dominates((&better, &data), (&worse, &worse_data)), - "context" - ); - let mut data = better_data.clone(); - data.usage = Err("uncosted".into()); - assert!(!dominates((&better, &data), (&worse, &worse_data)), "error"); -} - -// A guarantee is at least as strong when equal, exact, or no larger in one -// metric and scope; different metrics and unknown bounds do not compare. -#[test] -fn guarantee_strength_compares_constant_bounds_in_one_metric() { - let strong = relative(0.01, 0.01); - let weak = relative(0.02, 0.02); - let exact = AccuracyGuarantee::exact("exact"); - let rank = AccuracyGuarantee::new( - AccuracyMetric::Rank, - BoundExpr::Constant { value: 0.001 }, - ProbabilityExpr::Zero, - Vec::new(), - ); - assert!(guarantee_at_least_as_strong(Some(&strong), Some(&weak))); - assert!(!guarantee_at_least_as_strong(Some(&weak), Some(&strong))); - assert!(guarantee_at_least_as_strong(Some(&exact), Some(&weak))); - assert!(guarantee_at_least_as_strong(None, None)); - assert!(!guarantee_at_least_as_strong(Some(&strong), None)); - assert!(!guarantee_at_least_as_strong(Some(&rank), Some(&weak))); -} - -// Among dominated candidates the dominating one is reported; of two equal -// candidates the smaller id is kept; a candidate better in one number and worse in -// another is kept. -#[test] -fn pruning_keeps_the_smallest_equal_class_and_incomparable_ones() { - let producers = vec![vec![PhysicalCandidateId(9)]]; - let keep = Vec::new(); - let schema = rows_node(1).schema.clone(); - let key = DominanceKey { - group: MemoGroupId(0), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - shareable: false, - producer_candidates_per_input: &producers, - readers: Vec::new(), - output_references: vec![0], - reader_constraint_ids: Vec::new(), - may_keep_across_evaluations_per_sole_reader_query: &keep, - output_schema: &schema, - operator_kind: OperatorKindForReaders::Other( - std::mem::discriminant(&rows_node(1).operator), - "Values", - ), - }; - let profiles = vec![ - DominanceProfile { - same: key.clone(), - guarantee: None, - }; - 4 - ]; - let mut worse = usage(); - worse.output_rows = 20; - let mut incomparable = usage(); - incomparable.cpu_ops_per_execution = 5.0; - incomparable.held_bytes = 20; - let data = vec![ - data_with(usage()), - data_with(usage()), - data_with(worse), - data_with(incomparable), - ]; - let pruned = prune_dominated_candidates(&profiles, &data); - let summary: Vec<_> = pruned - .iter() - .map(|pruned| (pruned.class.0, pruned.dominated_by.0)) - .collect(); - assert_eq!(summary, vec![(1, 0), (2, 0)]); -} - -// Through `prepare_milp_inputs`: of two deduplications in one memo group -// over one leaf, read by one reader and costing the same, the later one is -// deleted with its links, and the reader reads the other. -#[test] -fn a_dominated_class_is_deleted_with_its_links() { - let schema = Schema::lifted( - vec![ - Field::plain("a", DataType::Float64, false), - Field::plain("b", DataType::Float64, false), - ], - None, - ); - let leaf = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0)); 2]; 4], - schema, - })) - .unwrap(); - let dedup = |cols: Vec| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Dedup { - cols, - child: Rc::clone(&leaf), - })) - .unwrap() - }; - // Two equal nodes: equal schemas and costs. - let (first, second) = (dedup(vec![0]), dedup(vec![0])); - let reader = reader_of(&first); - let physical = catalog( - vec![(leaf, 0), (first, 1), (second, 1), (reader, 2)], - vec![ - candidate(0, &[], false), - candidate(1, &[&[0]], false), - candidate(2, &[&[0]], false), - candidate(3, &[&[1, 2]], true), - ], - vec![root(2)], - ); - let inputs = prepared(&physical); - let (leaf, kept, deleted, top) = (id(0), id(1), id(2), id(3)); - assert_eq!( - inputs.candidate_data[kept.0].usage, - inputs.candidate_data[deleted.0].usage - ); - assert_eq!( - inputs - .pruned_candidates - .iter() - .map(|pruned| (pruned.class, pruned.dominated_by)) - .collect::>(), - vec![(deleted, kept)] - ); - assert_eq!( - inputs.producer_candidates_per_input[top.0], - vec![vec![kept]] - ); - assert!(inputs.producer_candidates_per_input[deleted.0].is_empty()); - assert_eq!(inputs.contexts[0].options, vec![leaf, kept, top]); -} - -// --------------------------------------------------------------------------- -// Execution contexts -// --------------------------------------------------------------------------- - -/// A leaf, a materialized (shareable) middle, and two queries each reading -/// the middle through its own stateless top; a third query reads only its -/// own leaf. Candidate `i` runs operator node `i`. -fn shared_middle() -> PhysicalASAPMemoCandidates { - let leaf = values_node(1.0); - let middle = reader_of(&leaf); - let top_0 = reader_of(&middle); - let top_1 = reader_of(&middle); - let lone = values_node(2.0); - let candidates = vec![ - candidate(0, &[], false), - PhysicalCandidate { - materialization: Materialization::MatAtQuery(Medium::Memory), - ..candidate(1, &[&[0]], false) - }, - candidate(2, &[&[1]], true), - candidate(3, &[&[1]], true), - candidate(4, &[], true), - ]; - catalog( - vec![(leaf, 0), (middle, 1), (top_0, 2), (top_1, 3), (lone, 4)], - candidates, - vec![root(2), root(3), root(4)], - ) -} - -// Each query context holds its own top and the shared middle as a read -// across its boundary; the middle's own context holds the leaf below it; -// no `I_r` is read, so there is none; a query that cannot reach the middle holds only its -// own candidate. -#[test] -fn execution_contexts_on_a_hand_built_catalog() { - let physical = shared_middle(); - let inputs = prepared(&physical); - let (leaf, middle, top_0, top_1, lone) = (id(0), id(1), id(2), id(3), id(4)); - assert!(CandidateLinks::new(&physical).is_shareable(middle)); - assert!(inputs.pruned_candidates.is_empty()); - let contexts: Vec<(ContextKind, Vec)> = inputs - .contexts - .iter() - .map(|context| (context.kind, context.options.clone())) - .collect(); - assert_eq!( - contexts, - vec![ - (ContextKind::Query(0), vec![middle, top_0]), - (ContextKind::Query(1), vec![middle, top_1]), - (ContextKind::Query(2), vec![lone]), - (ContextKind::Shared(middle), vec![leaf]), - ] - ); - assert_eq!(inputs.context_of_shareable, BTreeMap::from([(middle, 3)])); - assert!(inputs.contexts[3].computes(RunKind::PerContext)); - assert!(!inputs.contexts[3].computes(RunKind::Shareable)); - assert_eq!( - inputs.output_candidates, - vec![vec![top_0], vec![top_1], vec![lone]] - ); -} - -/// Stage 2's output for `queries` and what Stage 3 also reads. -struct ThroughStages1And2 { - physical: PhysicalASAPMemoCandidates, - entries: Vec, - data: DataWorkload, -} - -/// `queries` (PromQL over metric `m`, each predictable and repeating every -/// minute with a 1 % error target, over metric `m` with 100 series) -/// through Stage 1 and Stage 2 as the facade runs them, over the complete pane-width domain, with raw data and -/// query-time results retained. -fn through_stages_1_and_2(queries: &[&str]) -> ThroughStages1And2 { - use asap_logical_asap_optimization::test_support::lower_promql; - let roots: Vec = queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query, target()))) - .collect(); - through_stages_1_and_2_of(roots) -} - -/// The accuracy target of [`through_stages_1_and_2`]'s queries. -fn target() -> asap_types::ir::properties::AccuracyTarget { - asap_types::ir::properties::AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -/// [`through_stages_1_and_2`] for query roots `roots`. -fn through_stages_1_and_2_of(roots: Vec) -> ThroughStages1And2 { - use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; - use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; - use asap_logical_asap_optimization::test_support::entry_with; - use asap_types::workload::{ - DurationMs, Predictability, QueryRecurrence, RepeatedDemand, RepetitionInterval, - }; - let target = target(); - let every_minute = - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(60_000))); - let entries: Vec<_> = roots - .iter() - .map(|_| QueryWorkloadEntry { - predictability: Predictability::Predictable { known_at: None }, - ..entry_with(target.clone(), every_minute.clone()) - }) - .collect(); - let mut data = DataWorkload::sampled_every(["m".to_string()], DurationMs(1_000)); - let source = data.sources.get_mut("m").expect("m"); - source.series_count = Some(100); - source.values_non_negative = true; - let targets = vec![target.clone(); roots.len()]; - let pass1 = generate_per_computation_logical_memo_candidates( - asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - &targets, - &asap_accuracy::BuiltinAccuracyModel, - &asap_accuracy::NoAccuracyEvidence, - &data, - None, - ) - .unwrap(); - let logical = generate_logical_asap_memo_candidates(pass1, &entries).unwrap(); - let physical = asap_physical_asap_optimization::generate_physical_asap_memo_candidates( - &logical, &entries, &data, true, true, - ) - .unwrap(); - // Stage 2 copies each node's `accuracy_budget_parts` from its memo - // candidates, the largest for a factored node. - for node in &physical.operator_nodes_of_candidates_for_memo_groups { - let largest = node - .candidates_for_memo_group - .iter() - .map(|id| logical.memo.candidates[id.group.0 as usize][id.index].accuracy_budget_parts) - .max(); - assert_eq!(Some(node.accuracy_budget_parts), largest); - } - ThroughStages1And2 { - physical, - entries, - data, - } -} - -// Through the real Stages 1 and 2: two quantile queries over 1m and 5m -// windows of one metric can share a pane chain. Both queries reach it, it -// is an option of both query contexts with a context of its own, and every -// context computes only what runs there. -// No quotient: every approximate candidate weighs 1. -#[test] -fn two_queries_share_a_pane_chain_through_the_real_pipeline() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.9, m[15m])", - "quantile_over_time(0.9, m[1h])", - ]); - let physical = &stages.physical; - let links = CandidateLinks::new(physical); - let inputs = prepare_milp_inputs( - physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let shared_chains: Vec = (0..physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|candidate| { - matches!( - physical.physical_node(*candidate).asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) - ) && links.is_shareable(*candidate) - && inputs.contexts[0].options.contains(candidate) - && inputs.contexts[1].options.contains(candidate) - }) - .collect(); - assert!(!shared_chains.is_empty()); - for &chain in &shared_chains { - let context = inputs.context_of_shareable[&chain]; - assert_eq!(inputs.contexts[context].kind, ContextKind::Shared(chain)); - } - // Every context computes only candidates of its own kind and reads the - // others across its boundary. - for context in &inputs.contexts { - for &option in &context.options { - let kind = inputs.kind[option.0]; - assert!( - context.computes(kind) - || kind == RunKind::Shareable - || (kind == RunKind::Ingestion - && !matches!(context.kind, ContextKind::Ingestion(_))) - ); - } - } - for weights in &inputs.accuracy_weights { - assert!(!weights.is_empty()); - assert!( - weights - .values() - .all(|&weight| weight == AccuracyWeight::One), - "{weights:?}" - ); - } -} - -// --------------------------------------------------------------------------- -// Accuracy weights -// --------------------------------------------------------------------------- - -/// The sketch algorithm and `parts` of each approximate candidate of -/// `physical`, with its weight for query 0 (`None`: no weight). -fn sketch_weights( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, -) -> Vec<(SketchAlgorithm, usize, Option)> { - (0..physical.candidates.len()) - .filter(|&class| inputs.candidate_data[class].approximate) - .map(|class| { - let id = PhysicalCandidateId(class); - let algorithm = match physical.physical_node(id).asap() { - Some( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => *kind.algorithm(), - _ => unreachable!("an approximate candidate is a sketch"), - }; - ( - algorithm, - inputs.candidate_data[class].accuracy_budget_parts, - inputs.accuracy_weights[0].get(&id).copied(), - ) - }) - .collect() -} - -// `q0.5 / q0.9` through the real stages, weights from the composition: a -// KLL rank error has no rule through a quotient, so Pass 1 prunes every KLL -// candidate (#1129); a DDSketch relative -// error composes, but a lone denominator sized for the whole target -// (`parts = 1`) misses it (`ε/(1 − ε) > ε`) while one sized for two parts -// meets it. -#[test] -fn ratio_weights_come_from_the_composition() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - ]); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let weights = sketch_weights(&stages.physical, &inputs); - let of = |algorithm: SketchAlgorithm, parts: usize| -> BTreeSet> { - weights - .iter() - .filter(|(a, p, _)| *a == algorithm && *p == parts) - .map(|(_, _, weight)| *weight) - .collect() - }; - let missed = Some(AccuracyWeight::Unusable( - UnusableReason::AccuracyTargetMissed, - )); - // Pass 1 prunes every KLL: its rank error composes no guarantee - // through the quotient. - for parts in [1, 2] { - assert_eq!( - of(SketchAlgorithm::Kll, parts), - BTreeSet::new(), - "{weights:?}" - ); - } - // At one part: numerators meet the target, denominators miss it. - assert_eq!( - of(SketchAlgorithm::DDSketch, 1), - BTreeSet::from([Some(AccuracyWeight::One), missed]), - "{weights:?}" - ); - assert_eq!( - of(SketchAlgorithm::DDSketch, 2), - BTreeSet::from([Some(AccuracyWeight::One)]), - "{weights:?}" - ); - // Two DDSketches at two parts compose: no pair is excluded. - assert!(inputs.accuracy_excluded_pairs[0].is_empty()); -} - -// RangeHeavy T8+T5's shape: `q0.9 / q0.5` with both operands also queried -// alone. One DDSketch pane chain sized for two shares of the budget serves -// both quotient operands and both lone quantiles, as the old Stage 3 -// engine planned it: the check accepts the quotient over it (`(α + α)/(1 − α) ≤ ε` at -// `α = 1 − (1 + ε)^{−1/2}`), so it weighs 1 for every query. A chain sized -// for the whole budget (`α = ε`) never serves the quotient's denominator. -#[test] -fn one_pane_chain_serves_a_ratio_and_its_operands() { - let stages = through_stages_1_and_2(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - "quantile_over_time(0.5, m[5m])", - ]); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - let alpha = |class: usize| match stages - .physical - .physical_node(PhysicalCandidateId(class)) - .asap() - { - Some(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => match kind.params() { - asap_types::ir::schema::SketchParams::DDSketch { alpha } => Some(*alpha), - _ => None, - }, - _ => None, - }; - let weight = |query: usize, class: usize| { - inputs.accuracy_weights[query] - .get(&PhysicalCandidateId(class)) - .copied() - }; - let two_shares = 1.0 - 1.01f64.powf(-0.5); - let shared: Vec = (0..stages.physical.candidates.len()) - .filter(|&class| alpha(class).is_some()) - .filter(|&class| (0..3).all(|query| weight(query, class) == Some(AccuracyWeight::One))) - .collect(); - assert!(!shared.is_empty()); - for &class in &shared { - assert!( - (alpha(class).unwrap() - two_shares).abs() < 1e-9, - "{:?}", - alpha(class) - ); - } - // A chain sized for the whole budget weighs 1 for the ratio only where - // its values are the numerator's alone, so it never also serves both - // lone quantiles. - for class in (0..stages.physical.candidates.len()).filter(|&class| alpha(class) == Some(0.01)) { - assert!( - !(0..3).all(|query| weight(query, class) == Some(AccuracyWeight::One)), - "{class}" - ); - } -} - -// The checks keep what they build for later checks and skip repeated -// exact stand-ins: on RangeHeavy T8+T5's shape (a quotient and both -// operands alone), on a quotient of three quantiles (where pairs are -// excluded) and on a ratio with one approximate operand, the weights and -// excluded pairs equal those with no kept builds and no skipped stand-in. -#[test] -fn kept_builds_give_the_weights_of_fresh_builds() { - kept_builds_match_fresh_builds(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m])", - "quantile_over_time(0.9, m[5m])", - "quantile_over_time(0.5, m[5m])", - ]); - let three = kept_builds_match_fresh_builds(&[ - "quantile_over_time(0.9, m[5m]) / quantile_over_time(0.5, m[5m]) / quantile_over_time(0.1, m[5m])", - ]); - assert!(!three.accuracy_excluded_pairs[0].is_empty()); - // A scalar ratio of two output references whose approximate values sit - // under one operand only: the other reference's output candidates - // reach no member, and its first exact stand-in must still be built, - // or the ratio has no root to check and every approximate candidate - // would weigh nothing. - use asap_logical_asap_optimization::test_support::lower_promql; - use asap_types::ir::scalar::{ArithmeticOpKind, ExprSemantics}; - let operand = |query: &str| { - Box::new(ScalarExpr::PromqlScalarFromVector(lower_promql( - query, - target(), - ))) - }; - let ratio = QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Div, - left: operand("avg_over_time(m[5m])"), - right: operand("quantile_over_time(0.5, m[5m])"), - semantics: ExprSemantics::Promql, - }); - let ratio = kept_builds_match_fresh_builds_of(vec![ratio]); - assert!(!ratio.accuracy_weights[0].is_empty()); -} - -/// Asserts that `queries`' accuracy weights and excluded pairs are the -/// same with kept builds and with none ([`accuracy_positions::FRESH_BUILDS`]); -/// returns the former. -fn kept_builds_match_fresh_builds(queries: &[&str]) -> MilpInputs { - use asap_logical_asap_optimization::test_support::lower_promql; - kept_builds_match_fresh_builds_of( - queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query, target()))) - .collect(), - ) -} - -/// [`kept_builds_match_fresh_builds`] for query roots `roots`. -fn kept_builds_match_fresh_builds_of(roots: Vec) -> MilpInputs { - let stages = through_stages_1_and_2_of(roots); - let prepare = |fresh: bool| { - accuracy_positions::FRESH_BUILDS.with(|cell| cell.set(fresh)); - let inputs = prepare_milp_inputs( - &stages.physical, - &stages.entries, - &stages.data, - &CostInputs::default(), - ); - accuracy_positions::FRESH_BUILDS.with(|cell| cell.set(false)); - inputs - }; - let (kept, fresh) = (prepare(false), prepare(true)); - assert!(kept - .accuracy_weights - .iter() - .any(|weights| !weights.is_empty())); - assert_eq!(kept.accuracy_weights, fresh.accuracy_weights); - assert_eq!(kept.accuracy_excluded_pairs, fresh.accuracy_excluded_pairs); - kept -} - -// --------------------------------------------------------------------------- -// Ingestion pipelines per refresh window -// --------------------------------------------------------------------------- - -/// Ingestion-time views of metric `m`, each read by its own query: -/// - candidates 2 and 3: a KLL and a DDSketch over `TimeRange(10 s)` (one -/// refresh window, (10 s, 0)); -/// - 5: a KLL over `TimeRange(60 s)`, window (60 s, 0); -/// - 8: a KLL over `TimeRange(10 s)` over `TimeShift(5 s)`, window -/// (10 s, 5 s), through two `NotMaterialized` ingestion-time nodes; -/// - 9: a KLL over the whole scan, an unbounded window. -/// -/// Candidates 0, 1, 4, 6 and 7 (the scan, ranges and shift) are -/// `NotMaterialized` at ingestion time; 10–14 are the query-time readers. -fn ingestion_views_of_three_windows_and_an_unbounded_one() -> PhysicalASAPMemoCandidates { - use asap_types::ir::operator::{Reduction, Source, TimeShift}; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{GroupingStrategy, SketchKind, SketchParams, SummaryUpdate}; - use asap_types::ir::TimeRangeKind; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap(); - let range = |seconds: u64, child: &Rc| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(seconds), - kind: TimeRangeKind::Range, - child: Rc::clone(child), - })) - .unwrap() - }; - let sketch = |algorithm: SketchAlgorithm, child: &Rc| { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() - }; - let range_10s = range(10, &scan); - let range_60s = range(60, &scan); - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 5_000, - at: None, - }, - child: Rc::clone(&scan), - })) - .unwrap(); - let range_10s_shifted = range(10, &shifted); - let views = [ - sketch(SketchAlgorithm::Kll, &range_10s), - sketch(SketchAlgorithm::DDSketch, &range_10s), - sketch(SketchAlgorithm::Kll, &range_60s), - sketch(SketchAlgorithm::Kll, &range_10s_shifted), - sketch(SketchAlgorithm::Kll, &scan), - ]; - let mut nodes = vec![ - (scan, 0), - (range_10s, 1), - (views[0].clone(), 2), - (views[1].clone(), 3), - (range_60s, 4), - (views[2].clone(), 5), - (shifted, 6), - (range_10s_shifted, 7), - (views[3].clone(), 8), - (views[4].clone(), 9), - ]; - let at_ingestion = |node: usize, producers: &[&[usize]], view: bool| PhysicalCandidate { - timing: ExecutionTiming::IngestionTime, - materialization: if view { - Materialization::MatAtIngestion(Medium::Memory) - } else { - Materialization::NotMaterialized - }, - ..candidate(node, producers, false) - }; - let mut candidates = vec![ - at_ingestion(0, &[], false), - at_ingestion(1, &[&[0]], false), - at_ingestion(2, &[&[1]], true), - at_ingestion(3, &[&[1]], true), - at_ingestion(4, &[&[0]], false), - at_ingestion(5, &[&[4]], true), - at_ingestion(6, &[&[0]], false), - at_ingestion(7, &[&[6]], false), - at_ingestion(8, &[&[7]], true), - at_ingestion(9, &[&[0]], true), - ]; - let mut roots = Vec::new(); - for (index, view) in [2, 3, 5, 8, 9].into_iter().enumerate() { - let group = 10 + index as u32; - nodes.push((reader_of(&nodes[view].0), group)); - candidates.push(candidate(10 + index, &[&[view]], true)); - roots.push(root(group)); - } - catalog(nodes, candidates, roots) -} - -// One `I_r` per refresh window: the two views over `TimeRange(10 s)` share -// one; the 60 s view and the shifted 10 s view (window (10 s, 5 s), through -// a `NotMaterialized` shift and range) have their own; the unbounded view -// gets an `Alone` one. Each `I_r` holds the scan and the stateless nodes -// below its views, and each view's context reads across into its `I_r`. -#[test] -fn execution_contexts_have_one_ingestion_pipeline_per_refresh_window() { - let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); - let links = CandidateLinks::new(&physical); - let alive = vec![true; physical.candidates.len()]; - let windows = refresh_windows(&physical, &links); - let window = |width_ms, phase_ms| RefreshWindow { width_ms, phase_ms }; - assert_eq!( - windows, - HashMap::from([ - (id(2), Some(window(10_000, 0))), - (id(3), Some(window(10_000, 0))), - (id(5), Some(window(60_000, 0))), - (id(8), Some(window(10_000, 5_000))), - (id(9), None), - ]) - ); - let contexts = execution_contexts( - &physical, - &links.kind, - &producer_candidates_left(&links, &alive), - &output_candidates(&physical, &alive), - &windows, - |_| true, - ); - let ingestion: Vec<(ContextKind, Vec)> = contexts.contexts[5..9] - .iter() - .map(|context| (context.kind, context.options.clone())) - .collect(); - let pipeline = |width_ms, phase_ms| { - ContextKind::Ingestion(IngestionPipeline::Window(window(width_ms, phase_ms))) - }; - assert_eq!( - ingestion, - vec![ - (pipeline(10_000, 0), vec![id(0), id(1)]), - (pipeline(10_000, 5_000), vec![id(0), id(6), id(7)]), - (pipeline(60_000, 0), vec![id(0), id(4)]), - ( - ContextKind::Ingestion(IngestionPipeline::Alone(id(9))), - vec![id(0)] - ), - ] - ); - assert_eq!( - contexts.ingestion_context_of_shareable, - BTreeMap::from([(id(2), 5), (id(3), 5), (id(5), 7), (id(8), 6), (id(9), 8)]) - ); - let shared = contexts.context_of_shareable[&id(8)]; - assert_eq!(contexts.contexts[shared].options, vec![id(7)]); -} - -// Every producer candidate of every input reads one view window, the -// assumption behind computing a refresh window from first producers. -#[test] -fn producers_of_an_input_read_one_view_window() { - let physical = ingestion_views_of_three_windows_and_an_unbounded_one(); - assert!(producer_window_disagreements_of_all(&physical).is_empty()); -} diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 77100f7df..a4168acc7 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -1,5858 +1,27 @@ -//! #509 Stage 3, plan selection (MVP): the only stage that computes cost. -//! -//! Each CandidatePhysicalASAPDAG from Stage 2 is checked against every -//! query's accuracy target: the query's output meets it when its root -//! guarantee does (theoretical: Stage 1 composes it with the built-in -//! analytical rules, and this stage binds the statistics it left symbolic -//! from the data workload and the deployment's operand-domain contracts), -//! or when the deployment's empirical evidence shows this -//! candidate's output does (`AccuracyEvidenceProvider::output_guarantee`, -//! read only here). The deployment's accuracy model may instead accept the -//! output directly, whatever its guarantee says -//! (`AccuracyModel::accepts_output_regardless_of_guarantee`); the -//! accept-all model the facade passes to ignore accuracy guarantees -//! (`asap_accuracy::IgnoreAccuracyGuarantees`) accepts every output so, and -//! the report records each one the guarantee check would have rejected. Before that, for -//! a query with a target, Count-Min is admitted only over weights proven -//! non-negative, which the executor needs to run it; a query without one -//! constrains nothing. -//! A miss rejects the candidate as invalid, with a reason; only this stage -//! discards valid candidates. -//! A candidate whose evaluation of some query takes longer than that -//! query's latency bound, or than its repeat interval, is rejected as -//! invalid too ("Latency" below). -//! Every valid candidate's resource usage is measured node by node over its -//! physical DAG ([`ResourceUsage`](asap_cost::ResourceUsage)), and the deployment's [`CostModel`](asap_cost::CostModel) -//! chooses among them ("Cost models" below). The rest are reported valid -//! but costlier (or slower, or larger). -//! The group selection MILP ([`select_plan`], #1095) selects one candidate -//! per memo group and execution context without constructing the Cartesian -//! product of local candidates; it prices cost model 1 only (#888). -//! -//! ## Cost model: cost per second of wall time -//! -//! A candidate's cost is what running it costs per second ([`COST_UNIT`](asap_cost::COST_UNIT)), -//! so queries with different recurrences add up. A node here is one -//! operator node of the candidate's physical plan DAG (`PhysicalASAPDAG`), -//! not a machine. Each node's price for one -//! computation is multiplied by how often it is computed per second -//! ([`NodeCost::cost_per_second`](asap_cost::NodeCost::cost_per_second)): -//! -//! * A `NotMaterialized` node is recomputed for each consumer that reads it -//! and is not stored: it is charged once per consumer -//! ([`NodeCost::multiplier`](asap_cost::NodeCost::multiplier)) at the sum of their rates. Its consumers are -//! the queries whose sub-DAG reaches it and the materialized nodes above -//! it, counted without looking through a materialized node: a node below -//! one is computed when that node is. The `MatAtIngestion` consumers of -//! one refresh window (`PhysicalASAPDAG::refresh_window`: for a tumbling window -//! chain, its tumbling-window width) are refreshed in one ingestion run and count as -//! one consumer: a `Scan` under a KLL chain and a DDSketch chain of 10 s -//! tumbling windows is computed once per second of arriving data, under chains of -//! 10 s and 60 s tumbling windows twice. Stage 2 keeps a shared node one -//! node (it copies it only for readers at two timings). A shared -//! `NotMaterialized` input costs what reading it separately does (#509 -//! Example 4) only when every consumer reads the same window: a query-time `Scan` is priced over the longest window -//! read above it, so each consumer pays for that window, and a scan a 1 m -//! and a 5 m window share costs more than two separate scans. Stage 2 -//! offers the highest shared node built once per evaluation (`MatAtQuery`) -//! as another candidate. -//! * A `MatAtQuery` node is computed once for all the queries that reach -//! it, at their combined rate: queries repeating at the same interval are -//! evaluated together, so each distinct interval counts once. -//! * A `MatAtIngestion` node runs as data arrives: it is priced over one -//! second of arriving rows, once per second, whatever its readers' rates. -//! A tumbling-window chain maintained this way updates only its newest -//! tumbling window: each arriving row is placed in it and updates it. -//! * A query repeating every `I` ms runs `1000 / I` times per second, and an -//! estimated rate runs at that rate. One-off, scheduled and unknown -//! queries are spread over the settings' `non_repeating_query_horizon_seconds` -//! (default 3600 s): `invocations / horizon` per second. -//! -//! Under cost model 1, memory is charged as bytes held per second -//! ([`ResourceUsage::average_memory_bytes`](asap_cost::ResourceUsage::average_memory_bytes)), at -//! `cost_per_memory_byte_second` per byte per second. A `MatAtIngestion` -//! node keeps its state every second: twice for a whole window (the window -//! being built and the completed one queries read) and `N + 1` tumbling windows for a -//! chain of `N` (the `N` the query merges and the one being built). A -//! `MatAtQuery` node's state is reused only within one evaluation, so it is -//! held only while that evaluation runs: its bytes times the latency of the -//! longest query reading it, once per evaluation. A query-time -//! `NotMaterialized` node's working memory (a sketch being built, an exact -//! aggregate's values) is charged once per consumer that recomputes it: -//! for a query, its bytes times that query's latency, at its rate; for a -//! materialized node reading it (directly or through other -//! `NotMaterialized` nodes), its bytes times the latency of the longest -//! query reading that node, at the node's rate, as a `MatAtQuery` state is. -//! A `MatAtQueryKeptAcrossEvaluations` tumbling-window chain (#509 Example -//! 4's B3, offered by Stage 2 when the deployment keeps query-time results, -//! `DeploymentCapabilities::query_time_results_retained`) is kept across -//! evaluations: each evaluation scans one tumbling window of raw data and builds the -//! newest tumbling window, and the `N − 1` older tumbling windows, which complete the readers' -//! look-back, are kept every second (in memory, all `N` at the peak). When -//! Stage 2 offers both query-time options for a chain, plan selection -//! prices each and the cheaper wins. -//! -//! A query whose workload entry declares a range evaluation -//! ([`RangeEvaluation`](asap_types::workload::RangeEvaluation): each request evaluates it at every `step_ms` over -//! the last `span_ms`, as a dashboard's `query_range` does) is priced as -//! `S = span_ms / step_ms + 1` evaluations per request (61 for a 1 h span -//! at 1 min steps): -//! -//! * every query-time computation that is not kept across evaluations -//! (`NotMaterialized`, `MatAtQuery`) runs once per step, `S` times per -//! request; a `MatAtIngestion` state is maintained as before, and the -//! merge and estimate over it run once per step; -//! * a chain kept across evaluations is built once per request: Stage 2 -//! offers it only when the requests repeat at its tumbling-window width, so in the -//! steady state each request builds its newest tumbling window and every other tumbling window -//! any of its steps reads was built by an earlier request or step; -//! * a tumbling-window chain kept beyond one evaluation (ingestion time or kept -//! across evaluations) retains its input's offset plus its window plus -//! the longest `span_ms` among its readers (`RealizedPlan::tumbling_window_retention_ms`), -//! so the earliest step of the next request still finds its tumbling windows: that -//! many more tumbling windows are kept every second. -//! -//! Latency stays per evaluation (one step). Without a range evaluation -//! `S = 1` and the retention is the window, so every price is as above. -//! Memory a computation holds only while it runs counts in -//! [`ResourceUsage::peak_memory_bytes`](asap_cost::ResourceUsage::peak_memory_bytes), which cost models 2 and 3 read, and -//! under cost model 1 for the time it is held, as above. (A `NotMaterialized` -//! node computed at ingestion time is a streaming operator holding one row, -//! and is not charged.) -//! -//! Prices come from `asap-cost`'s analytical model (`analytical::estimate_operator`) over edge -//! statistics derived from the [`DataWorkload`](asap_types::workload::DataWorkload) and a fixed default group -//! count; summary build and estimation are priced as rows × sketch depth and -//! groups × k. A query-time scan reads the longest window above it (a time -//! shift moves a window without widening it); a time shift is free, and a -//! time range is priced by the rows it keeps. A tumbling-window chain is -//! priced as one build over its whole window plus one operation per row to -//! place it in its tumbling window, and keeps one state per group and tumbling window; a merge -//! costs one operation per state it reads. Kept state is sized by its bytes: a sketch by its parameters, a -//! fixed-size exact accumulator (sum, count, min, max, rate, ...) as one -//! value per group, and exact state that grows with the data (quantile, -//! top-k, distinct count, L2, entropy keep every value or item) as one value -//! per input row of its group, assuming rows spread evenly over groups and, -//! for distinct items, that every row may be distinct. These numbers are -//! illustrative, not calibrated: a binary, join or concat node is priced from -//! its first input only (MVP; follow-up). -//! A cost error on any candidate aborts selection; it is reachable only from -//! bad declared rates or bad [`CostInputs`](asap_cost::CostInputs). -//! -//! ## Measured operation costs -//! -//! A deployment whose runtime measured its own costs gives them per -//! operator ([`CostInputs::measured_operation_costs`](asap_cost::CostInputs::measured_operation_costs), -//! [`MeasuredOperationCosts`](asap_cost::MeasuredOperationCosts)): for an operator kind at ingestion or query -//! time (and, for a summary operator, its sketch or exact accumulator; for -//! an aggregate, each measure), CPU ns per unit and per computation. A node -//! with an entry takes `units × ns_per_unit + ns_per_invocation` per -//! computation, its scanning included; the units are what the operator -//! processes ([`OperatorKind`](asap_cost::OperatorKind): samples a scan reads, rows a build inserts, -//! states a merge reads, groups an estimate reads, ...). A query-time scan -//! is the fetch and decode of raw data samples from the database holding them, -//! so a plan that reads raw data pays for it. A sketch whose parameters -//! no entry has exactly is priced between the two measured configurations -//! of the same sketch nearest in state size, on a log scale, or from the -//! nearest one outside their range. A kept state's bytes are the entry's -//! measured `memory_bytes_per_state` when it has one. The newest window's -//! compaction (see "Latency") merges with the state's query-time merge -//! entry too, and each query request pays the table's `ns_per_evaluation` -//! once, on the node it reads first and in its latency. -//! -//! A node without an entry is priced from the formulas above, its CPU -//! operations times one calibration factor: the measured time of every -//! node the table prices, over all candidates, divided by what the -//! formulas give for those nodes. Without it a table that measures some -//! operators (a slow sketch) and not others (the exact plan's scan and -//! aggregate) makes the unmeasured plan look cheap only because its nodes -//! keep the formulas' scale. The factor is one global heuristic, not a -//! measurement: an unmeasured operator whose real cost is far from the -//! measured ones' ratio is still mispriced, so a partial table can still -//! mis-rank plans (e.g. a cheap raw scan and exact quantile priced at the -//! slow sketch's ratio make the exact rebuild look costlier than it is). -//! Only a table covering every operator kind the candidates use makes the -//! choice the measured one. Each node reports how it was priced -//! ([`NodePriceSource`](asap_cost::NodePriceSource)). -//! Measured times count as CPU operations of [`ResourceSpeed`](asap_cost::ResourceSpeed), so latency -//! and all three cost models read them like any other operation. -//! -//! ## Cost models -//! -//! [`CostInputs::cost_model`](asap_cost::CostInputs::cost_model) (the cost inputs [`select_plan`] receives) chooses how -//! usage becomes a choice; the default is cost model 1 with -//! [`ResourceUnitCost::illustrative`](asap_cost::ResourceUnitCost::illustrative). -//! [`select_plan`] searches under cost model 1 only; cost models 2 and 3 -//! are an `UnsupportedCostModel` error until #888 (the whole-plan reference -//! pricer still ranks plans under all three): -//! -//! * [`CostModel::UsageOverTime`](asap_cost::CostModel::UsageOverTime) (cost model 1, pay-as-you-go): each -//! node's CPU operations, scanned bytes and memory byte-seconds per second -//! (and disk bytes, unpriced by default) times their prices; the cheapest -//! candidate wins. -//! * [`CostModel::ProvisionedCapacity`](asap_cost::CostModel::ProvisionedCapacity) (cost model 2): the candidate's peak -//! memory, peak CPU cores and disk times their prices per second; the -//! cheapest wins. Peak memory assumes every evaluation overlaps: kept -//! in-memory state plus each computation's working memory, once per -//! consumer that recomputes it. Peak cores are the ingestion work's cores -//! plus one per query evaluating at once. -//! * [`CostModel::ClusterCapacity`](asap_cost::CostModel::ClusterCapacity) (cost model 3, own cluster): no prices. -//! A candidate over the cluster's memory, disk or cores is invalid, and -//! [`ClusterObjective`](asap_cost::ClusterObjective) chooses among the rest (lowest worst-query -//! latency, or least peak memory). -//! -//! Disk holds state materialized on `Medium::Disk` (Stage 2 places none -//! there yet) and the raw data query-time scans read: per source, the -//! longest window a scan reads. A deployment that does not retain raw data -//! (`DeploymentCapabilities::raw_data_retained`) keeps no raw data samples -//! after they arrive, so Stage 2 generates no query-time scan of raw data. -//! -//! ## Latency -//! -//! Following the sketch-bench RQE cost model (§5, batch latency), a query's -//! latency per evaluation is its longest chain of work, each step on a -//! full core of elastic CPU (no queueing): the compaction of the newest -//! window of an ingestion-time state it reads, then its own query-time -//! work. Its own work is the sum, over every query-time node it reaches -//! (shared ones included, each once), of one computation's CPU operations -//! and scanned bytes timed by [`ResourceSpeed`](asap_cost::ResourceSpeed)'s `ms_per_cpu_op` and -//! `ms_per_scanned_byte`. A state whose ingestion work needs `k` cores keeps -//! `k` partial copies of its newest window, and compacting them takes -//! `(k − 1)` merges for each of its states per window, each one operation -//! (times the calibration factor), or the state's measured merge time when -//! the measured table has one; with -//! one core there is nothing to compact. A query with -//! `Some(bound)` whose latency is above the -//! bound makes the candidate invalid, with the latency as the reason -//! ([`RejectionCause::LatencyBoundExceeded`]). With or without a bound, a -//! query repeating every `I` ms whose latency is above `I` makes the -//! candidate invalid too ([`RejectionCause::LatencyOverRepeatInterval`]): -//! each evaluation starts before the last one ends, so it falls further -//! behind with every evaluation. Only a fixed interval is checked: an -//! estimated rate, a schedule or a one-time query has none. A range -//! evaluation is checked by its latency per step, as priced: the latency -//! model gives each step a full core, so a request's steps may run in -//! parallel. Run one after another they take `S` times as long, which this -//! check does not assume. -//! [`ResourceUsage::latency_per_query`](asap_cost::ResourceUsage::latency_per_query) reports every query's latency. -//! -//! No memory limit applies outside cost model 3, where the cluster's memory -//! is declared ([`ClusterLimits::memory_bytes`](asap_cost::ClusterLimits::memory_bytes)); without one, a selected -//! plan whose peak memory is over `LARGE_PEAK_MEMORY_BYTES` (10^12) gets a -//! warning, which `asap_planner` reports ([`large_peak_memory_warning`]). - -#[cfg(test)] -use asap_types::workload::RepeatedDemand; +//! #509 Stage 3, plan selection (concepts doc +//! `memo-groups-properties-requirements.md`, section 3, "Stage 3: baseline +//! selection and realization", and sections 4 and 5). +//! +//! [`select_plan`] reads Stage 2's physical candidates $\mathcal C$ and the +//! admissible part of the physical edges $\mathcal E$, selects with a MILP +//! over $x_v$, $y_{u,c,i}$, $z_{q,v}$ and $t_v$ (constraints (A)–(I); +//! (G) and (H) hold because $y$ exists only on admissible edges), and +//! decodes $\{x = 1\}$ and $\{y = 1\}$ into a +//! `DagOfSelectedCandidates`, which the injected realizer builds. The +//! extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, +//! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ +//! (section 5), and accuracy at each query root, constraint (K), with +//! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from +//! `asap-cost`. -mod accuracy_binding; -mod checks; -mod models; -// #1095 §2.0: the group selection MILP's inputs (dominance, execution -// contexts, accuracy weights). -mod groups; -// #1095 §2.0: the group selection MILP and the decode of its solution. mod milp; +mod models; mod report; -// Stage 3's entry, `select_plan`, over the group selection MILP (#1095). mod select; -// The whole-plan selector behind `reference_pricer` and the tests. -#[cfg(any(test, feature = "reference-pricer"))] -mod selector; + pub use models::AccuracyCheckInputs; -pub(crate) use models::SelectionModels; pub use report::{ large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, InexactObjective, RejectionCause, SearchReport, SearchStatus, Selection, }; pub use select::select_plan; -#[cfg(test)] -pub(crate) use selector::select_among_plans; - -/// The whole-workload reference pricer (feature `reference-pricer`): the -/// selector Stage 3 used before the memo search, over whole physical plans -/// a developer tool enumerates (`asap_devtools::memo_pricing`, -/// `planner_fuzz`). Not part of Stage 3's interface. -#[cfg(feature = "reference-pricer")] -pub mod reference_pricer { - pub use crate::report::{Rejection, WholePlanSelection}; - use crate::{AccuracyCheckInputs, ErrorSelectingPlan, SelectionModels}; - use asap_cost::CostInputs; - use asap_types::physical::PhysicalWorkloadPlan; - use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - - /// Check, price and rank every plan of `candidates`, keeping every - /// candidate: the selected one, each valid one's cost and accepted - /// accuracy, and why each other one lost. - /// - /// Note: with measured operation costs and no frozen calibration factor, - /// the formulas are calibrated over `candidates` alone, so costs are not - /// on the scale of a facade run's selection (`--price-enumerated`). - pub fn select_among_plans( - candidates: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, - cost_inputs: &CostInputs, - ) -> Result { - crate::selector::select_among_plans( - candidates, - entries, - data, - SelectionModels::new(accuracy_checks, cost_inputs), - ) - } -} - -/// `roots` sized as Pass 1 sizes them for `targets` over `data` -/// ([`with_summaries_sized_for_targets`](asap_accuracy::sizing::with_summaries_sized_for_targets)). -#[cfg(test)] -pub(crate) fn sized_as_pass1( - roots: &[asap_types::ir::QueryRoot], - targets: &[asap_types::ir::properties::AccuracyTarget], - data: &asap_types::workload::DataWorkload, -) -> Vec { - asap_accuracy::sizing::with_summaries_sized_for_targets(roots, targets, data) - .expect("sized summaries rebuild") -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::accuracy_binding::bound_root_guarantee; - use crate::accuracy_binding::{bind_statistics, workload_statistic}; - use crate::checks::guarantee_check; - use crate::checks::{accuracy_check, CheckFailure}; - use crate::report::WholePlanSelection; - use crate::selector::PlanSelector; - use asap_accuracy::composition::INPUT_ROW_COUNT; - use asap_accuracy::estimators::hll; - use asap_accuracy::propagation::root_guarantee; - use asap_cost::analytical::AnalyticalCostError; - use asap_cost::execution_demand::evaluation_rate; - use asap_cost::state_bytes::{sketch_shape, summary_shape, StateInputRows}; - use asap_cost::{kept_tumbling_window_chains, node_map, plan_cost, NodeMap}; - use asap_cost::{lookup, StateFamily}; - use asap_cost::{ - CalibrationFit, ClusterLimits, ClusterObjective, CostInputs, CostModel, NodePriceSource, - OperatorKind, PlanCost, ResourceUsage, COST_SOURCE, MEASURED_COST_SOURCE, - MIXED_COST_SOURCE, - }; - use asap_cost::{MeasuredOperationCosts, OperatorCost, ResourceUnitCost}; - use asap_types::ir::operator::AggIntent; - use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, - }; - use asap_types::ir::properties::{AccuracyGuarantee, AccuracyScope, ExecutionTiming}; - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::schema::DataType; - use asap_types::ir::schema::{ - FieldDataType, Schema, SketchAlgorithm, SketchParams, WeightDomain, - }; - use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; - use asap_types::physical::PhysicalWorkloadPlan; - use asap_types::physical::{Materialization, Medium}; - use asap_types::workload::bounds::with_confirmed_weights; - use asap_types::workload::bounds::{declared_series, series_read, values_per_result}; - use asap_types::workload::{ - DataWorkload, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, - }; - use std::collections::{BTreeMap, BTreeSet}; - use std::rc::Rc; - - /// [`plan_cost`] of one candidate alone, its calibration factor fitted - /// over its own measured nodes (as [`select_plan`] fits it over all). - fn self_calibrated_cost( - dag: &PhysicalASAPDAG, - nodes: &NodeMap<'_>, - kept_chains: &std::collections::HashSet, - recurrence: &[QueryRecurrence], - range_evaluations: &[Option], - data: &DataWorkload, - settings: &CostInputs, - ) -> Result { - let at = |formula_scale| { - plan_cost( - dag, - nodes, - kept_chains, - recurrence, - range_evaluations, - data, - settings, - formula_scale, - ) - }; - let (_, fit) = at(1.0)?; - Ok(at(fit.factor())?.0) - } - - /// [`self_calibrated_cost`] with `dag`'s node map and kept tumbling-window chains. - fn price( - dag: &PhysicalASAPDAG, - recurrence: &[QueryRecurrence], - data: &DataWorkload, - settings: &CostInputs, - ) -> Result { - self_calibrated_cost( - dag, - &node_map(dag), - &kept_tumbling_window_chains(dag), - recurrence, - &[], - data, - settings, - ) - } - use asap_accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence, - }; - use asap_logical_asap_optimization::test_support::{entry, entry_with, lower_promql}; - use asap_logical_asap_optimization::DeploymentCapabilities; - use asap_physical_asap_optimization::optimize_physical_asap; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::properties::{AccuracyMetric, BoundExpr, ProbabilityExpr}; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::schema::{GroupingStrategy, SketchKind}; - use asap_types::ir::{ExprSemantics, ScalarExpr}; - use asap_types::workload::{ - DataArrival, LabelSetDistinctValues, PrometheusMetricType, SourceFacts, - }; - - /// `G` of `root` over `data` ([`values_per_result`]). - fn values_of(root: &QueryRoot, data: &DataWorkload) -> Option { - values_per_result(root, series_read(root, data), data) - } - - /// [`bound_root_guarantee`] of `root` over `data`. - fn bound_of( - root: &QueryRoot, - data: &DataWorkload, - models: &SelectionModels<'_>, - ) -> Option { - bound_root_guarantee(root, series_read(root, data), data, models) - } - - /// `n` queries repeating every 10 s, as Example 1's panels do. - fn recurrences(n: usize) -> Vec { - vec![repeating(10_000); n] - } - - /// A query repeating every `interval_ms`. - fn repeating(interval_ms: u32) -> QueryRecurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - asap_types::workload::RepetitionInterval(interval_ms), - )) - } - - /// The metrics and tables the tests' queries read. - const TEST_SOURCES: [&str; 5] = ["m", "x", "a", "b", "t"]; - - /// Data arriving at 10 000 rows per second, each test source holding - /// 10 000 series. - fn data() -> DataWorkload { - with_series(Some(10_000)) - } - - /// [`data`] with every test source holding `series` (`None`: unknown). - fn with_series(series: Option) -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 10_000.0, - }, - sources: TEST_SOURCES - .iter() - .map(|source| { - let facts = SourceFacts { - series_count: series, - ..SourceFacts::default() - }; - (source.to_string(), facts) - }) - .collect(), - } - } - - /// Explicit domain assumption needed for interpolated DDS quantiles. - fn non_negative_data() -> DataWorkload { - let mut data = data(); - for facts in data.sources.values_mut() { - facts.values_non_negative = true; - } - data - } - - /// `data` with each of `types`' metrics declared of its type. - fn with_metric_types( - mut data: DataWorkload, - types: &[(&str, PrometheusMetricType)], - ) -> DataWorkload { - for (metric, metric_type) in types { - data.sources - .entry(metric.to_string()) - .or_default() - .metric_type = Some(*metric_type); - } - data - } - - /// Classic HLL's failure probability over a source binds only that - /// source's distinct bound: a bound keyed to another source leaves it - /// unknown. - #[test] - fn classic_hll_failure_probability_binds_only_its_own_sources_bound() { - let statistic = asap_logical_asap_optimization::test_support::hll_classic_failure_statistic( - 15, 0.05, "a", - ); - let bounded = |source: &str| { - let mut data = data(); - data.sources - .get_mut(source) - .unwrap() - .max_distinct_per_evaluation = Some(100); - data - }; - assert_eq!(workload_statistic(&statistic, None, &bounded("b")), None); - assert_eq!( - workload_statistic(&statistic, None, &bounded("a")), - Some(hll::classic_failure_probability(15, 0.05, 100)) - ); - assert!(hll::classic_failure_probability(15, 0.05, 100) <= 0.01); - } - - /// Exact (P1), whole-expression CMS+heap (P2) and CountSketch+heap (P3) realizations of - /// one approximate top-k query, in Pass 1 catalog order. - fn candidates() -> Vec { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let root = lower_promql("topk by (job) (10, sum_over_time(m[1m]))", target.clone()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let topk = inventory - .targets - .iter() - .position(|t| t.choice_count() > 2) - .unwrap(); - [0, 1, 2] - .into_iter() - .map(|index| { - let mut choice = vec![0; inventory.targets.len()]; - choice[topk] = index; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - let roots = sized_as_pass1(&roots, std::slice::from_ref(&target), &data()); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0); - candidate.id = format!("P{}", index + 1); - candidate - }) - .collect() - } - - /// A summary whose analytical guarantee misses the target is rejected as - /// invalid with a reason; the exact plan is then selected. (Top-K - /// estimates carry no guarantee since #575, so a quantile summary is used.) - #[test] - fn accuracy_failing_candidate_is_rejected_as_invalid() { - let planned = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates("quantile_over_time(0.99, m[1m])", &planned); - let strict = AccuracyTarget::max_error(1e-6).failure_probability(1e-9); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - let selection = select_among_plans( - &candidates, - &[entry_with(strict, repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - assert!(!summaries.contains(&selection.selected.as_str())); - for id in &summaries { - let rejection = selection.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::AccuracyTargetMissed); - assert!( - rejection.reason.contains("misses target"), - "{}", - rejection.reason - ); - assert!(!selection.costs.contains_key(*id)); - } - } - - /// For a query with a target, Count-Min over weights not proven - /// non-negative is invalid. - #[test] - fn count_min_over_signed_weights_is_rejected_as_invalid() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let selection = select_among_plans( - &candidates(), - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative - ); - assert!(p2.reason.contains("non-negative"), "{}", p2.reason); - } - - /// Count-Min over raw data samples of `m` is admitted only when the data - /// workload declares `m` a counter: a gauge, an undeclared metric or a - /// counter under another name leaves the requirement unmet. Admitted, - /// the top-k still needs evidence about its output. - #[test] - fn count_min_over_raw_data_samples_needs_a_declared_counter() { - use asap_types::workload::PrometheusMetricType; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let with_types = |types: &[(&str, PrometheusMetricType)]| with_metric_types(data(), types); - for (types, admitted) in [ - (vec![("m", PrometheusMetricType::Counter)], true), - (vec![("m", PrometheusMetricType::Gauge)], false), - (vec![("m_total", PrometheusMetricType::Counter)], false), - (vec![], false), - ] { - let data = with_types(&types); - let selection = select_among_plans( - &candidates(), - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - // Admitted weights leave the top-k output without a guarantee, - // which the built-in model does not accept; unproven weights - // reject the plan before any accuracy check. - let expected_cause = if admitted { - RejectionCause::NoAccuracyGuarantee - } else { - RejectionCause::CountMinWeightsNotProvenNonNegative - }; - assert_eq!(p2.cause, expected_cause); - assert_eq!( - !p2.reason.contains("non-negative"), - admitted, - "{types:?}: {}", - p2.reason - ); - if admitted { - assert!(p2.reason.contains("no guarantee"), "{}", p2.reason); - } else { - assert!( - p2.reason.contains("not declared a counter"), - "{}", - p2.reason - ); - } - } - } - - /// The Count-Min sketch algorithms (plain or with a heap) `candidate` - /// builds. - fn count_min_builds(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if matches!( - kind.algorithm(), - SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap - ) => - { - Some(*kind.algorithm()) - } - _ => None, - }) - .collect() - } - - /// `increase` (and `rate`, `irate`) over a counter is non-negative, since - /// resets are corrected: the Count-Min-with-heap top-k of - /// `topk(10, sum by (job) (increase(data_total[5m])))` passes the weight - /// check when `data_total` is declared a counter, and goes on to the - /// accuracy and cost checks. Over a gauge it is still rejected for its - /// weights (issue #773). - #[test] - fn count_min_over_a_counter_derivative_needs_a_declared_counter() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - // An `offset` moves the window, not the samples' sign. - for query in [ - "topk(10, sum by (job) (increase(data_total[5m])))", - "topk(10, sum by (job) (rate(data_total[5m])))", - "topk(10, sum by (job) (irate(data_total[5m])))", - "topk(10, sum by (job) (rate(data_total[5m] offset 1h)))", - ] { - let candidates = all_candidates(query, &target); - let count_min: Vec<&str> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!count_min.is_empty(), "{query}"); - for (metric_type, admitted) in [ - (PrometheusMetricType::Counter, true), - (PrometheusMetricType::Gauge, false), - ] { - let data = with_metric_types(data(), &[("data_total", metric_type)]); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{query}: {error}"), - }; - for id in &count_min { - let weights_rejected = rejected.iter().any(|r| { - &r.id == id - && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "{query} over a {metric_type:?}, {id}: {rejected:?}" - ); - } - } - } - } - - /// The `_sum` series of a declared histogram family is a counter: with - /// only the family `latency` declared a histogram, the Count-Min-with-heap - /// top-k of `topk(10, sum by (job) (increase(latency_sum[5m])))` passes - /// the weight check. Undeclared, or with `latency` a gauge, it does not - /// (issue #1165). - #[test] - fn count_min_over_a_declared_histograms_sum_series_is_admitted() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let query = "topk(10, sum by (job) (increase(latency_sum[5m])))"; - let candidates = all_candidates(query, &target); - let count_min: Vec<&str> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!count_min.is_empty()); - for (family_type, admitted) in [ - (Some(PrometheusMetricType::Histogram), true), - (Some(PrometheusMetricType::Gauge), false), - (None, false), - ] { - let mut data = data(); - data.sources.insert( - "latency".into(), - SourceFacts { - series_count: Some(10_000), - metric_type: family_type, - ..SourceFacts::default() - }, - ); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{family_type:?}: {error}"), - }; - for id in &count_min { - let weights_rejected = rejected.iter().any(|r| { - &r.id == id && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "latency declared {family_type:?}, {id}: {rejected:?}" - ); - } - } - } - - /// A gauge declared non-negative - /// ([`SourceFacts::values_non_negative`]) proves its raw data samples, and - /// plain sums of them, non-negative: the Count-Min-with-heap top-k of - /// `topk by (job) (10, sum_over_time(m[1m]))` (the ASAPQuery workloads' - /// top-k-sum shape) passes the weight check, and the selected plan carries - /// [`NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples`]. A rate or - /// increase of that gauge stays unproven: a reset correction over a - /// gauge can still give a negative value. - #[test] - fn count_min_over_raw_data_samples_of_a_gauge_declared_non_negative_is_admitted() { - use asap_types::ir::schema::NonNegativeWeightProof; - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - gauge.sources.get_mut("m").unwrap().values_non_negative = true; - for (query, admitted) in [ - ("topk by (job) (10, sum_over_time(m[1m]))", true), - ("topk(10, sum by (job) (increase(m[5m])))", false), - ] { - let candidates = all_candidates(query, &target); - let count_min: Vec<&PhysicalWorkloadPlan> = candidates - .iter() - .filter(|c| count_min_builds(c).contains(&SketchAlgorithm::CmsWithHeap)) - .collect(); - assert!(!count_min.is_empty(), "{query}"); - let rejected = match select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &gauge, - SelectionModels::builtin(), - ) { - Ok(selection) => selection.rejected, - Err(ErrorSelectingPlan::NoValidCandidate(rejected)) => rejected, - Err(error) => panic!("{query}: {error}"), - }; - for candidate in &count_min { - let weights_rejected = rejected.iter().any(|r| { - r.id == candidate.id - && r.cause == RejectionCause::CountMinWeightsNotProvenNonNegative - }); - assert_eq!( - weights_rejected, !admitted, - "{query}, {}: {rejected:?}", - candidate.id - ); - if !admitted { - continue; - } - // The proof the executor reads. - let proofs: Vec = with_confirmed_weights(&candidate.roots, &gauge) - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { - input, - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::CmsWithHeap => { - Some(input.weight_domain.clone()) - } - _ => None, - }) - .collect(); - assert!(!proofs.is_empty(), "{}", candidate.id); - assert!( - proofs.iter().all(|domain| *domain - == WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples - }), - "{}: {proofs:?}", - candidate.id - ); - } - } - } - - /// The sign gate is an indicator that only 1 (proven) opens: any other - /// value a binder supplies, 0 included, leaves it unbound rather than - /// reading as a perfect bound. Other statistics pass through. - #[test] - fn the_sign_gate_opens_only_at_one() { - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let gated = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic("m"), - }, - ], - }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - for value in [0.0, 0.5, 2.0, -1.0] { - assert_eq!( - bind_statistics(&gated, &|_| Some(value)) - .spec - .error - .evaluate(), - None, - "{value}" - ); - } - assert_eq!( - bind_statistics(&gated, &|_| Some(1.0)) - .spec - .error - .evaluate(), - Some(0.01) - ); - let mut norm = gated; - norm.spec.error = BoundExpr::Unknown { - statistic: INPUT_ROW_COUNT.into(), - }; - assert_eq!( - bind_statistics(&norm, &|_| Some(0.0)).spec.error.evaluate(), - Some(0.0) - ); - } - - /// A relative bound gated on the sign of one metric's samples is bound - /// when the data workload declares that metric a counter, and stays - /// unknown for a gauge, an undeclared metric or a counter by another - /// name. - #[test] - fn sign_of_counter_raw_data_samples_is_bound_from_the_declared_type() { - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let gated = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { value: 0.01 }, - BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic("m"), - }, - ], - }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let node = lower_promql("sum(m)", AccuracyTarget::exact()); - let root = QueryRoot::Operator(Rc::new((*node).clone().with_guarantee(Some(gated)))); - let with = |types: &[(&str, PrometheusMetricType)]| with_metric_types(data(), types); - let bound = |data: &DataWorkload| { - bound_of(&root, data, &SelectionModels::builtin()) - .unwrap() - .spec - .error - .evaluate() - }; - assert_eq!( - bound(&with(&[("m", PrometheusMetricType::Counter)])), - Some(0.01) - ); - for types in [ - vec![("m", PrometheusMetricType::Gauge)], - vec![("m_total", PrometheusMetricType::Counter)], - vec![], - ] { - assert_eq!(bound(&with(&types)), None, "{types:?}"); - } - } - - /// Every consumer of a run sees the selected plan with its weight - /// requirements answered: the logical plan (`selected_logical`) and the - /// Offered one at a time and keeping only the best so far, the selector - /// selects what `select_among_plans` selects over every candidate at - /// once, at the same cost, under every cost model. - #[test] - fn streaming_selection_selects_what_select_plan_selects() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let root = lower_promql("quantile_over_time(0.99, m[5m])", target.clone()); - let entries = [entry_with(target, repeating(60_000))]; - let data = with_series(Some(10_000)); - let candidates = &exhaustive_candidates(&[QueryRoot::Operator(root)], &entries, &data); - assert!(candidates.len() > 2); - let cluster = |objective| { - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective, - }) - }; - for cost_model in [ - CostModel::UsageOverTime(ResourceUnitCost::illustrative()), - CostModel::ProvisionedCapacity(ResourceUnitCost::illustrative()), - cluster(ClusterObjective::LowestLatency), - cluster(ClusterObjective::LeastMemory), - ] { - let settings = CostInputs { - cost_model, - ..CostInputs::default() - }; - let models = SelectionModels::builtin().with_cost_inputs(&settings); - let all = select_among_plans(candidates, &entries, &data, models).unwrap(); - let mut selector = PlanSelector::new(&entries, &data, models, false).unwrap(); - for candidate in candidates { - selector.offer(candidate).unwrap(); - } - let (streamed, built) = selector.finish().unwrap(); - assert_eq!(streamed.selected, all.selected); - assert_eq!(built.id, all.selected); - assert_eq!(streamed.plan, all.plan); - assert_eq!(streamed.costs[&streamed.selected], all.costs[&all.selected]); - assert_eq!(streamed.candidates_checked, candidates.len() as u64); - // Only the selected candidate is kept. - assert_eq!(streamed.costs.len(), 1); - } - } - - /// The selected plan's open requirement becomes a proof when confirmed - /// and stays unproven otherwise; the executor reads no data workload. - #[test] - fn confirmed_weights_rewrite_only_the_summary_update() { - use asap_types::ir::schema::NonNegativeWeightProof; - use asap_types::workload::PrometheusMetricType; - let p2 = &candidates()[1]; - let domains = |roots: &[QueryRoot]| -> Vec { - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => Some(input.weight_domain.clone()), - _ => None, - }) - .collect() - }; - assert_eq!( - domains(&p2.roots), - [WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }] - ); - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let confirmed = with_confirmed_weights(&p2.roots, &counter); - assert_eq!( - domains(&confirmed), - [WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }] - ); - let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - gauge.sources.get_mut("m").unwrap().values_non_negative = true; - assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &gauge)), - [WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples - }] - ); - assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &data())), - [WeightDomain::UnknownOrSigned] - ); - // Nothing open: the same roots. - let exact = &candidates()[0]; - let QueryRoot::Operator(before) = &exact.roots[0] else { - panic!() - }; - let QueryRoot::Operator(after) = &with_confirmed_weights(&exact.roots, &counter)[0] else { - panic!() - }; - assert!(Rc::ptr_eq(before, after)); - } - - /// Each node is priced under its DAG id, and the total is the sum; - /// every candidate is either selected or rejected as costlier. The - /// CountSketch+heap estimate has no analytical guarantee, so deployment - /// evidence admits it. - #[test] - fn per_node_costs_cover_the_dag_and_sum_to_the_total() { - let candidates = candidates(); - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let selection = select_among_plans( - &candidates, - &[entry_with( - target - .clone() - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1), - repeating(10_000), - )], - &data(), - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - for candidate in candidates.iter().filter(|c| c.id != "P2") { - let cost = &selection.costs[&candidate.id]; - let ids: Vec<_> = candidate.dag.nodes.iter().map(|n| n.id).collect(); - let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); - keys.sort(); - let mut sorted = ids.clone(); - sorted.sort(); - assert_eq!(keys, sorted); - let sum: f64 = cost - .per_node - .values() - .map(|n| n.cost_per_second.unwrap()) - .sum(); - assert_eq!(cost.cost_per_second.unwrap(), sum); - assert!(cost.cost_per_second.unwrap() > 0.0); - } - let costlier: Vec<_> = selection - .rejected - .iter() - .filter(|r| r.cause == RejectionCause::Costlier) - .collect(); - assert_eq!(costlier.len(), 1); - assert_ne!(costlier[0].id, selection.selected); - } - - /// Query 1 is `sum(...)` over query 2's output. Every node reached by - /// both queries is `NotMaterialized`, so recomputed for each: charged - /// twice. Materializing query 2's root (`MatAtQuery`) charges it once, - /// and the nodes below it once, since they run only when it does. - #[test] - fn not_materialized_shared_nodes_are_charged_per_query() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let mut candidate = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0); - let [outer, inner] = candidate.dag.roots[..] else { - panic!("two roots") - }; - // Kept state is priced apart; this compares computations only. - let settings = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let cost = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); - let multipliers: BTreeMap<_, _> = cost - .per_node - .iter() - .map(|(id, c)| (*id, c.multiplier)) - .collect(); - assert_eq!(multipliers[&outer], 1); - assert_eq!(multipliers[&inner], 2); - assert!(multipliers.len() > 2, "nodes below the shared root"); - for (id, times) in &multipliers { - if *id != outer { - assert_eq!(*times, 2, "node {id} is read by both queries"); - } - } - - let node = candidate - .dag - .nodes - .iter_mut() - .find(|n| n.id == inner) - .unwrap(); - node.materialization = Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - let stored = price(&candidate.dag, &recurrences(2), &data(), &settings).unwrap(); - for (id, c) in &stored.per_node { - assert_eq!(c.multiplier, 1, "node {id} is computed once"); - let once = cost.per_node[id].cost_per_second.unwrap() / multipliers[id] as f64; - assert!( - (c.cost_per_second.unwrap() - once).abs() <= 1e-12 * once, - "node {id}" - ); - } - assert!(stored.cost_per_second.unwrap() < cost.cost_per_second.unwrap()); - } - - /// A `NotMaterialized` node two queries read is computed for each, so - /// its working memory is held once per query, each time for that - /// query's latency: `bytes · Σ rate_q · latency_q`, not the longest - /// latency at both rates. Query 1 is `sum(...)` over query 2's output, - /// so it runs longer; they repeat every 10 s and every 20 s. - #[test] - fn shared_not_materialized_memory_is_held_for_each_reading_query() { - let outer = lower_promql("sum(sum_over_time(m[1m]))", AccuracyTarget::exact()); - let inner = outer.children()[0].clone(); - let candidate = optimize_physical_asap( - "L", - &[QueryRoot::Operator(outer), QueryRoot::Operator(inner)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0); - let recurrence = [repeating(10_000), repeating(20_000)]; - let cost = |price_per_byte_second| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = price_per_byte_second); - price(&candidate.dag, &recurrence, &data(), &settings).unwrap() - }; - let (with, without) = (cost(1.0), cost(0.0)); - let [outer_ms, inner_ms] = [0, 1].map(|q| with.usage.latency_per_query[q].total_ms()); - assert!(outer_ms > inner_ms, "{outer_ms} vs {inner_ms}"); - let mut checked = 0; - for (id, node) in &with.per_node { - let Some(held) = node.detail.split("; holds ").nth(1) else { - continue; - }; - if node.multiplier != 2 { - continue; - } - let bytes: f64 = held.split(' ').next().unwrap().parse().unwrap(); - let term = - node.cost_per_second.unwrap() - without.per_node[id].cost_per_second.unwrap(); - let expected = bytes * (0.1 * outer_ms + 0.05 * inner_ms) / 1_000.0; - assert!(close(term, expected), "node {id}: {term} vs {expected}"); - checked += 1; - } - assert!(checked > 0, "a shared node holds memory"); - } - - /// Every combination of Pass 1 candidates of one query, as Stage 2 - /// candidates `P1`, `P2`, ... - fn all_candidates(query: &str, target: &AccuracyTarget) -> Vec { - all_candidates_sized(query, target, &|roots| roots) - } - - /// [`all_candidates`] with each candidate's logical roots passed through - /// `size` before Stage 2. - fn all_candidates_sized( - query: &str, - target: &AccuracyTarget, - size: &dyn Fn(Vec) -> Vec, - ) -> Vec { - let root = lower_promql(query, target.clone()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|choice: Vec| { - (0..target.choice_count()).map(move |index| { - let mut choice = choice.clone(); - choice.push(index); - choice - }) - }) - .collect(); - } - choices - .iter() - .enumerate() - .map(|(index, choice)| { - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, choice, - ) - .unwrap(); - let roots = size(roots); - let mut candidate = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0); - candidate.id = format!("P{}", index + 1); - candidate - }) - .collect() - } - - fn has_summary_estimate(candidate: &PhysicalWorkloadPlan) -> bool { - candidate - .dag - .nodes - .iter() - .any(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryEstimate { .. }))) - } - - /// [`per_series_kll`]'s candidates with their summaries sized as Pass 1 - /// sizes them for the target over `data` ([`sized_as_pass1`]). - fn per_series_kll_sized_as_pass1( - scope: AccuracyScope, - data: &DataWorkload, - ) -> (Vec, AccuracyTarget) { - let (_, target) = per_series_kll(scope); - let requested = target.clone().accuracy_scope(AccuracyScope::PerValue); - let candidates = - all_candidates_sized("quantile_over_time(0.99, x[5m])", &requested, &|roots| { - sized_as_pass1(&roots, std::slice::from_ref(&target), data) - }) - .into_iter() - .filter(|c| !kll_params(&c.roots).is_empty()) - .collect(); - (candidates, target) - } - - /// The KLL candidates of `quantile_over_time(0.99, x[5m])`, per series, - /// within rank error 0.01 at δ = 0.01 over `scope`. - fn per_series_kll(scope: AccuracyScope) -> (Vec, AccuracyTarget) { - let requested = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank); - let candidates = all_candidates("quantile_over_time(0.99, x[5m])", &requested) - .into_iter() - .filter(|c| !kll_params(&c.roots).is_empty()) - .collect::>(); - assert!(!candidates.is_empty()); - let target = requested - .clone() - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(scope); - (candidates, target) - } - - /// Bad input: a workload with a different number of entries than a - /// candidate has query roots is an error, not a rejection. - #[test] - fn an_entry_count_mismatch_is_an_error() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let entries = vec![entry_with(target, repeating(10_000)); 2]; - let Err(ErrorSelectingPlan::EntryCount { - candidate, - roots, - entries, - }) = select_among_plans(&candidates(), &entries, &data(), SelectionModels::builtin()) - else { - panic!("an entry count mismatch is an error") - }; - assert_eq!((candidate.as_str(), roots, entries), ("P1", 1, 2)); - } - - fn kll_params(roots: &[QueryRoot]) -> Vec { - roots - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::Kll => Some(kind.params().clone()), - _ => None, - }) - .collect() - } - - /// Stage 3 only checks: a KLL Pass 1 sized per value (δ) misses a - /// whole-result target over `G` = 10 000 series and is rejected, not - /// enlarged; sized as Pass 1 sizes it for that target (δ / G = 1e-6, a - /// larger `k`), it is valid, and the selected plan runs that size. - #[test] - fn a_whole_result_target_is_checked_not_resized() { - let (per_value_sized, whole) = per_series_kll(AccuracyScope::WholeResult); - let entries = [entry_with(whole.clone(), repeating(10_000))]; - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = select_among_plans( - &per_value_sized, - &entries, - &data(), - SelectionModels::builtin(), - ) else { - panic!("a per-value size misses the whole-result target") - }; - assert!(rejected - .iter() - .all(|r| r.cause == RejectionCause::AccuracyTargetMissed)); - let (sized, _) = per_series_kll_sized_as_pass1(AccuracyScope::WholeResult, &data()); - let selection = - select_among_plans(&sized, &entries, &data(), SelectionModels::builtin()).unwrap(); - let ( - SketchParams::Kll { k: per_value_k, .. }, - SketchParams::Kll { - k, - failure_probability, - }, - ) = ( - &kll_params(&per_value_sized[0].roots)[0], - &kll_params(&selection.plan)[0], - ) - else { - panic!("KLL builds") - }; - assert_eq!(*failure_probability, Some(0.01 / 10_000.0)); - assert!(k > per_value_k, "{k} vs {per_value_k}"); - let selected = sized.iter().find(|c| c.id == selection.selected).unwrap(); - assert_eq!(kll_params(&selection.plan), kll_params(&selected.roots)); - } - - /// With its metric declared a counter, the whole-expression CMS+heap - /// candidate (P2) proves its weights non-negative, is valid and is - /// priced: its per-node costs cover its DAG and sum to its total. - #[test] - fn a_declared_counter_prices_the_count_min_heap_candidate() { - use asap_types::workload::PrometheusMetricType; - let candidates = candidates(); - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let selection = select_among_plans( - &candidates, - &[entry_with(target, repeating(10_000))], - &counter, - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - let p2 = candidates.iter().find(|c| c.id == "P2").unwrap(); - let cost = &selection.costs["P2"]; - let mut ids: Vec<_> = p2.dag.nodes.iter().map(|n| n.id).collect(); - ids.sort(); - let mut keys: Vec<_> = cost.per_node.keys().copied().collect(); - keys.sort(); - assert_eq!(keys, ids); - let sum: f64 = cost - .per_node - .values() - .map(|n| n.cost_per_second.unwrap()) - .sum(); - assert_eq!(cost.cost_per_second.unwrap(), sum); - } - - /// The KLL failure probability each selected sketch is sized for. - fn kll_failure_probabilities(roots: &[QueryRoot]) -> Vec> { - kll_params(roots) - .into_iter() - .map(|params| match params { - SketchParams::Kll { - failure_probability, - .. - } => failure_probability, - other => panic!("{other:?}"), - }) - .collect() - } - - /// A target on the groups holding at least 5% of the records covers at - /// most 20 of them, so Pass 1 sizes each sketch for δ / 20 and the target is - /// met even when the number of values the result returns is unbound - /// (here a source with no declared series count), where a whole-result - /// target stays unmet. With 10 000 series declared it still needs only - /// δ / 20, not the whole result's δ / 10 000. - #[test] - fn a_record_share_target_sizes_each_sketch_for_one_over_share_groups() { - for data in [with_series(None), data()] { - let (candidates, five_percent) = per_series_kll_sized_as_pass1( - AccuracyScope::GroupsWithRecordShareAtLeast(0.05), - &data, - ); - let selection = select_among_plans( - &candidates, - &[entry_with(five_percent.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .expect("a KLL candidate meets the 5% target"); - assert_eq!( - kll_failure_probabilities(&selection.plan), - vec![Some(0.01 / 20.0)] - ); - } - } - - /// Without a bound on the values one evaluation returns, a per-value - /// guarantee does not meet a whole-result target: a PromQL vector whose - /// source declares no series count, and a SQL relation (until a bound on - /// its groups is an input). A scalar returns one value. - #[test] - fn an_unbound_number_of_values_leaves_a_whole_result_target_unmet() { - let (candidates, whole) = per_series_kll(AccuracyScope::WholeResult); - let no_series_count = with_series(None); - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = select_among_plans( - &candidates, - &[entry_with(whole, repeating(10_000))], - &no_series_count, - SelectionModels::builtin(), - ) else { - panic!("the KLL candidates are invalid") - }; - assert!( - rejected[0].reason.contains("is not bound"), - "{}", - rejected[0].reason - ); - let table = asap_logical_asap_optimization::test_support::scan_from( - asap_types::ir::operator::Source::Table { - table_ref: "t".into(), - }, - Schema::new(vec![asap_types::ir::schema::Field::plain( - "v", - DataType::Float64, - false, - )]), - ); - assert_eq!(values_of(&QueryRoot::Operator(table), &data()), None); - assert_eq!( - values_of( - &QueryRoot::Scalar(ScalarExpr::literal_f64(1.0)), - &DataWorkload::default() - ), - Some(1) - ); - } - - /// A root that returns at most one row has `G` = 1, provable from the - /// query with no data workload: a global aggregate, PromQL or SQL, and - /// what selects, renames or joins single rows above it. A grouped SQL - /// aggregate stays unbound, and a grouped PromQL one, or a global top-k - /// (k rows), is bound by the series count of the source it reads. - #[test] - fn a_root_of_at_most_one_row_returns_one_value() { - let none = DataWorkload::default(); - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - assert_eq!(values_of(&root("sum(rate(x[5m]))"), &none), Some(1)); - assert_eq!(values_of(&root("topk(5, rate(x[5m]))"), &none), None); - assert_eq!(values_of(&root("sum by (job) (rate(x[5m]))"), &none), None); - assert_eq!( - values_of(&root("sum by (job) (rate(x[5m]))"), &data()), - Some(10_000) - ); - let table = || { - asap_logical_asap_optimization::test_support::scan_from( - asap_types::ir::operator::Source::Table { - table_ref: "t".into(), - }, - Schema::new(vec![ - asap_types::ir::schema::Field::plain("k", DataType::Utf8, false), - asap_types::ir::schema::Field::plain("v", DataType::Float64, false), - ]), - ) - }; - let count = || AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }; - let global = asap_logical_asap_optimization::test_support::agg(vec![], count(), table()); - let grouped = asap_logical_asap_optimization::test_support::agg(vec![0], count(), table()); - assert_eq!( - values_of(&QueryRoot::Operator(global.clone()), &none), - Some(1) - ); - assert_eq!(values_of(&QueryRoot::Operator(grouped), &data()), None); - let cross = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { - kind: asap_types::ir::operator::operator_properties::JoinKind::Cross, - pred: asap_types::ir::Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - left: global.clone(), - right: global, - })) - .unwrap(); - assert_eq!(values_of(&QueryRoot::Operator(cross), &none), Some(1)); - } - - /// `G` and the input row count are bound per query from the sources - /// it reads: another source's series do not count, a query over two - /// sources is bound by their sum, and one source without a series - /// count leaves it unbound. Pricing reads the same per-source series, with - /// the default for an undeclared source. - #[test] - fn values_per_result_is_bound_by_the_series_of_the_sources_read() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let series = |declared: &[(&str, u64)]| DataWorkload { - sources: declared - .iter() - .map(|(source, series)| { - let facts = SourceFacts { - series_count: Some(*series), - ..SourceFacts::default() - }; - (source.to_string(), facts) - }) - .collect(), - ..DataWorkload::default() - }; - let x = root("sum by (job) (rate(x[5m]))"); - let both = root("sum by (job) (rate(x[5m])) + sum by (job) (rate(m[5m]))"); - assert_eq!(values_of(&x, &series(&[("x", 10)])), Some(10)); - assert_eq!(values_of(&x, &series(&[("m", 10)])), None); - let declared = series(&[("x", 10), ("m", 20)]); - assert_eq!(values_of(&x, &declared), Some(10)); - assert_eq!(values_of(&both, &declared), Some(30)); - assert_eq!(series_read(&both, &declared), Some(30)); - assert_eq!(values_of(&both, &series(&[("x", 10)])), None); - // Pricing sums the series of the sources a node reads; an - // undeclared source, or none, leaves them unknown. - assert_eq!(declared_series(&["x", "m"].into(), &declared), Some(30)); - assert_eq!(declared_series(&["x", "y"].into(), &declared), None); - assert_eq!(declared_series(&BTreeSet::new(), &declared), None); - } - - /// The exact physical DAG of `queries` planned together, over `data`. - fn exact_dag(queries: &[&str], data: &DataWorkload) -> PhysicalASAPDAG { - let roots: Vec<_> = queries - .iter() - .map(|query| { - let root = lower_promql(query, AccuracyTarget::exact()); - QueryRoot::Operator(root) - }) - .collect(); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - optimize_physical_asap("L1", &roots, &[], data, false) - .unwrap() - .remove(0) - .dag - } - - /// A node's operator, timing, work and price, to compare nodes across - /// DAGs whose ids differ. - fn node_prices(cost: &PlanCost) -> Vec<(OperatorKind, ExecutionTiming, String, f64)> { - cost.per_node - .values() - .map(|node| { - ( - node.operator, - node.timing, - node.detail.clone(), - node.cost_per_second.unwrap(), - ) - }) - .collect() - } - - /// No fact bounds the rows of a scan whose source declares neither an - /// arrival rate (`rows_per_second`, or `series_count` with - /// `sample_interval`) nor `rows_held`: pricing refuses, naming the - /// source and the missing fact, instead of assuming a rate. - #[test] - fn a_scan_without_a_row_fact_is_refused_naming_its_source() { - let data = DataWorkload::default(); - let error = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap_err() - .1; - assert!( - matches!(&error, AnalyticalCostError::MissingSourceFact { source_name, .. } if source_name == "m"), - "{error}" - ); - assert!(error.to_string().contains("rows_held"), "{error}"); - } - - // #955 example 1: `rate(m[5m] offset 1h)` at query time reads the 5 - // minutes `(T - 65m, T - 60m]`, so it scans the samples `rate(m[5m])` - // scans but keeps and prices the 65 minutes of raw `m` `rate(m[65m])` - // keeps. - #[test] - fn an_offset_window_at_query_time_keeps_its_whole_raw_lookback() { - // The scan's samples read and raw data bytes kept, from its detail. - let scan_of = |query: &str| { - let dag = exact_dag(&[query], &data()); - let scan = node_of(&dag, is_scan); - let cost = price(&dag, &recurrences(1), &data(), &CostInputs::default()).unwrap(); - let detail = &cost.per_node[&scan].detail; - let kept_bytes: u64 = detail - .split("; keeps ") - .find_map(|part| { - part.split(" bytes of raw data on disk") - .next()? - .parse() - .ok() - }) - .unwrap_or_else(|| panic!("no raw data kept: {detail}")); - (scanned_samples(&cost, scan), kept_bytes) - }; - let (offset_samples, offset_kept) = scan_of("sum(rate(m[5m] offset 1h))"); - let (five_minutes_samples, five_minutes_kept) = scan_of("sum(rate(m[5m]))"); - let (_, sixty_five_minutes_kept) = scan_of("sum(rate(m[65m]))"); - assert_eq!(offset_samples, five_minutes_samples); - assert_eq!(offset_kept, sixty_five_minutes_kept); - assert_eq!(offset_kept, 13 * five_minutes_kept); - } - - /// Data at rest is bounded by the rows its source holds: a 5-minute - /// window of `m`, which holds 5,000 rows and declares no arrival rate, - /// reads at most those 5,000. - #[test] - fn rows_held_bound_a_scan_of_data_at_rest() { - let data = DataWorkload { - arrival: DataArrival::AtRest, - sources: [( - "m".to_string(), - SourceFacts { - rows_held: Some(5_000), - ..SourceFacts::default() - }, - )] - .into(), - }; - let cost = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - let details: Vec = node_prices(&cost) - .into_iter() - .map(|(_, _, d, _)| d) - .collect(); - assert!( - details.iter().any(|d| d.starts_with("scan 5000 samples;")), - "{details:?}" - ); - } - - /// With no declared `series_count`, a per-series reduction is priced at - /// one state per row read (each series has at least one), not at an - /// assumed series count: 1,000 rows per second over 5 minutes. - #[test] - fn undeclared_series_are_bound_by_the_rows_read() { - let data = DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1_000.0, - }, - sources: [("m".to_string(), SourceFacts::default())].into(), - }; - let cost = price( - &exact_dag(&["sum(rate(m[5m]))"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - let details: Vec = node_prices(&cost) - .into_iter() - .map(|(_, _, d, _)| d) - .collect(); - assert!( - details.iter().any(|d| d.contains("into 300000 states")), - "{details:?}" - ); - } - - /// Each scan is priced at its own source's rate, `series_count` series - /// each sampled every `sample_interval`, not at the whole workload's - /// `rows_per_second`: adding a query on a second metric, which a - /// runtime declares by raising `rows_per_second` by that metric's rate, - /// leaves the first metric's node prices unchanged. - #[test] - fn a_query_on_another_metric_leaves_a_metrics_node_prices_unchanged() { - // `a` and `b`: 10 000 series sampled every 100 ms, 100 000 rows/s each. - let ingesting = |rows_per_second| DataWorkload { - arrival: DataArrival::Ingesting { rows_per_second }, - sources: ["a", "b"] - .into_iter() - .map(|metric| { - let facts = SourceFacts { - series_count: Some(10_000), - sample_interval: Some(asap_types::workload::DurationMs(100)), - ..SourceFacts::default() - }; - (metric.to_string(), facts) - }) - .collect(), - }; - let settings = CostInputs::default(); - let on_a = "sum(sum_over_time(a[1m]))"; - let alone = ingesting(100_000.0); - let alone = price( - &exact_dag(&[on_a], &alone), - &recurrences(1), - &alone, - &settings, - ) - .unwrap(); - let joint = ingesting(200_000.0); - let joint = price( - &exact_dag(&[on_a, "sum(sum_over_time(b[1m]))"], &joint), - &recurrences(2), - &joint, - &settings, - ) - .unwrap(); - let mut joint_nodes = node_prices(&joint); - for node in node_prices(&alone) { - let found = joint_nodes.iter().position(|other| *other == node); - assert!(found.is_some(), "{node:?} not in {joint_nodes:#?}"); - joint_nodes.remove(found.unwrap()); - } - // Each metric's scan reads its own 100 000 rows per second. - assert_eq!( - joint.usage.scanned_bytes_per_second, - 2.0 * alone.usage.scanned_bytes_per_second - ); - } - - /// A partitioned sort and limit (`topk by (job)`) take their partition - /// count from `job`'s declared distinct values. With `job` undeclared, - /// no count is invented: each series is in one partition and each row - /// read in at most one, so the partitions are at most the series and - /// at most the rows the sort reads. - #[test] - fn a_partitioned_topk_has_one_partition_per_declared_label_value() { - let details = |declared: &[(&str, u64)], series: u64| { - let mut data = with_series(Some(series)); - data.sources.get_mut("m").unwrap().distinct_values_per_label = declared - .iter() - .map(|(label, count)| (label.to_string(), *count)) - .collect(); - let cost = price( - &exact_dag(&["topk by (job) (3, m)"], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - node_prices(&cost) - .into_iter() - .filter(|(operator, ..)| { - matches!(operator, OperatorKind::Sort | OperatorKind::Limit) - }) - .map(|(_, _, detail, _)| detail) - .collect::>() - }; - let declared = details(&[("job", 10)], 10_000); - assert!( - declared.iter().any(|d| d.contains(" in 10 partitions;")), - "{declared:?}" - ); - assert!( - declared.iter().any(|d| d.starts_with("limit to 30 rows;")), - "{declared:?}" - ); - // Undeclared: the 2,000 rows read, fewer than the 10,000 series. - let undeclared = details(&[("instance", 10)], 10_000); - assert!( - undeclared - .iter() - .any(|d| d.starts_with("sort 2000 rows in 2000 partitions;")), - "{undeclared:?}" - ); - assert!( - undeclared - .iter() - .any(|d| d.starts_with("limit to 2000 rows;")), - "{undeclared:?}" - ); - // 50 series hold at most 50 groups, declared or not. - let few = details(&[("job", 10_000)], 50); - assert!( - few.iter().any(|d| d.contains(" in 50 partitions;")), - "{few:?}" - ); - let few = details(&[], 50); - assert!( - few.iter().any(|d| d.contains(" in 50 partitions;")), - "{few:?}" - ); - } - - /// `x` holding 10,000 series over 4 regions, 25 services and 625 - /// endpoints, each endpoint in one service (the eval's RollupHeavy - /// `http`), with `(service, endpoint)`'s 625 combinations declared when - /// `declared_set`. - fn hierarchical_labels(declared_set: bool) -> DataWorkload { - let mut data = with_series(Some(10_000)); - let facts = data.sources.get_mut("x").unwrap(); - facts.distinct_values_per_label = [ - ("region".to_string(), 4), - ("service".to_string(), 25), - ("endpoint".to_string(), 625), - ] - .into(); - if declared_set { - facts.distinct_values_per_label_set = vec![LabelSetDistinctValues { - labels: vec!["service".to_string(), "endpoint".to_string()], - distinct_values: 625, - }]; - } - data - } - - /// A grouping is priced at the smallest bound the declared facts give: - /// the declared `(service, endpoint)` set's 625 combinations, not - /// 25 x 625 capped at the 10,000 series; a superset grouping at the set - /// times `region`'s 4; without the set, at the per-label product capped - /// at the series. A partitioned top-k takes the same count. - #[test] - fn a_grouping_is_priced_at_its_declared_label_set_count() { - let details = |query: &str, declared_set: bool| { - let data = hierarchical_labels(declared_set); - let cost = price( - &exact_dag(&[query], &data), - &recurrences(1), - &data, - &CostInputs::default(), - ) - .unwrap(); - node_prices(&cost) - .into_iter() - .map(|(_, _, detail, _)| detail) - .collect::>() - }; - let sum = |keys: &str| format!("sum by ({keys}) (sum_over_time(x[15m]))"); - for (query, declared_set, expected) in [ - (sum("service, endpoint"), true, "into 625 states"), - (sum("region, service, endpoint"), true, "into 2500 states"), - ( - sum("service, endpoint"), - false, - "Sum accumulator into 10000 states", - ), - (sum("region, service"), true, "into 100 states"), - ( - "topk by (service, endpoint) (3, sum_over_time(x[15m]))".to_string(), - true, - " in 625 partitions;", - ), - // A relabeled key no longer holds the declared set's values: - // `endpoint` now joins `region` and `endpoint` (up to 4 x 625). - ( - r#"sum by (service, endpoint) (label_join(sum_over_time(x[15m]), "endpoint", "-", "region", "endpoint"))"#.to_string(), - true, - "into 10000 states", - ), - // A binary operator pairs the labels of two rows: per-label only. - ( - "sum by (service, endpoint) (sum_over_time(x[15m]) / on (service) group_left (endpoint) sum_over_time(x[15m]))".to_string(), - true, - "into 10000 states", - ), - ] { - let details = details(&query, declared_set); - assert!( - details.iter().any(|d| d.contains(expected)), - "{query} (set declared: {declared_set}): {details:#?}" - ); - } - } - - /// A grouped root's `G` takes the declared label set's count too: the - /// set's 625 for `(service, endpoint)`, times `region`'s 4 for a - /// superset; without the set, the per-label product capped at the - /// series. - #[test] - fn values_per_result_is_bound_by_a_declared_label_set() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - for (query, declared_set, values) in [ - ("sum by (service, endpoint) (rate(x[5m]))", true, 625), - ( - "sum by (region, service, endpoint) (rate(x[5m]))", - true, - 2_500, - ), - ("sum by (service, endpoint) (rate(x[5m]))", false, 10_000), - ] { - assert_eq!( - values_of(&root(query), &hierarchical_labels(declared_set)), - Some(values), - "{query} (set declared: {declared_set})" - ); - } - } - - /// A grouped root's `G` is also bound by its keys' declared distinct - /// values, at most the series read: `by (job)` with 10 jobs returns at - /// most 10 values, also below a sort; a key without a declared count, a - /// relabeled key, `without (...)` and a top-k (several rows per group) - /// keep the series bound. - #[test] - fn values_per_result_is_bound_by_the_declared_distinct_values_of_its_keys() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let mut data = with_series(Some(10_000)); - data.sources.get_mut("x").unwrap().distinct_values_per_label = - [("job".to_string(), 10), ("instance".to_string(), 5_000)].into(); - for (query, values) in [ - ("sum by (job) (rate(x[5m]))", 10), - ("sum by (job, instance) (rate(x[5m]))", 10_000), - ("sum by (job, region) (rate(x[5m]))", 10_000), - ("sum without (instance) (rate(x[5m]))", 10_000), - ("topk by (job) (3, rate(x[5m]))", 10_000), - ("sum by (job) (rate(x[5m]) + rate(m[5m]))", 20_000), - // A relabeled key no longer holds the declared label's values. - ( - r#"sum by (job) (label_replace(rate(x[5m]), "job", "$1", "instance", "(.*)"))"#, - 10_000, - ), - ("sort_desc(sum by (job) (rate(x[5m])))", 10), - // `count_values("job", ...)` overwrites `job` with sample values. - ( - r#"sum by (job) (count_values by (job) ("job", rate(x[5m])))"#, - 10_000, - ), - ] { - assert_eq!(values_of(&root(query), &data), Some(values), "{query}"); - } - } - - /// `G` counts the values the root can return: an `or` returns the - /// values of both branches (here up to `series(x)` series plus the - /// groups of `x`), a constant vector one more, while a join or a fold - /// of one source counts it once. - #[test] - fn values_per_result_adds_the_branches_of_an_or() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let data = with_series(Some(10)); - for (query, values) in [ - ("rate(x[5m]) or sum without (instance) (rate(x[5m]))", 20), - ("rate(x[5m]) or vector(0)", 11), - ("rate(x[5m]) or rate(m[5m])", 20), - ("(rate(x[5m]) or vector(0)) or vector(1)", 12), - ("sum by (job) (rate(x[5m]) or rate(x[5m]))", 20), - ("rate(x[5m]) / rate(x[5m])", 10), - ("sum without (instance) (rate(x[5m])) * rate(x[5m])", 10), - ("rate(x[5m]) + rate(m[5m])", 20), - ] { - assert_eq!(values_of(&root(query), &data), Some(values), "{query}"); - } - } - - /// `vector(s)` returns one row, but the operators below it still read - /// their sources' rows: the input row count of - /// `vector(scalar(max(rate(x[5m]))))` counts the 10 series the `max` - /// folds (plus the one constant row), never shrinking going up the tree, - /// while its `G` is 1. - #[test] - fn a_vector_of_a_scalar_keeps_the_rows_read_below_it() { - let root = - |query: &str| QueryRoot::Operator(lower_promql(query, AccuracyTarget::max_error(0.01))); - let data = with_series(Some(10)); - let vector = root("vector(scalar(max(rate(x[5m]))))"); - assert_eq!(series_read(&vector, &data), Some(11)); - assert_eq!(values_of(&vector, &data), Some(1)); - assert_eq!(values_of(&root("vector(1)"), &data), Some(1)); - let or = root("rate(x[5m]) or vector(scalar(max(rate(x[5m]))))"); - assert_eq!(series_read(&or, &data), Some(21)); - } - - /// Deployment evidence that every output was measured exact. - struct EveryOutputMeasuredExact; - - impl AccuracyEvidenceProvider for EveryOutputMeasuredExact { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::exact("measured")) - } - } - - /// Deployment evidence of a small error in a fixed metric for every - /// output that is not exact. - struct EveryOutputMeasuredIn(AccuracyMetric); - - impl AccuracyEvidenceProvider for EveryOutputMeasuredIn { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::new( - self.0, - BoundExpr::Constant { value: 1e-6 }, - ProbabilityExpr::Constant { value: 1e-6 }, - vec![], - )) - } - } - - /// A bound stated in another metric misses the target however small it - /// is: a relative bound does not bound a top-k frequency's L1-scaled - /// error, so only the exact candidate stays valid; the same evidence in - /// the target's metric admits the sketches. - #[test] - fn a_bound_in_another_metric_misses_the_target() { - let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let target = AccuracyTarget::max_error(0.5) - .failure_probability(0.5) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let select = |metric| { - select_among_plans( - &candidates(), - &[entry_with(target.clone(), repeating(10_000))], - &counter, - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredIn(metric)), - ) - .unwrap() - }; - let relative = select(AccuracyMetric::Relative); - assert_eq!(relative.selected, "P1"); - let mut invalid: Vec<_> = relative - .rejected - .iter() - .filter(|r| r.cause.failed_a_check()) - .collect(); - invalid.sort_by(|a, b| a.id.cmp(&b.id)); - assert_eq!( - invalid.iter().map(|r| r.id.as_str()).collect::>(), - ["P2", "P3"] - ); - for rejected in invalid { - assert!( - rejected.reason.contains("no evidence that it meets"), - "{}", - rejected.reason - ); - } - let l1 = select(AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!( - l1.rejected.iter().all(|r| !r.cause.failed_a_check()), - "{:?}", - l1.rejected - ); - } - - /// A difference of two quantile estimates has no output guarantee, though - /// each estimate is certified: its summary candidates stay listed as - /// invalid with a reason. The deployment's empirical accuracy for the - /// output admits them. - #[test] - fn combined_estimates_need_evidence_about_the_output() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates( - "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", - &target, - ); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - - let builtin = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - for id in &summaries { - let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, RejectionCause::NoAccuracyGuarantee); - assert!( - rejection.reason.contains("no guarantee"), - "{}", - rejection.reason - ); - } - assert!(!summaries.contains(&builtin.selected.as_str())); - - let measured = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin().with_evidence(&EveryOutputMeasuredExact), - ) - .unwrap(); - for id in &summaries { - assert!(measured.costs.contains_key(*id), "{id} is valid"); - } - } - - /// Deployment evidence of a large absolute error (1.0) for every output. - struct EveryOutputMeasuredLoose; - - impl AccuracyEvidenceProvider for EveryOutputMeasuredLoose { - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - Some(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: 1.0 }, - ProbabilityExpr::Constant { value: 1e-6 }, - vec![], - )) - } - } - - /// A model that accepts every output directly, whatever its guarantee. - struct AcceptsEveryOutput; - - impl AccuracyModel for AcceptsEveryOutput { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } - } - - /// The hook overrides guarantees, not the executor's requirement on - /// Count-Min update weights: over samples of a metric declared a gauge, - /// the CmsWithHeap candidate (P2) is rejected as - /// `CountMinWeightsNotProvenNonNegative` under a model that accepts - /// every output, as under the built-in one. - #[test] - fn unproven_count_min_weights_reject_even_an_accepting_model() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - for accuracy in [ - &AcceptsEveryOutput as &dyn AccuracyModel, - &BuiltinAccuracyModel, - ] { - let selection = select_among_plans( - &candidates(), - &[entry_with(target.clone(), repeating(10_000))], - &gauge, - SelectionModels::builtin().with_accuracy(accuracy), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative, - "{}", - p2.reason - ); - assert!( - p2.reason.contains("not declared a counter"), - "{}", - p2.reason - ); - assert!(p2.cause.failed_a_check()); - } - } - - /// `IgnoreAccuracyGuarantees` (issue #774) does not override the - /// executor's requirement on Count-Min update weights either: over - /// samples of a gauge, P2's CmsWithHeap is still rejected as - /// `CountMinWeightsNotProvenNonNegative`. - #[test] - fn unproven_count_min_weights_reject_even_when_ignoring_guarantees() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - let settings = CostInputs { - ..CostInputs::default() - }; - let selection = select_among_plans( - &candidates(), - &[entry_with(target, repeating(10_000))], - &gauge, - SelectionModels::builtin() - .with_cost_inputs(&settings) - .with_accuracy(&asap_accuracy::IgnoreAccuracyGuarantees), - ) - .unwrap(); - let p2 = selection.rejected.iter().find(|r| r.id == "P2").unwrap(); - assert_eq!( - p2.cause, - RejectionCause::CountMinWeightsNotProvenNonNegative, - "{}", - p2.reason - ); - } - - /// With `IgnoreAccuracyGuarantees` (issue #774), each summary - /// candidate the built-in model rejects (no guarantee, or one that - /// misses the target) is valid, and its accepted output records the - /// cause the guarantee check would have rejected it with; an output - /// whose guarantee meets the target records none. - #[test] - fn ignoring_guarantees_accepts_and_marks_outputs_the_check_rejects() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE); - let entries = [entry_with(target, repeating(10_000))]; - let settings = CostInputs { - ..CostInputs::default() - }; - let builtin = - select_among_plans(&candidates(), &entries, &data(), SelectionModels::builtin()) - .unwrap(); - let ignoring = select_among_plans( - &candidates(), - &entries, - &data(), - SelectionModels::builtin() - .with_cost_inputs(&settings) - .with_accuracy(&asap_accuracy::IgnoreAccuracyGuarantees), - ) - .unwrap(); - let mut marked = 0; - for rejection in builtin.rejected.iter().filter(|r| { - matches!( - r.cause, - RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed - ) - }) { - let [accepted] = ignoring.accepted[&rejection.id].as_slice() else { - panic!("{} has one query", rejection.id) - }; - assert!(accepted.accepted_regardless_of_guarantee); - assert_eq!(accepted.guarantee_check_rejection, Some(rejection.cause)); - assert!(accepted.to_string().contains("ignoring"), "{accepted}"); - marked += 1; - } - assert!(marked > 0, "{:?}", builtin.rejected); - for (id, queries) in &builtin.accepted { - assert_eq!( - ignoring.accepted[id][0].guarantee_check_rejection, None, - "{id}" - ); - assert_eq!(queries[0].guarantee_check_rejection, None, "{id}"); - } - } - - /// `node` with every CmsWithHeap summary build replaced by a Hydra - /// Count-Min over the same update. - fn with_hydra_count_min(node: &Rc) -> Rc { - use asap_types::ir::schema::{HydraKind, HydraParams, SketchParams}; - let mut rebuilt = node.with_new_children(with_hydra_count_min).unwrap(); - if let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &mut rebuilt.operator { - if let FieldDataType::Sketch(kind, _) = family { - if let SketchParams::CmsWithHeap { width, depth, .. } = kind.params() { - let (width, depth) = (*width, *depth); - *family = FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - }, - ); - } - } - } - Rc::new(rebuilt) - } - - /// A Hydra Count-Min over unproven weights is rejected by its own name: - /// the reason says `HydraCms`, not `Cms`. - #[test] - fn unproven_hydra_count_min_weights_name_hydra_cms() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1); - let gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); - let mut p2 = candidates().into_iter().find(|c| c.id == "P2").unwrap(); - p2.roots = p2 - .roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator(with_hydra_count_min(node)), - other => other.clone(), - }) - .collect(); - let CheckFailure { cause, reason, .. } = - accuracy_check(&p2, &[target], &gauge, &SelectionModels::builtin()).unwrap_err(); - assert_eq!(cause, RejectionCause::CountMinWeightsNotProvenNonNegative); - assert!( - reason.contains("HydraCms needs non-negative update weights"), - "{reason}" - ); - assert!(reason.contains("not declared a counter"), "{reason}"); - } - - /// The model's hook has priority over every guarantee. The summary - /// candidates of a difference of quantile estimates have no theoretical - /// guarantee: with evidence that misses the target the built-in model - /// rejects them as `AccuracyTargetMissed`, and with no evidence as - /// `NoAccuracyGuarantee`; a model that accepts every output accepts them - /// in both cases, regardless of guarantee. - #[test] - fn the_accuracy_model_accepts_an_output_whatever_its_guarantee() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates( - "quantile_over_time(0.99, m[1m]) - quantile_over_time(0.5, m[1m])", - &target, - ); - let summaries: Vec<_> = candidates - .iter() - .filter(|c| has_summary_estimate(c)) - .map(|c| c.id.as_str()) - .collect(); - assert!(!summaries.is_empty()); - let select = |accuracy: &dyn AccuracyModel, evidence: &dyn AccuracyEvidenceProvider| { - select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::ABSOLUTE), - repeating(10_000), - )], - &data(), - SelectionModels::builtin() - .with_accuracy(accuracy) - .with_evidence(evidence), - ) - .unwrap() - }; - for (evidence, builtin_cause) in [ - ( - &EveryOutputMeasuredLoose as &dyn AccuracyEvidenceProvider, - RejectionCause::AccuracyTargetMissed, - ), - (&NoAccuracyEvidence, RejectionCause::NoAccuracyGuarantee), - ] { - let builtin = select(&BuiltinAccuracyModel, evidence); - let accepting = select(&AcceptsEveryOutput, evidence); - for id in &summaries { - let rejection = builtin.rejected.iter().find(|r| r.id == *id).unwrap(); - assert_eq!(rejection.cause, builtin_cause, "{}", rejection.reason); - assert!(accepting.costs.contains_key(*id), "{id} is valid"); - assert!( - accepting.accepted[*id][0].accepted_regardless_of_guarantee, - "{id}" - ); - } - } - } - - /// `candidates` with every state node (a summary build or an exact - /// aggregate) kept at ingestion time, so selection charges its kept - /// bytes every second (a `MatAtQuery` state is held only for one - /// evaluation). - fn keeping_state(mut candidates: Vec) -> Vec { - for candidate in &mut candidates { - for node in &mut candidate.dag.nodes { - if matches!( - node.payload, - Payload::ASAP(ASAPOp::SummaryAgg { .. }) - | Payload::NonASAP(NonASAPOp::Aggregate { .. }) - ) { - node.materialization = - Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); - } - } - } - candidates - } - - /// Kept exact quantile state keeps every value of its group: over one - /// group of 10 000 values it costs more than a quantile sketch sized for - /// ε = 0.01, δ = 0.01, which is selected. Over 100 groups of 100 values - /// (100 jobs declared), fewer than the sketch's `k`, the sketch keeps - /// every value too, in an item array grown past them, so its state is - /// no smaller than the exact state, which is selected. - #[test] - fn exact_quantile_state_costs_more_than_a_sketch_over_large_groups() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let mut data = data(); - data.sources.get_mut("x").unwrap().distinct_values_per_label = - [("job".to_string(), 100)].into(); - for (query, large_groups) in [ - ("quantile(0.99, rate(x[5m]))", true), - ("quantile by (job) (0.99, rate(x[5m]))", false), - ] { - let checked = target.clone().accuracy_metric(AccuracyMetric::Rank); - let candidates = keeping_state(all_candidates_sized(query, &target, &|roots| { - sized_as_pass1(&roots, std::slice::from_ref(&checked), &data) - })); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Rank), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let kept_bytes = |sketch: bool| { - let candidate = candidates - .iter() - .find(|c| { - has_summary_estimate(c) == sketch && selection.costs.contains_key(&c.id) - }) - .unwrap(); - selection.costs[&candidate.id].usage.average_memory_bytes - }; - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - match large_groups { - true => { - assert!(has_summary_estimate(selected), "{query}: {selection:?}"); - assert!(kept_bytes(true) < kept_bytes(false), "{query}"); - } - false => { - assert!(!has_summary_estimate(selected), "{query}: {selection:?}"); - assert!(kept_bytes(true) >= kept_bytes(false), "{query}"); - } - } - } - } - - /// Exact L2 state keeps one counter per distinct value of its group, so - /// it is priced like exact quantile state, which keeps every value. - #[test] - fn exact_l2_state_is_priced_like_exact_quantile_state() { - let aggregate_cost = |query: &str| { - let candidates = all_candidates(query, &AccuracyTarget::exact()); - let selection = select_among_plans( - &candidates, - &[entry(repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let exact = candidates - .iter() - .find(|c| !has_summary_estimate(c)) - .unwrap(); - selection.costs[&exact.id] - .per_node - .values() - .find(|n| n.detail.starts_with("hash aggregate")) - .unwrap() - .cost_per_second - }; - assert_eq!( - aggregate_cost("l2_over_time(x[5m])"), - aggregate_cost("quantile_over_time(0.99, x[5m])") - ); - } - - /// Kept exact distinct-count state keeps every item of its group: over - /// 10 000 groups of 300 rows it is smaller than a distinct-count sketch - /// sized for ε = 0.01 and is selected; over one group of 3 000 000 rows a - /// sketch is. - #[test] - fn exact_distinct_count_beats_a_sketch_over_small_groups() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let one_series = with_series(Some(1)); - for (data, sketch_wins) in [(data(), false), (one_series, true)] { - let candidates = keeping_state(all_candidates("distinct_over_time(x[5m])", &target)); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Relative), - repeating(10_000), - )], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - assert_eq!(has_summary_estimate(selected), sketch_wins, "{selection:?}"); - } - } - - /// The candidates of `query` that build a DDSketch. - fn ddsketch_candidates(candidates: &[PhysicalWorkloadPlan]) -> Vec { - candidates - .iter() - .filter(|c| { - c.dag.nodes.iter().any(|node| { - matches!(&node.payload, - Payload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind.algorithm() == &SketchAlgorithm::DDSketch) - }) - }) - .map(|c| c.id.clone()) - .collect() - } - - /// A per-series DDSketch quantile is within a relative error, which - /// carries through `avg` and `sum` only over non-negative values, and - /// through `max` whatever their sign. With `x` declared a gauge whose - /// samples are non-negative, the DDSketch candidate of - /// `avg by (region) (quantile_over_time(0.99, x[100s]))` (and of its `sum`) - /// is valid and accepted with a relative-error guarantee; without the - /// declaration it is rejected. Even `max` needs the primitive interpolated - /// quantile contract, which can fail across zero. - #[test] - fn a_non_negative_gauge_carries_a_ddsketch_relative_bound_through_avg() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - // An `offset` moves the window, not the samples' sign. - for (outer, window, needs_sign) in [ - ("avg", "x[100s]", true), - ("sum", "x[100s]", true), - ("max", "x[100s]", true), - ("avg", "x[100s] offset 1h", true), - ] { - let query = format!("{outer} by (region) (quantile_over_time(0.99, {window}))"); - let candidates = all_candidates(&query, &target); - let ddsketch = ddsketch_candidates(&candidates); - assert!(!ddsketch.is_empty(), "{query}"); - for values_non_negative in [true, false] { - let mut data = with_metric_types(data(), &[("x", PrometheusMetricType::Gauge)]); - data.sources.get_mut("x").unwrap().values_non_negative = values_non_negative; - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - for id in &ddsketch { - let accepted = selection.accepted.get(id); - if values_non_negative || !needs_sign { - let [accepted] = accepted.map(Vec::as_slice).unwrap_or_default() else { - panic!("{query}, {id} not valid: {:?}", selection.rejected); - }; - assert_eq!(accepted.accepted_metric, Some(AccuracyMetric::Relative)); - assert!(!accepted.accepted_regardless_of_guarantee); - } else { - assert!(accepted.is_none(), "{query}, {id}: {accepted:?}"); - let rejection = selection.rejected.iter().find(|r| &r.id == id).unwrap(); - assert!(rejection.cause.failed_a_check(), "{rejection:?}"); - } - } - } - } - } - - /// PromQL answers a quantile with φ outside [0, 1] as -Inf or +Inf, - /// which no sketch estimates and the executor's sketches refuse: such a - /// quantile gets no sketch candidate, so the exact plan is selected, and - /// a sketch never inherits its samples' sign for it, even over a gauge - /// declared non-negative. - #[test] - fn a_quantile_outside_the_unit_interval_is_answered_exactly() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - for query in [ - "avg by (region) (quantile_over_time(-1, x[100s]))", - "avg by (region) (quantile_over_time(2, x[100s]))", - "quantile_over_time(2, x[100s])", - "max by (region) (quantile_over_time(-1, x[100s]))", - ] { - let candidates = all_candidates(query, &target); - assert!( - candidates.iter().all(|c| !has_summary_estimate(c)), - "{query}: a sketch candidate is offered" - ); - let mut data = with_metric_types(data(), &[("x", PrometheusMetricType::Gauge)]); - data.sources.get_mut("x").unwrap().values_non_negative = true; - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ) - .unwrap(); - let selected = candidates - .iter() - .find(|c| c.id == selection.selected) - .unwrap(); - assert!( - !has_summary_estimate(selected), - "{query}: {}", - selection.selected - ); - } - } - - /// `topk(k, irate(m[w]))`, now a heavy-hitter ranking as `rate`'s is, - /// still plans: Stages 1-3 compose its candidates and select one. - #[test] - fn a_direct_irate_ranking_plans() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - for function in ["rate", "irate"] { - let query = format!("topk(10, {function}(data_total[5m]))"); - let candidates = all_candidates(&query, &target); - let data = with_metric_types(data(), &[("data_total", PrometheusMetricType::Counter)]); - let selection = select_among_plans( - &candidates, - &[entry_with(target.clone(), repeating(10_000))], - &data, - SelectionModels::builtin(), - ); - assert!(selection.is_ok(), "{query}: {selection:?}"); - } - } - - /// A candidate that ties the selected one is rejected for losing on - /// enumeration order, not as costlier. - #[test] - fn a_tie_is_reported_as_lost_on_enumeration_order() { - let mut candidates = candidates(); - candidates.truncate(1); - let mut twin = candidates[0].clone(); - twin.id = "P1-twin".into(); - candidates.push(twin); - let selection = select_among_plans( - &candidates, - &[entry(repeating(10_000))], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, "P1"); - let reason = &selection.rejected[0].reason; - assert!( - reason.contains("tied with the selected candidate P1") - && reason.contains("enumeration order"), - "{reason}" - ); - } - - /// At δ = 0.001 a KLL candidate meets the target and is valid. - #[test] - fn kll_candidate_meets_a_failure_probability_below_one_percent() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let candidates = all_candidates("quantile(0.99, rate(x[5m]))", &target); - let selection = select_among_plans( - &candidates, - &[entry_with( - target.clone().accuracy_metric(AccuracyMetric::Rank), - repeating(10_000), - )], - &data(), - SelectionModels::builtin(), - ) - .unwrap(); - let kll = candidates - .iter() - .find(|c| { - c.dag.nodes.iter().any(|n| { - matches!(&n.payload, Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. - }) if *kind.algorithm() == SketchAlgorithm::Kll) - }) - }) - .unwrap(); - assert!(selection.costs.contains_key(&kll.id), "{selection:?}"); - } - - /// `scalar(v)` inherits `v`'s guarantee; arithmetic over it composes - /// (`c · x` scales an absolute bound, `x − y` adds them); a rank error - /// does not compose (a rank interval counts as none); one over exact - /// operands only is exact. - #[test] - fn scalar_root_guarantees_compose_through_arithmetic() { - let node = lower_promql("sum(m)", AccuracyTarget::exact()); - let approximate = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let with = |guarantee: Option| { - ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(guarantee))) - }; - let doubled = |operand: ScalarExpr| { - QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(operand), - right: Box::new(ScalarExpr::literal_f64(2.0)), - semantics: ExprSemantics::Promql, - }) - }; - assert_eq!( - root_guarantee(&QueryRoot::Scalar(with(Some(approximate.clone())))), - Some(approximate.clone()) - ); - let scaled = root_guarantee(&doubled(with(Some(approximate.clone())))).unwrap(); - assert_eq!(scaled.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert!((scaled.spec.error.evaluate().unwrap() - 0.02).abs() < 1e-12); - assert_eq!(scaled.spec.failure_probability.evaluate(), Some(0.001)); - let difference = QueryRoot::Scalar(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Sub, - left: Box::new(with(Some(approximate.clone()))), - right: Box::new(with(Some(approximate.clone()))), - semantics: ExprSemantics::Promql, - }); - let sum = root_guarantee(&difference).unwrap(); - assert!((sum.spec.error.evaluate().unwrap() - 0.02).abs() < 1e-12); - let mut rank = approximate; - rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert_eq!(root_guarantee(&doubled(with(Some(rank)))), None); - assert!( - root_guarantee(&doubled(with(Some(AccuracyGuarantee::exact("exact"))))) - .is_some_and(|g| g.is_exact()) - ); - } - - /// Every candidate of the Stage 1 to Stage 3 run of one query. - fn run( - query: &str, - target: &AccuracyTarget, - data: &DataWorkload, - models: SelectionModels<'_>, - ) -> (Vec, WholePlanSelection) { - run_selecting(query, target, target, data, models) - } - - /// As [`run`], with summaries sized for `planned` and outputs checked - /// against `selected`. - fn run_selecting( - query: &str, - planned: &AccuracyTarget, - selected: &AccuracyTarget, - data: &DataWorkload, - models: SelectionModels<'_>, - ) -> (Vec, WholePlanSelection) { - let root = lower_promql(query, planned.clone()); - let entries = [entry_with(selected.clone(), QueryRecurrence::Unknown)]; - let roots = sized_as_pass1( - &[QueryRoot::Operator(root)], - std::slice::from_ref(planned), - data, - ); - let candidates = exhaustive_candidates(&roots, &entries, data); - let selection = select_among_plans(&candidates, &entries, data, models).unwrap(); - (candidates, selection) - } - - /// Every whole plan of `roots`: each combination of Pass 1's choices - /// composed, then every materialization assignment Stage 2 offers for - /// it, with unique ids. - fn exhaustive_candidates( - roots: &[QueryRoot], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - ) -> Vec { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(roots), - ) - .unwrap(); - let mut choices = vec![vec![]]; - for target in &inventory.targets { - choices = choices - .into_iter() - .flat_map(|choice: Vec| { - (0..target.choice_count()).map(move |index| { - let mut choice = choice.clone(); - choice.push(index); - choice - }) - }) - .collect(); - } - let mut candidates = Vec::new(); - for (index, choice) in choices.iter().enumerate() { - let Ok(composed) = compose_logical_candidate(&inventory, choice) else { - continue; - }; - let id = format!("L{}", index + 1); - let Ok(physical) = optimize_physical_asap(&id, &composed, entries, data, false) else { - continue; - }; - for (k, mut plan) in physical.into_iter().enumerate() { - plan.id = format!("P{}-{k}", index + 1); - plan.from_logical = id.clone(); - candidates.push(plan); - } - } - candidates - } - - fn sketches(candidate: &PhysicalWorkloadPlan) -> Vec { - candidate - .dag - .nodes - .iter() - .filter_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(*kind.algorithm()), - _ => None, - }) - .collect() - } - - fn reads_only(candidate: &PhysicalWorkloadPlan, algorithm: SketchAlgorithm) -> bool { - let sketches = sketches(candidate); - !sketches.is_empty() && sketches.iter().all(|a| *a == algorithm) - } - - fn output_guarantee(candidate: &PhysicalWorkloadPlan) -> Option { - root_guarantee(&candidate.roots[0]) - } - - /// `max by (job)` over per-series DDSketch quantiles keeps their relative - /// bound, and `2 · x` keeps it too (scaling preserves a relative error); - /// so those candidates meet the target. Over KLL (a rank error) the - /// result is a rank interval: no guarantee in the MVP. - #[test] - fn extremum_and_scaling_over_per_series_estimates_compose() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - for query in [ - "max by (job) (quantile_over_time(0.99, m[1m]))", - "2 * max by (job) (quantile_over_time(0.99, m[1m]))", - ] { - let (candidates, selection) = run( - query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty(), "{query}"); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ) - .expect(query); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{query}"); - assert!(g.spec.error.evaluate().unwrap() <= 0.01, "{query}"); - assert!(selection.costs.contains_key(&candidate.id), "{query}"); - } - for candidate in candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::Kll)) - { - assert_eq!( - bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin() - ), - None, - "{query}" - ); - } - } - } - - /// Negation, `abs` and `clamp` keep a DDSketch quantile's relative bound, - /// so those candidates are selectable; a shift by a constant, a - /// comparison (filter or `bool`) over the estimate compose to no - /// guarantee in the MVP, so those are not. - #[test] - fn lipschitz_functions_keep_and_shifts_and_comparisons_drop_a_relative_bound() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let q = "quantile_over_time(0.99, m[1m])"; - for (query, kept) in [ - (format!("-{q}"), true), - (format!("abs({q})"), true), - (format!("clamp({q}, 1, 100)"), true), - (format!("clamp_max({q}, 100)"), true), - (format!("{q} + 1"), false), - (format!("{q} > 5"), false), - (format!("{q} > bool 5"), false), - ] { - let (candidates, selection) = run( - &query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty(), "{query}"); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ); - if kept { - let g = g.expect(&query); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{query}"); - assert!(g.spec.error.evaluate().unwrap() <= 0.01, "{query}"); - assert!(selection.costs.contains_key(&candidate.id), "{query}"); - } else { - assert_eq!(g, None, "{query}"); - assert!(!selection.costs.contains_key(&candidate.id), "{query}"); - } - } - } - } - - /// A time-window fold over a subquery of estimates folds one estimate - /// per subquery step and series, so its failure probability is scaled - /// by the step count `⌈range / resolution⌉` (here 10), not by the input - /// series count. Without an explicit resolution the step count is not in - /// the query, and the fold has no guarantee. - #[test] - fn a_fold_over_subquery_estimates_counts_its_steps() { - let target = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let q = "quantile_over_time(0.99, m[1m])"; - let (candidates, selection) = run( - &format!("max_over_time({q}[10m:1m])"), - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty()); - for candidate in ddsketch { - let g = bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin(), - ) - .expect("a guarantee"); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(g.spec.error.evaluate().unwrap() <= 0.01); - let ProbabilityExpr::Scaled { count, .. } = &g.spec.failure_probability else { - panic!("{g:?}"); - }; - assert_eq!(count, &BoundExpr::Constant { value: 10.0 }); - assert!(selection.costs.contains_key(&candidate.id)); - } - let (candidates, _) = run( - &format!("max_over_time({q}[10m:])"), - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - for candidate in candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - { - assert_eq!( - bound_of( - &candidate.roots[0], - &non_negative_data(), - &SelectionModels::builtin() - ), - None - ); - } - } - - /// Counting estimates by a label keeps the count exact: the group keys - /// are labels, not the estimates (#669). - #[test] - fn counting_estimates_by_a_label_stays_exact() { - let target = AccuracyTarget::max_error(0.01); - let (candidates, _) = run( - "count by (le) (quantile_over_time(0.99, m[1m]))", - &target, - &data(), - SelectionModels::builtin(), - ); - // The inner quantile is a sketch; the count itself is exact. - let sketched: Vec<_> = candidates - .iter() - .filter(|c| { - let sketches = sketches(c); - !sketches.is_empty() - && sketches - .iter() - .all(|a| matches!(a, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) - }) - .collect(); - assert!(!sketched.is_empty()); - for candidate in sketched { - assert!( - output_guarantee(candidate).is_some_and(|g| g.is_exact()), - "{:?}", - sketches(candidate) - ); - } - } - - /// `avg by (job)` over per-series DDSketch quantiles has a relative bound - /// that needs the values' sign, a statistic no input supplies: it stays - /// symbolic and plan selection treats it as unmet. - #[test] - fn average_over_relative_estimates_needs_the_sign() { - use asap_logical_asap_optimization::test_support as pcc; - use asap_types::stages::HowToComputeAggregateMeasure; - let target = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "avg by (job) (quantile_over_time(0.99, m[1m]))", - target.clone(), - ); - let inventory = pcc::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - // Exercise the intended approximation directly; a joint optimizer may - // legitimately prefer an exact implementation or a different sharing - // topology, neither of which tests this compositional error contract. - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate - == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch) - }) - .unwrap_or(0) - }) - .collect(); - assert!(inventory - .targets - .iter() - .zip(&choices) - .any(|(target, &choice)| target.ways_to_compute[choice] - == HowToComputeAggregateMeasure::Summary(SketchAlgorithm::DDSketch))); - let outputs = pcc::compose_logical_candidate(&inventory, &choices).unwrap(); - let output = &outputs[0]; - let guarantee = root_guarantee(output).expect("a symbolic bound"); - assert!(guarantee.spec.error.has_unknown(), "{guarantee:?}"); - assert!(guarantee_check(0, output, &target, &data(), &SelectionModels::builtin()).is_err()); - } - - /// `max` folds one value per series, so its failure probability is - /// scaled by the symbolic input row count. Stage 3 binds it from the - /// series count of the source the query reads; without one, the target - /// is unmet. - #[test] - fn stage3_binds_the_input_row_count_from_the_data_workload() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Relative); - let query = "max by (job) (quantile_over_time(0.99, m[1m]))"; - let ddsketch = |candidates: &[PhysicalWorkloadPlan]| -> Vec { - candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .map(|c| c.id.clone()) - .collect() - }; - let (candidates, with) = run( - query, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids = ddsketch(&candidates); - assert!(!ids.is_empty()); - assert!(ids.iter().all(|id| with.costs.contains_key(id))); - let mut unknown = with_series(None); - for facts in unknown.sources.values_mut() { - facts.values_non_negative = true; - } - // Evaluate the same physical candidates: IDs from independent timed - // searches do not identify the same implementation or connections. - let without = select_among_plans( - &candidates, - &[entry_with(target, QueryRecurrence::Unknown)], - &unknown, - SelectionModels::builtin(), - ) - .unwrap(); - assert!(ids.iter().all(|id| !without.costs.contains_key(id))); - } - - /// Deployment contract: every operand lies in `[1, 1000]`. - struct BoundedOperands; - - impl AccuracyEvidenceProvider for BoundedOperands { - fn operand_domain(&self, _operand: &OperatorNode) -> Option { - Some(asap_accuracy::OperandDomain { - lower: 1.0, - upper: 1_000.0, - }) - } - } - - /// A ratio of two DDSketch quantiles has a closed-form relative bound, - /// `(a + b)/(1 − b)`, with no operand domain: each operand sized for - /// 0.01 gives about 0.0202, which meets a 0.05 target. - #[test] - fn ddsketch_ratio_meets_its_target_without_operand_domains() { - let planned = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let query = "quantile_over_time(0.99, m[1m]) / quantile_over_time(0.5, m[1m])"; - let (candidates, selection) = run_selecting( - query, - &planned, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids: Vec<_> = candidates - .iter() - .filter(|c| sketches(c) == [SketchAlgorithm::DDSketch, SketchAlgorithm::DDSketch]) - .map(|c| c.id.clone()) - .collect(); - // The Pass 1 candidate and its Pass 2 candidates sharing the input. - assert!(!ids.is_empty()); - assert!( - ids.iter().all(|id| selection.costs.contains_key(id)), - "{selection:?}" - ); - } - - /// `query` carrying `guarantee`, as a `scalar(…)` operand. - fn scalar_of(query: &str, guarantee: AccuracyGuarantee) -> ScalarExpr { - let node = lower_promql(query, AccuracyTarget::exact()); - ScalarExpr::PromqlScalarFromVector(Rc::new((*node).clone().with_guarantee(Some(guarantee)))) - } - - fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - - fn scalar_root(op: ArithmeticOpKind, left: ScalarExpr, right: ScalarExpr) -> QueryRoot { - QueryRoot::Scalar(ScalarExpr::Arithmetic { - op, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - }) - } - - fn norms(sources: &[(&str, f64)]) -> DataWorkload { - let mut data = data(); - for (source, l1) in sources { - data.sources - .entry(source.to_string()) - .or_default() - .stream_norms = Some(asap_types::workload::StreamNorms { - l1: Some(*l1), - l2: None, - }); - } - data - } - - /// A difference of two Count-Min estimates of the raw values of - /// different metrics is bound by each metric's own stream norm from the - /// data workload - /// (`0.01 · 100 + 0.02 · 1000`); a metric without a norm leaves the - /// bound unmet. - #[test] - fn stage3_binds_each_metric_stream_norm() { - // Count-Min summaries of each metric's raw value column. - let raw = |query: &str, g: AccuracyGuarantee| { - ScalarExpr::PromqlScalarFromVector( - asap_logical_asap_optimization::test_support::estimate_over( - asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary(query), - asap_types::ir::schema::SummaryInputExpr::Column( - asap_types::ir::scalar::ColumnRef::SampleValue, - ), - ), - g, - ), - ) - }; - let root = scalar_root( - ArithmeticOpKind::Sub, - raw( - "count by (job) (a)", - guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001), - ), - raw( - "count by (job) (b)", - guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.02, 0.001), - ), - ); - let models = SelectionModels::builtin(); - let both = bound_of(&root, &norms(&[("a", 100.0), ("b", 1_000.0)]), &models).unwrap(); - assert!( - (both.spec.error.evaluate().unwrap() - 21.0).abs() < 1e-9, - "{both:?}" - ); - let one = bound_of(&root, &norms(&[("a", 100.0)]), &models).unwrap(); - assert!(one.spec.error.has_unknown()); - } - - /// A stream norm is bound only for a summary of the source's raw value - /// column: a count over the same source (constant weight) needs the - /// source's row count, which nothing binds, and a summary of a computed - /// value (`m * 1000`) needs a norm nothing binds; both stay unmet. - #[test] - fn stage3_binds_a_stream_norm_only_for_the_raw_value_column() { - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::SummaryInputExpr; - let frequency = guarantee(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001); - let doubled = |summary: Rc| { - scalar_root( - ArithmeticOpKind::Mul, - ScalarExpr::literal_f64(2.0), - ScalarExpr::PromqlScalarFromVector( - asap_logical_asap_optimization::test_support::estimate_over( - summary, - frequency.clone(), - ), - ), - ) - }; - let models = SelectionModels::builtin(); - let data = norms(&[("m", 100.0)]); - let column = || SummaryInputExpr::Column(ColumnRef::SampleValue); - let raw = doubled(asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), - column(), - )); - let g = bound_of(&raw, &data, &models).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - 2.0).abs() < 1e-12, - "{g:?}" - ); - for unbound in [ - doubled(Rc::new( - asap_logical_asap_optimization::test_support::cms_summary("count by (job) (m)"), - )), - doubled(asap_logical_asap_optimization::test_support::weighted( - asap_logical_asap_optimization::test_support::cms_summary( - "count by (job) (m * 1000)", - ), - column(), - )), - ] { - let g = bound_of(&unbound, &data, &models).unwrap(); - assert!(g.spec.error.has_unknown(), "{g:?}"); - } - } - - /// A product and a quotient of two estimates are bound from the - /// deployment's operand domains (`[1, 1000]`): - /// `|x|·e_y + |y|·e_x + e_x·e_y` and `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))` - /// at the domain's extremes; without domains they stay unmet. A ratio of - /// relative errors needs no domain: `(a + b)/(1 − b)`. - #[test] - fn stage3_binds_products_and_quotients_from_operand_domains() { - let (x, y) = ( - scalar_of("sum(a)", guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001)), - scalar_of("sum(b)", guarantee(AccuracyMetric::ABSOLUTE, 0.02, 0.001)), - ); - let product = scalar_root(ArithmeticOpKind::Mul, x.clone(), y.clone()); - let quotient = scalar_root(ArithmeticOpKind::Div, x, y); - let bounded = SelectionModels::builtin().with_evidence(&BoundedOperands); - for (root, expected) in [ - (&product, 1_000.0 * 0.02 + 1_000.0 * 0.01 + 0.01 * 0.02), - ("ient, (1_000.0 * 0.02 + 1_000.0 * 0.01) / (1.0 - 0.02)), - ] { - let g = bound_of(root, &data(), &bounded).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - expected).abs() < 1e-9, - "{g:?}" - ); - assert!((g.spec.failure_probability.evaluate().unwrap() - 0.002).abs() < 1e-12); - let unbound = bound_of(root, &data(), &SelectionModels::builtin()).unwrap(); - assert!(unbound.spec.error.has_unknown(), "{unbound:?}"); - } - let ratio = scalar_root( - ArithmeticOpKind::Div, - scalar_of("sum(a)", guarantee(AccuracyMetric::Relative, 0.01, 0.0)), - scalar_of("sum(b)", guarantee(AccuracyMetric::Relative, 0.01, 0.0)), - ); - let g = bound_of(&ratio, &data(), &SelectionModels::builtin()).unwrap(); - assert!( - (g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-12, - "{g:?}" - ); - } - - /// A ratio divided by an exact vector, `(q99 / q50) / max_over_time(m)`, - /// keeps the inner ratio's relative bound, so the candidate is - /// selectable with no operand domain. - #[test] - fn a_scaled_ratio_keeps_its_relative_bound() { - let planned = AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative); - let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); - let query = "(quantile_over_time(0.99, m[1m]) / quantile_over_time(0.5, m[1m])) \ - / max_over_time(m[1m])"; - let (candidates, selection) = run_selecting( - query, - &planned, - &target, - &non_negative_data(), - SelectionModels::builtin(), - ); - let ids: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .map(|c| c.id.clone()) - .collect(); - assert!(!ids.is_empty()); - assert!( - ids.iter().all(|id| selection.costs.contains_key(id)), - "{selection:?}" - ); - } - - /// The input row count is bound only from the series count of the - /// sources the query reads: another source's declared series count - /// leaves it unbound. - #[test] - fn stage3_binds_only_the_series_count_of_the_sources_read() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.001); - let query = "max by (job) (quantile_over_time(0.99, m[1m]))"; - let mut other_source = data(); - other_source.sources.remove("m"); - let (candidates, selection) = - run(query, &target, &other_source, SelectionModels::builtin()); - let ddsketch: Vec<_> = candidates - .iter() - .filter(|c| reads_only(c, SketchAlgorithm::DDSketch)) - .collect(); - assert!(!ddsketch.is_empty()); - assert!(ddsketch - .iter() - .all(|c| !selection.costs.contains_key(&c.id))); - } - - /// `quantile_over_time(0.99, m[1m])` with its KLL candidate chosen, its - /// state materialized as `materialization`. - fn kll_dag(materialization: Materialization) -> PhysicalASAPDAG { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - let root = lower_promql( - "quantile_over_time(0.99, m[1m])", - AccuracyTarget::max_error(0.01).failure_probability(0.01), - ); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|t| { - t.ways_to_compute - .iter() - .position(|c| { - *c == asap_types::stages::HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::Kll, - ) - }) - .unwrap_or(0) - }) - .collect(); - let roots = asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - let QueryRoot::Operator(root) = &roots[0] else { - panic!("operator root") - }; - let state = OperatorNode::reachable(root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .expect("a KLL state"); - let mut assignment = - MaterializationAssignment::with_default(Materialization::NotMaterialized); - assignment.set(&state, materialization); - let timed = - apply_materialization_timings(root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids(&[ - QueryRoot::Operator(timed), - ]) - .unwrap() - .dag - } - - /// The physical DAG of `query`'s first per-computation candidates, - /// with nothing materialized. - fn first_candidate_dag(query: &str) -> PhysicalASAPDAG { - let root = lower_promql(query, AccuracyTarget::exact()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, &choice, - ) - .unwrap(); - optimize_physical_asap("L1", &roots, &[], &data(), false) - .unwrap() - .remove(0) - .dag - } - - /// A time shift is free and does not change what the scan below it - /// reads: the scan reads the window above the shift, as without it. A - /// time range is priced by the rows it keeps. - #[test] - fn time_shift_is_free_and_the_scan_reads_the_shifted_window() { - let every_minute = [repeating(60_000)]; - let plain = first_candidate_dag("sum_over_time(m[5m])"); - let shifted = first_candidate_dag("sum_over_time(m[5m] offset 1h)"); - let (plain_cost, shifted_cost) = ( - priced(&plain, &every_minute), - priced(&shifted, &every_minute), - ); - let cost_of = |dag: &PhysicalASAPDAG, cost: &PlanCost, pick: fn(&Payload) -> bool| { - cost.per_node[&node_of(dag, pick)].cost_per_second.unwrap() - }; - let is_shift = |p: &Payload| matches!(p, Payload::NonASAP(NonASAPOp::TimeShift { .. })); - let is_range = |p: &Payload| matches!(p, Payload::NonASAP(NonASAPOp::TimeRange { .. })); - assert_eq!(cost_of(&shifted, &shifted_cost, is_shift), 0.0); - assert!(close( - cost_of(&shifted, &shifted_cost, is_scan), - cost_of(&plain, &plain_cost, is_scan) - )); - assert!(close( - shifted_cost.cost_per_second.unwrap(), - plain_cost.cost_per_second.unwrap() - )); - let range = &plain_cost.per_node[&node_of(&plain, is_range)]; - assert!(range.detail.contains("keep"), "{}", range.detail); - } - - const AT_INGESTION: Materialization = - Materialization::MatAtIngestion(asap_types::physical::Medium::Memory); - const AT_QUERY: Materialization = - Materialization::MatAtQuery(asap_types::physical::Medium::Memory); - - fn node_of(dag: &PhysicalASAPDAG, pick: impl Fn(&Payload) -> bool) -> PhysicalASAPNodeId { - dag.nodes.iter().find(|n| pick(&n.payload)).unwrap().id - } - - fn is_scan(payload: &Payload) -> bool { - matches!(payload, Payload::NonASAP(NonASAPOp::Scan { .. })) - } - - fn is_build(payload: &Payload) -> bool { - matches!(payload, Payload::ASAP(ASAPOp::SummaryAgg { .. })) - } - - fn priced(dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]) -> PlanCost { - price(dag, recurrence, &data(), &CostInputs::default()).unwrap() - } - - /// The default settings with cost model 1's unit costs edited. - fn unit_costs(edit: impl FnOnce(&mut ResourceUnitCost)) -> CostInputs { - let mut prices = ResourceUnitCost::illustrative(); - edit(&mut prices); - CostInputs { - cost_model: CostModel::UsageOverTime(prices), - ..CostInputs::default() - } - } - - fn close(a: f64, b: f64) -> bool { - (a - b).abs() <= 1e-12 * a.abs().max(b.abs()).max(1e-300) - } - - /// An ingestion-time node is priced over one second of arriving rows, - /// once per second, whatever its readers' evaluation rate; a query-time - /// node scales with that rate. - #[test] - fn ingestion_time_node_is_priced_per_second_of_arriving_rows() { - let dag = kll_dag(AT_INGESTION); - let (fast, slow) = ( - priced(&dag, &[repeating(1_000)]), - priced(&dag, &[repeating(10_000)]), - ); - for id in [node_of(&dag, is_scan), node_of(&dag, is_build)] { - assert_eq!( - fast.per_node[&id].cost_per_second, - slow.per_node[&id].cost_per_second - ); - assert!(fast.per_node[&id].detail.contains("; x 1.0000/s")); - } - let estimate = dag.roots[0]; - assert!(close( - fast.per_node[&estimate].cost_per_second.unwrap(), - 10.0 * slow.per_node[&estimate].cost_per_second.unwrap() - )); - } - - /// Kept state is charged per byte per second: at ingestion time, twice - /// the state's bytes (the window being built and the completed one) for - /// every second. At query time the state is reused only within one - /// evaluation, so it is held only while that evaluation runs: its bytes - /// times the evaluation's latency, once per evaluation. State computed - /// and dropped within a query (`NotMaterialized`) is held while that - /// query runs too, and charged the same way. - #[test] - fn kept_state_is_charged_per_byte_per_second() { - let cost = |dag: &PhysicalASAPDAG, w| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); - price(dag, &recurrences(1), &data(), &settings).unwrap() - }; - for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { - let dag = kll_dag(materialization); - let build = node_of(&dag, is_build); - let (with, without) = (cost(&dag, 1.25e-7), cost(&dag, 0.0)); - let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = - &dag.nodes.iter().find(|n| n.id == build).unwrap().payload - else { - unreachable!() - }; - // One state per series: data() declares 10 000. At query time - // each sees its series' 12 samples: the 120 000 rows of its - // source's 1-min window (data()'s 10 000 rows per second over its - // five sources); at ingestion time its samples are unknown. - let samples = match materialization { - AT_INGESTION => StateInputRows::UNKNOWN, - _ => StateInputRows { - rows_per_state: Some(12), - non_negative: false, - }, - }; - let state = 10_000.0 * summary_shape(family, samples).1 as f64; - let byte_seconds = if materialization == AT_INGESTION { - 2.0 * state - } else { - let evaluation_s = with.usage.latency_per_query[0].total_ms() / 1_000.0; - assert!(evaluation_s > 0.0 && evaluation_s < 1.0, "{evaluation_s}"); - // Once per evaluation of the query repeating every 10 s. - assert!(with.per_node[&build].detail.contains("; x 0.1000/s")); - state * evaluation_s * 0.1 - }; - let term = with.per_node[&build].cost_per_second.unwrap() - - without.per_node[&build].cost_per_second.unwrap(); - assert!( - close(term, 1.25e-7 * byte_seconds), - "{materialization:?}: {term} vs {}", - 1.25e-7 * byte_seconds - ); - // The candidate's memory term covers every node's held memory, - // the build's included. - let candidate_term = with.cost_per_second.unwrap() - without.cost_per_second.unwrap(); - assert!(close( - candidate_term, - 1.25e-7 * with.usage.average_memory_bytes - )); - assert!(candidate_term >= term); - } - } - - /// Under cost model 1, memory is charged as bytes held per second - /// (average memory): the memory a computation holds while it runs (a - /// sketch being built, an exact quantile's values) counts for the time - /// it is held, once per evaluation. The memory price's term is exactly - /// the average memory, which is below the peak memory. - #[test] - fn memory_held_only_while_a_query_runs_is_charged_for_the_time_it_is_held() { - let memory_term = |dag: &PhysicalASAPDAG| { - let cost = |price_per_byte_second| { - let settings = - unit_costs(|c| c.cost_per_memory_byte_second = price_per_byte_second); - price(dag, &recurrences(1), &data(), &settings).unwrap() - }; - let (with, without) = (cost(1.0), cost(0.0)); - let term = with.cost_per_second.unwrap() - without.cost_per_second.unwrap(); - (term, with.usage) - }; - let mut dags: Vec<(String, PhysicalASAPDAG)> = [ - "quantile_over_time(0.99, m[1m])", - "quantile by (job) (0.99, m)", - ] - .into_iter() - .map(|query| { - let root = lower_promql(query, AccuracyTarget::exact()); - let dag = - optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - (query.to_string(), dag) - }) - .collect(); - for materialization in [AT_INGESTION, AT_QUERY, Materialization::NotMaterialized] { - dags.push((format!("{materialization:?}"), kll_dag(materialization))); - } - for (name, dag) in &dags { - let (term, usage) = memory_term(dag); - assert!( - close(term, usage.average_memory_bytes), - "{name}: {term} vs {}", - usage.average_memory_bytes - ); - assert!( - usage.peak_memory_bytes as f64 > usage.average_memory_bytes, - "{name}: {usage:?}" - ); - } - // Not materialized, nothing is kept, but the sketch being built is - // held while the query runs: the memory price adds to the cost. - let (term, _) = memory_term(&kll_dag(Materialization::NotMaterialized)); - assert!(term > 0.0, "{term}"); - } - - /// Kept exact state that grows with the data is charged by the values it - /// keeps: a materialized exact quantile keeps every value of its group, - /// a sum one value per group. - #[test] - fn kept_exact_state_is_charged_by_the_values_it_keeps() { - let kept = |query: &str| { - let root = lower_promql(query, AccuracyTarget::exact()); - let mut dag = - optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - let aggregate = dag - .nodes - .iter_mut() - .find(|n| matches!(n.payload, Payload::NonASAP(NonASAPOp::Aggregate { .. }))) - .unwrap(); - aggregate.materialization = AT_QUERY; - let id = aggregate.id; - let cost = |w| { - let settings = unit_costs(|c| c.cost_per_memory_byte_second = w); - // 10 jobs, so each group keeps many values. - let mut data = arriving(10_000.0); - data.sources.get_mut("m").unwrap().distinct_values_per_label = - [("job".to_string(), 10)].into(); - price(&dag, &recurrences(1), &data, &settings) - .unwrap() - .per_node[&id] - .cost_per_second - .unwrap() - }; - (cost(1.0) - cost(0.0)).round() as u64 - }; - let (quantile, sum) = ( - kept("quantile by (job) (0.99, m)"), - kept("sum by (job) (m)"), - ); - assert!(quantile > 10 * sum, "{quantile} vs {sum}"); - } - - /// A `NotMaterialized` node read by two queries is computed for each, at - /// each one's rate; `MatAtQuery`, it is computed once at their combined - /// rate, where queries at the same interval run together. - #[test] - fn rates_follow_per_consumer_charging() { - let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let one = optimize_physical_asap( - "L", - &[QueryRoot::Operator(root.clone())], - &[], - &data(), - false, - ) - .unwrap() - .remove(0) - .dag; - let mut two = optimize_physical_asap( - "L", - &[QueryRoot::Operator(root.clone()), QueryRoot::Operator(root)], - &[], - &data(), - false, - ) - .unwrap() - .remove(0) - .dag; - let without_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let total = |dag: &PhysicalASAPDAG, recurrence: &[QueryRecurrence]| { - price(dag, recurrence, &data(), &without_memory) - .unwrap() - .cost_per_second - .unwrap() - }; - let single = total(&one, &recurrences(1)); - let mixed = [repeating(10_000), repeating(20_000)]; - assert!(close(total(&two, &recurrences(2)), 2.0 * single)); - assert!(close(total(&two, &mixed), 1.5 * single)); - assert_eq!(two.roots[0], two.roots[1], "one root node for both queries"); - for node in &mut two.nodes { - node.materialization = AT_QUERY; - } - assert!(close(total(&two, &recurrences(2)), single)); - assert!(close(total(&two, &mixed), 1.5 * single)); - } - - /// Queries that refresh together at one interval and step evaluate at - /// the same instants: a node they share runs at the most steps among - /// them, not their sum. One over the last hour (61 steps) and one over - /// the last 10 min (11 steps), both every minute at 1-min steps, and a - /// third with no range (its one evaluation is the others' last step): - /// 61 evaluations a minute. A different step adds only the instants - /// the others do not evaluate at. - #[test] - fn readers_at_one_interval_and_step_share_their_steps() { - let every_minute = repeating(60_000); - let range = |span_ms: u64, step_ms: u64| Some(RangeEvaluation { step_ms, span_ms }); - let rate = |queries: &[Option]| { - evaluation_rate(queries.iter().map(|r| (&every_minute, *r)), 3_600.0).unwrap() - }; - let close = |a: f64, b: f64| (a - b).abs() < 1e-12; - assert!(close(rate(&[range(3_600_000, 60_000)]), 61.0 / 60.0)); - assert!(close( - rate(&[range(3_600_000, 60_000), range(600_000, 60_000), None]), - 61.0 / 60.0 - )); - assert!(close(rate(&[None, None]), 1.0 / 60.0)); - // A one-step range evaluates only at the request's end. - assert!(close( - rate(&[range(3_600_000, 60_000), range(0, 30_000)]), - 61.0 / 60.0 - )); - // A 30 s step over the last 10 min shares 11 of its 21 instants - // with the 1-min steps over the last hour: 71 distinct instants. - assert!(close( - rate(&[range(3_600_000, 60_000), range(600_000, 30_000)]), - 71.0 / 60.0 - )); - } - - /// Queries every 20 s at phases 0 and 5 s evaluate at different - /// instants: a node they share runs at both, twice as often as at one - /// phase. Phases 0 and 40 s are one phase on 20 s, and a query of - /// unknown phase runs with the known one. - #[test] - fn readers_at_different_phases_evaluate_apart() { - let at = |phase_ms: u64| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: asap_types::workload::RepetitionInterval(20_000), - evaluation_phase: asap_types::workload::TimestampMs(phase_ms), - }) - }; - let unknown = repeating(20_000); - let rate = |queries: &[&QueryRecurrence]| { - evaluation_rate(queries.iter().map(|r| (*r, None)), 3_600.0).unwrap() - }; - let close = |a: f64, b: f64| (a - b).abs() < 1e-12; - assert!(close(rate(&[&at(0), &at(5_000)]), 2.0 / 20.0)); - assert!(close(rate(&[&at(0), &at(40_000)]), 1.0 / 20.0)); - assert!(close(rate(&[&at(5_000), &unknown]), 1.0 / 20.0)); - assert!(close(rate(&[&at(0), &at(5_000), &unknown]), 2.0 / 20.0)); - } - - /// One-off and unknown recurrence are spread over the horizon: - /// invocations × per-evaluation cost / non_repeating_query_horizon_seconds. - #[test] - fn one_off_recurrence_is_spread_over_the_horizon() { - let root = lower_promql("sum by (job) (rate(m[1m]))", AccuracyTarget::exact()); - let dag = optimize_physical_asap("L", &[QueryRoot::Operator(root)], &[], &data(), false) - .unwrap() - .remove(0) - .dag; - let horizon = CostInputs::default().non_repeating_query_horizon_seconds; - assert_eq!(horizon, 3_600.0); - let per_evaluation = priced(&dag, &[repeating(1_000)]).cost_per_second.unwrap(); - let once = priced( - &dag, - &[QueryRecurrence::OneTime { - invocations: 3, - execute_at: None, - }], - ) - .cost_per_second - .unwrap(); - assert!(close(once, 3.0 * per_evaluation / horizon), "{once}"); - assert!(close( - priced(&dag, &[QueryRecurrence::Unknown]) - .cost_per_second - .unwrap(), - per_evaluation / horizon - )); - } - - /// Settings outside their domain are an error, not a price, checked once - /// before any candidate: a price the cost model reads that is negative, - /// the model's prices all 0 (cost model 1 with only provisioned prices - /// set), or a horizon that is not positive. A price the model does not - /// read is not checked. - #[test] - fn invalid_settings_are_rejected() { - let candidates = tumbling_window_candidates(10_000.0); - let select = |settings: &CostInputs| { - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(settings), - ) - }; - let only_provisioned = ResourceUnitCost { - cost_per_cpu_op: 0.0, - cost_per_scanned_byte: 0.0, - cost_per_memory_byte_second: 0.0, - cost_per_disk_byte_second: 0.0, - ..ResourceUnitCost::illustrative() - }; - for settings in [ - unit_costs(|c| c.cost_per_memory_byte_second = -1.0), - under(CostModel::UsageOverTime(only_provisioned.clone())), - CostInputs { - non_repeating_query_horizon_seconds: 0.0, - ..CostInputs::default() - }, - ] { - assert!( - matches!( - select(&settings), - Err(ErrorSelectingPlan::InvalidSettings(_)) - ), - "{settings:?}" - ); - } - assert!(select(&under(CostModel::ProvisionedCapacity(only_provisioned))).is_ok()); - assert!(select(&unit_costs( - |c| c.cost_per_provisioned_cpu_core_second = -1.0 - )) - .is_ok()); - } - - // ── Stage 2 materialization and the latency check ──────────────────── - - const MINUTE: u32 = 60_000; - - /// The p99 target [`tumbling_window_candidates`] are sized for. - fn p99_target() -> AccuracyTarget { - AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank) - } - - /// A predictable p99 query evaluated every minute, with no latency bound. - fn every_minute() -> QueryWorkloadEntry { - QueryWorkloadEntry { - predictability: asap_types::workload::Predictability::Predictable { known_at: None }, - ..entry_with(p99_target(), repeating(MINUTE)) - } - } - - /// [`data`] with each test source arriving at `rows_per_s`: pricing - /// gives each source its series share of the workload's rate. - fn arriving(rows_per_s: f64) -> DataWorkload { - DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: rows_per_s * TEST_SOURCES.len() as f64, - }, - ..data() - } - } - - /// The physical candidates of a per-series p99 over 5 min with its KLL - /// in five 1-min tumbling windows, every minute over data arriving at - /// `rows_per_s`: rebuilt at query time (B2), then maintained at - /// ingestion time (B1). - fn tumbling_window_candidates(rows_per_s: f64) -> Vec { - let candidates = - tumbling_window_candidates_for(rows_per_s, &DeploymentCapabilities::default()); - assert_eq!(candidates.len(), 2); - candidates - } - - /// [`tumbling_window_candidates`] for a deployment with `deployment`'s - /// capabilities: B2, B1, then, when it keeps query-time results, B3 - /// (built at query time and kept across evaluations). - fn tumbling_window_candidates_for( - rows_per_s: f64, - deployment: &DeploymentCapabilities, - ) -> Vec { - let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let kll = inventory.targets[0] - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - asap_types::stages::HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap(); - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, - &[kll], - ) - .unwrap(); - let roots = sized_as_pass1(&roots, &[p99_target()], &arriving(rows_per_s)); - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &[Some(u64::from(MINUTE))], - ); - let roots = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap() - .pop() - .unwrap() - .roots - .unwrap(); - let mut candidates = optimize_physical_asap( - "L1", - &roots, - &[every_minute()], - &arriving(rows_per_s), - deployment.query_time_results_retained, - ) - .unwrap(); - for (k, c) in candidates.iter_mut().enumerate() { - c.id = format!("P1-m{k}"); - } - candidates - } - - /// The raw data samples the scan `scan` reads per computation, from its - /// priced detail (`scan samples; ...`). - fn scanned_samples(cost: &PlanCost, scan: PhysicalASAPNodeId) -> u64 { - let detail = &cost.per_node[&scan].detail; - detail - .strip_prefix("scan ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|rows| rows.parse().ok()) - .unwrap_or_else(|| panic!("not a scan: {detail}")) - } - - /// A deployment that keeps query-time results across evaluations. - fn keeping_query_time_results() -> DeploymentCapabilities { - DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - } - } - - /// `query_time_results_retained` gates keeping a query-time result across - /// evaluations, and only that. Without it, Stage 2 offers the 1-min - /// tumbling windows of a query repeating every minute rebuilt (B2) or maintained - /// (B1); with it, also built at query time and kept (B3, - /// `MatAtQueryKeptAcrossEvaluations`). B3 is computed once per - /// evaluation, reads one tumbling window (1 min) of raw data, keeps that much raw - /// data on disk, and holds its 4 older tumbling windows in memory between - /// evaluations (on disk, if it were placed there). The same chain - /// `MatAtQuery` is reused only within one evaluation: the scan reads the - /// 5-min window and nothing is kept between evaluations. - #[test] - fn query_time_results_retained_keeps_a_tumbling_window_chain_across_evaluations() { - assert_eq!(tumbling_window_candidates(10_000.0).len(), 2); - let candidates = tumbling_window_candidates_for(10_000.0, &keeping_query_time_results()); - assert_eq!(candidates.len(), 3); - let b2 = &candidates[0].dag; - let b3 = &candidates[2].dag; - let chain = node_of(b3, is_chain); - let with_chain = |materialization: Materialization| { - let mut dag = b3.clone(); - for node in dag.nodes.iter_mut().filter(|n| n.id == chain) { - node.materialization = materialization; - } - dag - }; - assert_eq!( - *b3, - with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Memory - )) - ); - let recurrence = [repeating(MINUTE)]; - let settings = CostInputs::default(); - let kept = price(b3, &recurrence, &arriving(10_000.0), &settings).unwrap(); - let rebuilt = price(b2, &recurrence, &arriving(10_000.0), &settings).unwrap(); - let scan = node_of(b3, is_scan); - assert_eq!(scanned_samples(&kept, scan), 10_000 * 60); - assert_eq!( - scanned_samples(&rebuilt, node_of(b2, is_scan)), - 10_000 * 300 - ); - assert_eq!(kept.usage.disk_bytes * 5, rebuilt.usage.disk_bytes); - let Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { family, .. }) = - &b3.nodes.iter().find(|n| n.id == chain).unwrap().payload - else { - unreachable!() - }; - // Each tumbling window sees its series' 60 samples: the one minute of 10 000 - // rows per second over 10 000 series it reads. - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let (_, state_bytes) = summary_shape(family, samples); - let chain_cost = &kept.per_node[&chain]; - assert!(chain_cost.detail.contains("; x 0.0167/s"), "{chain_cost:?}"); - // One state per series: arriving() declares 10 000. - let between = 4 * 10_000 * state_bytes; - assert!(chain_cost - .detail - .contains(&format!("keeps {between} bytes across evaluations"))); - // On disk, the same N − 1 older tumbling windows are kept. - let on_disk = with_chain(Materialization::MatAtQueryKeptAcrossEvaluations( - Medium::Disk, - )); - let on_disk = price(&on_disk, &recurrence, &arriving(10_000.0), &settings).unwrap(); - assert_eq!( - on_disk.usage.disk_bytes, - kept.usage.disk_bytes + between, - "{on_disk:?}" - ); - // Shared within one evaluation, nothing is kept between them. - let shared = with_chain(Materialization::MatAtQuery(Medium::Memory)); - let within_one = price(&shared, &recurrence, &arriving(10_000.0), &settings).unwrap(); - assert_eq!(scanned_samples(&within_one, scan), 10_000 * 300); - assert!(!within_one.per_node[&chain] - .detail - .contains("across evaluations")); - assert!(within_one.usage.average_memory_bytes < kept.usage.average_memory_bytes); - } - - fn is_chain(payload: &Payload) -> bool { - matches!( - payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - } - - /// #509 Example 4, B1: maintained at ingestion time, a chain of N tumbling windows - /// pays the update of the newest tumbling window only, one second of arriving rows - /// once per second, and keeps N + 1 tumbling windows (the N the query merges and - /// the one being built) per series. - #[test] - fn a_chain_at_ingestion_pays_the_newest_tumbling_window_and_keeps_n_plus_one_tumbling_windows() - { - let candidates = tumbling_window_candidates(10_000.0); - let dag = &candidates[1].dag; - let chain = node_of(dag, is_chain); - let Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_count, - .. - }) = &dag.nodes.iter().find(|n| n.id == chain).unwrap().payload - else { - unreachable!() - }; - assert_eq!(*tumbling_window_count, 5); - // Each tumbling window sees its series' 60 samples: one minute of 10 000 rows - // per second over 10 000 series. - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let (depth, state_bytes) = summary_shape(family, samples); - let recurrence = [repeating(MINUTE)]; - let without_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let work = &price(dag, &recurrence, &arriving(10_000.0), &without_memory) - .unwrap() - .per_node[&chain]; - assert!(work.detail.contains("; x 1.0000/s"), "{work:?}"); - assert!( - close( - work.cost_per_second.unwrap(), - 10_000.0 * (depth + 1.0) * 1e-6 - ), - "{work:?}" - ); - let with_memory = &price( - dag, - &recurrence, - &arriving(10_000.0), - &CostInputs::default(), - ) - .unwrap() - .per_node[&chain]; - let kept = 6 * 10_000 * state_bytes; - assert!( - close( - with_memory.cost_per_second.unwrap() - work.cost_per_second.unwrap(), - 1.25e-7 * kept as f64 - ), - "{with_memory:?}" - ); - assert!(with_memory.detail.contains(&format!("keeps {kept} bytes"))); - } - - /// The latency check (S6, sketch-bench cost model §5): a query's - /// latency is its query-time work on one core. Rebuilding the tumbling windows at - /// query time (B2) takes longer than merging the maintained ones (B1); - /// with a bound between the two, B2 is rejected as invalid with its - /// latency and B1 is selected. - #[test] - fn a_candidate_over_a_querys_latency_bound_is_rejected_with_its_latency() { - let candidates = tumbling_window_candidates(10_000.0); - let latency = |c: &PhysicalWorkloadPlan| { - let cost = price( - &c.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &CostInputs::default(), - ) - .unwrap(); - cost.usage.latency_per_query[0] - }; - let (b2, b1) = (latency(&candidates[0]), latency(&candidates[1])); - assert_eq!((b2.compaction_ms, b1.compaction_ms), (0.0, 0.0)); - assert!(b1.query_ms < b2.query_ms, "{b1:?} vs {b2:?}"); - let bound = (b1.query_ms + b2.query_ms) / 2.0; - let mut bounded = every_minute(); - bounded.requirements.max_latency_ms = Some(bound); - let selection = select_among_plans( - &candidates, - &[bounded], - &arriving(10_000.0), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[1].id); - let rejected = &selection.rejected[0]; - assert_eq!(rejected.id, candidates[0].id); - assert_eq!(rejected.cause, RejectionCause::LatencyBoundExceeded); - assert!( - rejected.reason.starts_with(&format!( - "q1: latency {:.1} ms per evaluation (compaction 0.0 ms + query", - b2.total_ms() - )) && rejected - .reason - .ends_with(&format!("is over its {bound} ms bound")), - "{}", - rejected.reason - ); - assert!(!selection.costs.contains_key(&candidates[0].id)); - } - - /// A query evaluated every minute whose evaluation takes longer than a - /// minute never keeps up, so even without a latency bound the - /// candidate is invalid. Over data fast enough that rebuilding the - /// tumbling windows at query time (B2) takes over a minute while merging the - /// maintained ones (B1) does not, B2 is rejected with its latency and - /// the interval, and B1 is selected. - #[test] - fn a_candidate_slower_than_a_querys_repeat_interval_is_rejected_without_a_bound() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let latency = |c: &PhysicalWorkloadPlan| { - price( - &c.dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap() - .usage - .latency_per_query[0] - }; - let (b2, b1) = (latency(&candidates[0]), latency(&candidates[1])); - assert!( - b1.total_ms() < f64::from(MINUTE) && b2.total_ms() > f64::from(MINUTE), - "{b1:?} vs {b2:?}" - ); - let unbounded = every_minute(); - assert_eq!(unbounded.requirements.max_latency_ms, None); - let selection = select_among_plans( - &candidates, - &[unbounded], - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[1].id); - let rejected = &selection.rejected[0]; - assert_eq!(rejected.id, candidates[0].id); - assert_eq!(rejected.cause, RejectionCause::LatencyOverRepeatInterval); - assert!(rejected.cause.failed_a_check()); - assert_eq!( - rejected.reason, - format!( - "q1: latency {:.1} ms per evaluation (compaction {:.1} ms + query {:.1} ms) is \ - over its {MINUTE} ms repeat interval, so its evaluations cannot keep up", - b2.total_ms(), - b2.compaction_ms, - b2.query_ms - ) - ); - assert!(!selection.costs.contains_key(&candidates[0].id)); - } - - /// With no repeat interval (a one-time query) there is nothing to keep - /// up with: the slow candidate stays valid and is selected when it is - /// cheaper. - #[test] - fn a_one_time_query_has_no_repeat_interval_to_keep_up_with() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let once = QueryWorkloadEntry { - recurrence: QueryRecurrence::OneTime { - invocations: 1, - execute_at: None, - }, - ..every_minute() - }; - let selection = select_among_plans( - &candidates, - &[once], - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap(); - // Run once an hour, rebuilding at query time is the cheaper plan, - // over a minute per evaluation or not. - assert_eq!(selection.selected, candidates[0].id); - assert_eq!(selection.rejected.len(), 1, "{:?}", selection.rejected); - assert_eq!(selection.rejected[0].id, candidates[1].id); - assert_eq!(selection.rejected[0].cause, RejectionCause::Costlier); - } - - /// A query over both its latency bound and its repeat interval is - /// reported over its bound: the bound is the user's own requirement. - /// The interval is also checked for a repetition at a declared phase. - #[test] - fn a_broken_latency_bound_is_reported_before_the_repeat_interval() { - let rows_per_s = 1e9; - let candidates = tumbling_window_candidates(rows_per_s); - let mut bounded = QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: asap_types::workload::RepetitionInterval(MINUTE), - evaluation_phase: asap_types::workload::TimestampMs(0), - }), - ..every_minute() - }; - let select = |entry: &QueryWorkloadEntry| { - select_among_plans( - &candidates, - std::slice::from_ref(entry), - &arriving(rows_per_s), - SelectionModels::builtin(), - ) - .unwrap() - }; - let unbounded = select(&bounded); - assert_eq!( - unbounded.rejected[0].cause, - RejectionCause::LatencyOverRepeatInterval - ); - bounded.requirements.max_latency_ms = Some(f64::from(MINUTE)); - let selection = select(&bounded); - assert_eq!(selection.selected, candidates[1].id); - assert_eq!(selection.rejected[0].id, candidates[0].id); - assert_eq!( - selection.rejected[0].cause, - RejectionCause::LatencyBoundExceeded - ); - } - - /// When a maintained state's ingestion work needs more than one core, - /// its `k` workers' partial copies of the newest window are merged - /// before the query reads it: `(k − 1)` merges of one operation for - /// each of its states per window, on the query's chain. - #[test] - fn ingestion_work_over_one_core_adds_the_newest_windows_compaction() { - let latency = |rows_per_s: f64| { - let candidates = tumbling_window_candidates(rows_per_s); - price( - &candidates[1].dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap() - .usage - .latency_per_query[0] - }; - assert_eq!(latency(10_000.0).compaction_ms, 0.0); - let fast = latency(1e10); - // 10,000 series, one state each per window, 1 ns per merge. - let merges = fast.compaction_ms / (10_000.0 * 1e-6); - assert!( - merges >= 1.0 && (merges - merges.round()).abs() < 1e-6, - "{fast:?}" - ); - } - - /// The newest window's compaction merges the `k` workers' partial - /// sketches with the state's measured query-time merge when the table - /// has one: each of the `(k − 1)·I` merges takes the measured 1,200 ns - /// instead of one operation. - #[test] - fn compaction_prices_merges_from_the_measured_merge() { - let candidates = tumbling_window_candidates(1e10); - let dag = &candidates[1].dag; - let kll = dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => Some(kind.params().clone()), - _ => None, - }) - .expect("a KLL build"); - let latency = price( - dag, - &[repeating(MINUTE)], - &arriving(1e10), - &measured(vec![sketch_entry(OperatorKind::SummaryMerge, kll, 1200.0)]), - ) - .unwrap() - .usage - .latency_per_query[0]; - assert!(latency.compaction_ms > 0.0, "{latency:?}"); - let merges = latency.compaction_ms / 1200e-6; - assert!( - merges >= 1.0 && (merges - merges.round()).abs() < 1e-6, - "{latency:?}" - ); - } - - // ── Cost models ────────────────────────────────────────────────────── - - /// [`CostInputs`] with `cost_model`. - fn under(cost_model: CostModel) -> CostInputs { - CostInputs { - cost_model, - ..CostInputs::default() - } - } - - /// The candidate's usage is what its nodes are charged for: priced at - /// one unit for a single resource under cost model 1 (CPU, scanned - /// bytes, memory byte-seconds or disk), the nodes' prices add up to the - /// candidate's usage of it. - #[test] - fn node_prices_add_up_to_the_candidates_usage() { - let only = |set: fn(&mut ResourceUnitCost)| { - let mut prices = ResourceUnitCost { - cost_per_cpu_op: 0.0, - cost_per_scanned_byte: 0.0, - cost_per_memory_byte_second: 0.0, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - }; - set(&mut prices); - CostInputs { - cost_model: CostModel::UsageOverTime(prices), - ..CostInputs::default() - } - }; - // Each resource's unit price and the candidate's usage of it. - type Resource = (fn(&mut ResourceUnitCost), fn(&ResourceUsage) -> f64); - let resources: [Resource; 4] = [ - (|p| p.cost_per_cpu_op = 1.0, |u| u.cpu_ops_per_second), - ( - |p| p.cost_per_scanned_byte = 1.0, - |u| u.scanned_bytes_per_second, - ), - ( - |p| p.cost_per_memory_byte_second = 1.0, - |u| u.average_memory_bytes, - ), - ( - |p| p.cost_per_disk_byte_second = 1.0, - |u| u.disk_bytes as f64, - ), - ]; - for candidate in tumbling_window_candidates(10_000.0) { - for (set, used) in resources { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &only(set), - ) - .unwrap(); - let charged: f64 = cost - .per_node - .values() - .filter_map(|n| n.cost_per_second) - .sum(); - assert!( - close(charged, used(&cost.usage)), - "{}: {charged} vs {}", - candidate.id, - used(&cost.usage) - ); - assert_eq!(cost.usage.latency_per_query.len(), 1); - } - } - } - - /// Cost model 2 prices the candidate's peaks per second: provisioned - /// memory, cores and disk; its nodes carry no price. B1 (maintained - /// tumbling windows) needs a core for ingestion and one for the query; B2 (rebuilt - /// at query time) one. - #[test] - fn provisioned_capacity_prices_peak_memory_cores_and_disk() { - let prices = ResourceUnitCost { - cost_per_disk_byte_second: 1e-9, - ..ResourceUnitCost::illustrative() - }; - let settings = under(CostModel::ProvisionedCapacity(prices.clone())); - let candidates = tumbling_window_candidates(10_000.0); - let cores: Vec = candidates - .iter() - .map(|candidate| { - let cost = price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings, - ) - .unwrap(); - let usage = &cost.usage; - let expected = prices.cost_per_provisioned_memory_byte_second - * usage.peak_memory_bytes as f64 - + prices.cost_per_provisioned_cpu_core_second * f64::from(usage.peak_cpu_cores) - + prices.cost_per_disk_byte_second * usage.disk_bytes as f64; - assert_eq!(cost.cost_per_second, Some(expected)); - assert!(cost.per_node.values().all(|n| n.cost_per_second.is_none())); - usage.peak_cpu_cores - }) - .collect(); - assert_eq!(cores, [1, 2]); - } - - /// Cost model 3 has no prices; a candidate over the cluster's cores is - /// invalid with the reason, and the rest are chosen by the objective. - #[test] - fn cluster_capacity_rejects_a_candidate_over_its_cores() { - let limits = |cpu_cores| ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores, - objective: ClusterObjective::LowestLatency, - }; - let candidates = tumbling_window_candidates(10_000.0); - let select = |cpu_cores| { - let settings = under(CostModel::ClusterCapacity(limits(cpu_cores))); - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap() - }; - let unlimited = select(None); - assert_eq!(unlimited.selected, candidates[1].id, "B1 merges faster"); - assert!(unlimited - .costs - .values() - .all(|c| c.cost_per_second.is_none())); - let one_core = select(Some(1)); - assert_eq!(one_core.selected, candidates[0].id); - let b1 = one_core - .rejected - .iter() - .find(|r| r.id == candidates[1].id) - .unwrap(); - assert_eq!(b1.cause, RejectionCause::OverClusterCapacity); - assert_eq!(b1.reason, "2 CPU cores at peak is over the cluster's 1"); - } - - /// Raw data kept on disk for query-time scans is counted once per - /// source, for the longest window read: two queries over `m` keep its - /// last 5 minutes, as the 5-minute query alone does. - #[test] - fn raw_data_on_disk_is_the_longest_window_per_source() { - let disk = |queries: &[&str]| { - let roots: Vec<_> = queries - .iter() - .map(|q| QueryRoot::Operator(lower_promql(q, AccuracyTarget::exact()))) - .collect(); - let dag = optimize_physical_asap("L", &roots, &[], &data(), false) - .unwrap() - .remove(0) - .dag; - priced(&dag, &recurrences(queries.len())).usage.disk_bytes - }; - let five_minutes = disk(&["sum_over_time(m[5m])"]); - assert!(five_minutes > disk(&["sum_over_time(m[1m])"])); - assert_eq!( - disk(&["sum_over_time(m[1m])", "sum_over_time(m[5m])"]), - five_minutes - ); - } - - /// Cost model 3's disk limit: B2 reads its 5-minute window of raw data - /// at query time and keeps it on disk; B1 keeps none. A cluster with - /// less disk than B2's raw data leaves only B1. - #[test] - fn cluster_capacity_rejects_a_candidate_over_its_disk() { - let candidates = tumbling_window_candidates(10_000.0); - let select = |disk_bytes| { - let settings = under(CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes, - cpu_cores: None, - objective: ClusterObjective::LeastMemory, - })); - select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap() - }; - let unlimited = select(u64::MAX); - let raw = unlimited.costs[&candidates[0].id].usage.disk_bytes; - assert!(raw > 0); - assert_eq!(unlimited.costs[&candidates[1].id].usage.disk_bytes, 0); - assert_eq!(unlimited.selected, candidates[0].id, "B2 needs less memory"); - let small = select(raw - 1); - assert_eq!(small.selected, candidates[1].id); - let b2 = small - .rejected - .iter() - .find(|r| r.id == candidates[0].id) - .unwrap(); - assert_eq!(b2.cause, RejectionCause::OverClusterCapacity); - assert_eq!( - b2.reason, - format!("disk {raw} bytes is over the cluster's {} bytes", raw - 1) - ); - } - - /// State materialized on disk is counted as disk, not memory: the KLL - /// state kept on disk at ingestion time (its window being built and the - /// completed one) or for one evaluation at query time adds its bytes to - /// disk, nothing to average memory, and costs only its disk price. - #[test] - fn state_on_disk_is_counted_as_disk_not_memory() { - use asap_types::physical::Medium; - let settings = unit_costs(|c| c.cost_per_disk_byte_second = 1e-9); - let work_only = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let state = |dag: &PhysicalASAPDAG| node_of(dag, is_build); - for (on_disk, in_memory) in [ - (Materialization::MatAtIngestion(Medium::Disk), AT_INGESTION), - (Materialization::MatAtQuery(Medium::Disk), AT_QUERY), - ] { - let (disk_dag, memory_dag) = (kll_dag(on_disk), kll_dag(in_memory)); - let disk = price(&disk_dag, &recurrences(1), &data(), &settings).unwrap(); - let memory = price(&memory_dag, &recurrences(1), &data(), &settings).unwrap(); - // The bytes kept are the same, on disk instead of in memory. - let bytes = disk.usage.disk_bytes - memory.usage.disk_bytes; - assert!(bytes > 0); - assert!( - memory.per_node[&state(&memory_dag)] - .detail - .contains(&format!(" {bytes} bytes")), - "{in_memory:?}: {bytes}" - ); - assert!( - disk.usage.average_memory_bytes < memory.usage.average_memory_bytes, - "{on_disk:?}" - ); - assert_eq!( - disk.usage.peak_memory_bytes + bytes, - memory.usage.peak_memory_bytes, - "{on_disk:?}" - ); - // On disk, the state costs only its disk price beside the work. - let work = price(&memory_dag, &recurrences(1), &data(), &work_only) - .unwrap() - .per_node[&state(&memory_dag)] - .cost_per_second - .unwrap(); - let on_disk_node = &disk.per_node[&state(&disk_dag)]; - assert!( - close( - on_disk_node.cost_per_second.unwrap(), - work + 1e-9 * bytes as f64 - ), - "{on_disk_node:?}" - ); - } - } - - /// Under every cluster objective, two candidates with the same usage - /// tie, and the earlier is selected; the later is reported as tied. - /// Under the Pareto front neither dominates the other: both are on it. - #[test] - fn cluster_objectives_keep_the_earlier_of_tied_candidates() { - let b2 = tumbling_window_candidates(10_000.0).remove(0); - let twin = PhysicalWorkloadPlan { - id: format!("{}-twin", b2.id), - ..b2.clone() - }; - let candidates = [b2, twin]; - for objective in [ - ClusterObjective::LowestLatency, - ClusterObjective::LeastMemory, - ] { - let settings = under(CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective, - })); - let selection = select_among_plans( - &candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(&settings), - ) - .unwrap(); - assert_eq!(selection.selected, candidates[0].id, "{objective:?}"); - let twin = &selection.rejected[0]; - assert_eq!(twin.id, candidates[1].id); - assert_eq!(twin.cause, RejectionCause::TiedLostOnEnumerationOrder); - assert!( - twin.reason.starts_with(&format!( - "tied with the selected candidate {}", - candidates[0].id - )), - "{objective:?}: {}", - twin.reason - ); - } - } - - /// Every Pass 2 window-composition candidate of a per-series p50 over - /// 2 min and a p99 over 5 min, both every minute, with both builds in - /// 1-min tumbling windows: `shared` picks one chain read by both (the 2-min query - /// merging its newest 2 tumbling windows) or a chain each. Its physical candidate - /// with every chain maintained at ingestion time. - fn nested_windows_at_ingestion(shared: bool, rows_per_s: f64) -> PhysicalWorkloadPlan { - let queries = [ - ("quantile_over_time(0.5, m[2m])", MINUTE), - ("quantile_over_time(0.99, m[5m])", MINUTE), - ]; - shared_windows_at_ingestion(queries, shared, rows_per_s) - } - - /// [`nested_windows_at_ingestion`] for two queries, each with its own - /// repeat interval in ms. - fn shared_windows_at_ingestion( - queries: [(&str, u32); 2], - shared: bool, - rows_per_s: f64, - ) -> PhysicalWorkloadPlan { - use asap_logical_asap_optimization::test_support as pcc; - use asap_types::stages::HowToComputeAggregateMeasure; - let inventory = pcc::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups( - &queries - .iter() - .map(|(query, _)| QueryRoot::Operator(lower_promql(query, p99_target()))) - .collect::>(), - ), - ) - .unwrap(); - let kll: Vec = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - matches!( - c, - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::Kll) - ) - }) - .unwrap() - }) - .collect(); - let roots: Vec<_> = pcc::compose_logical_candidate(&inventory, &kll).unwrap(); - let rule_candidates = - asap_logical_asap_optimization::test_support::tumbling_window_candidates( - &roots, - &queries.map(|(_, interval_ms)| Some(u64::from(interval_ms))), - ); - let roots = - asap_logical_asap_optimization::test_support::window_composition_pass2_candidates( - &roots, - &rule_candidates, - ) - .unwrap() - .into_iter() - .find(|c| { - c.tumbling_window_candidates.len() == 2 && c.shared_chains.is_empty() != shared - }) - .unwrap() - .roots - .unwrap(); - optimize_physical_asap( - "L1", - &roots, - &queries.map(|(_, interval_ms)| QueryWorkloadEntry { - predictability: asap_types::workload::Predictability::Predictable { - known_at: None, - }, - ..entry_with(p99_target(), repeating(interval_ms)) - }), - &arriving(rows_per_s), - false, - ) - .unwrap() - .into_iter() - .find(|c| { - c.dag.nodes.iter().all(|n| match n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - matches!(n.materialization, Materialization::MatAtIngestion(_)) - } - _ => n.materialization == Materialization::NotMaterialized, - }) - }) - .unwrap() - } - - /// One chain of 5 tumbling windows maintained at ingestion time serves a 5-min and - /// a 2-min window: the 2-min query's merge reads its newest 2 tumbling windows per - /// series, the 5-min query's all 5, and the plan costs less than a chain - /// per query, since each arriving row updates one tumbling window instead of two. - #[test] - fn a_merge_of_the_newest_tumbling_windows_is_priced_by_the_tumbling_windows_it_reads() { - let rows_per_s = 10_000.0; - let shared = nested_windows_at_ingestion(true, rows_per_s); - let chains: Vec<_> = shared - .dag - .nodes - .iter() - .filter(|n| is_chain(&n.payload)) - .collect(); - assert_eq!(chains.len(), 1); - let recurrence = [repeating(MINUTE), repeating(MINUTE)]; - let settings = CostInputs::default(); - let cost = price(&shared.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - let states_read = |newest: Option| -> u64 { - let merge = shared - .dag - .nodes - .iter() - .find(|n| { - matches!(&n.payload, Payload::ASAP(ASAPOp::SummaryMerge { newest_tumbling_windows, .. }) - if *newest_tumbling_windows == newest) - }) - .unwrap(); - let detail = &cost.per_node[&merge.id].detail; - detail - .strip_prefix("merge ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|states| states.parse().ok()) - .unwrap_or_else(|| panic!("not a merge: {detail}")) - }; - let (newest_two, all_five) = (states_read(Some(2)), states_read(None)); - assert!(all_five > 0); - assert_eq!(newest_two * 5, all_five * 2); - let separate = nested_windows_at_ingestion(false, rows_per_s); - let separate_cost = - price(&separate.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - assert!( - cost.cost_per_second.unwrap() < separate_cost.cost_per_second.unwrap(), - "{:?} vs {:?}", - cost.cost_per_second, - separate_cost.cost_per_second - ); - } - - /// A 10-min quantile every minute (10 × 1-min tumbling windows) and one every 5 - /// min (2 × 5-min tumbling windows), maintained at ingestion time: one chain of - /// 1-min tumbling windows serves both, the 5-min query merging all 10 tumbling windows once - /// per 5 min, and costs less than a chain per query. - #[test] - fn queries_at_different_intervals_share_one_chain_at_ingestion_time() { - let rows_per_s = 10_000.0; - let queries = [ - ("quantile_over_time(0.5, m[10m])", MINUTE), - ("quantile_over_time(0.99, m[10m])", 5 * MINUTE), - ]; - let shared = shared_windows_at_ingestion(queries, true, rows_per_s); - let chains: Vec<_> = shared - .dag - .nodes - .iter() - .filter(|n| is_chain(&n.payload)) - .collect(); - assert_eq!(chains.len(), 1); - assert!(matches!( - chains[0].payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 60_000, - tumbling_window_count: 10, - .. - }) - )); - let recurrence = [repeating(MINUTE), repeating(5 * MINUTE)]; - let settings = CostInputs::default(); - let cost = price(&shared.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - let separate = shared_windows_at_ingestion(queries, false, rows_per_s); - let separate_cost = - price(&separate.dag, &recurrence, &arriving(rows_per_s), &settings).unwrap(); - assert!( - cost.cost_per_second.unwrap() < separate_cost.cost_per_second.unwrap(), - "{:?} vs {:?}", - cost.cost_per_second, - separate_cost.cost_per_second - ); - assert!(cost.usage.average_memory_bytes < separate_cost.usage.average_memory_bytes); - // Each query merges all 10 tumbling windows per series at its own rate: once - // a minute and once per 5 min. - let mut merges: Vec<(u64, String)> = shared - .dag - .nodes - .iter() - .filter(|n| matches!(n.payload, Payload::ASAP(ASAPOp::SummaryMerge { .. }))) - .map(|n| { - let detail = &cost.per_node[&n.id].detail; - let states = detail - .strip_prefix("merge ") - .and_then(|rest| rest.split(' ').next()) - .and_then(|states| states.parse().ok()) - .unwrap_or_else(|| panic!("not a merge: {detail}")); - let rate = detail.split("; x ").nth(1).unwrap_or_default().to_string(); - (states, rate) - }) - .collect(); - merges.sort_by(|a, b| a.1.cmp(&b.1)); - assert_eq!(merges.len(), 2); - assert_eq!(merges[0].0, merges[1].0); - assert!(merges[0].1.starts_with("0.0033/s"), "{merges:?}"); - assert!(merges[1].1.starts_with("0.0167/s"), "{merges:?}"); - } - - /// A 1-hour quantile every minute (60 × 1-min tumbling windows) and every 45 s - /// (80 × 45-s tumbling windows) can share only a chain at their gcd, 240 × 15-s - /// tumbling windows. Maintained at ingestion time it keeps more states than both - /// per-query chains together, so it is priced higher than the - /// per-query chains maintained at ingestion time. - #[test] - fn a_shared_chain_at_a_fine_gcd_is_offered_but_costs_more() { - let rows_per_s = 10_000.0; - let queries = [ - ("quantile_over_time(0.5, m[1h])", MINUTE), - ("quantile_over_time(0.99, m[1h])", 45_000), - ]; - let shared = shared_windows_at_ingestion(queries, true, rows_per_s); - let chain = node_of(&shared.dag, is_chain); - let chain = shared.dag.nodes.iter().find(|n| n.id == chain).unwrap(); - assert!(matches!( - chain.payload, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms: 15_000, - tumbling_window_count: 240, - .. - }) - )); - let separate = shared_windows_at_ingestion(queries, false, rows_per_s); - let recurrence = [repeating(MINUTE), repeating(45_000)]; - let settings = CostInputs::default(); - let price_of = |candidate: &PhysicalWorkloadPlan| { - price( - &candidate.dag, - &recurrence, - &arriving(rows_per_s), - &settings, - ) - .unwrap() - .cost_per_second - .unwrap() - }; - assert!( - price_of(&shared) > price_of(&separate), - "{} vs {}", - price_of(&shared), - price_of(&separate) - ); - } - - /// With more than one ingest worker, each worker beyond the first holds - /// its own partial copy of a maintained state's newest window: B1's - /// peak memory grows by (k − 1) tumbling windows of states, its average memory - /// (and so cost model 1's price) does not. - #[test] - fn partial_copies_of_the_newest_window_add_to_peak_memory() { - let rows_per_s = 1e9; - let b1 = tumbling_window_candidates(rows_per_s).remove(1); - let chain = node_of(&b1.dag, is_chain); - let cost = price( - &b1.dag, - &[repeating(MINUTE)], - &arriving(rows_per_s), - &CostInputs::default(), - ) - .unwrap(); - let detail = &cost.per_node[&chain].detail; - let workers = cost.usage.peak_cpu_cores - 1; - assert!(workers > 1, "{workers}"); - // The chain keeps 6 tumbling windows (5 read and the one being built). - let kept: u64 = detail - .split("; keeps ") - .nth(1) - .and_then(|rest| rest.split(' ').next()) - .and_then(|bytes| bytes.parse().ok()) - .unwrap_or_else(|| panic!("{detail}")); - let copies = workers - 1; - let bytes = u64::from(copies) * (kept / 6); - assert!( - detail.contains(&format!( - "; {copies} partial copies of its newest window, {bytes} bytes" - )), - "{detail}" - ); - // At the peak, not on average: the kept bytes are B1's only - // averaged memory beside negligible query-time holds. - assert!(cost.usage.peak_memory_bytes >= kept + bytes); - assert!(cost.usage.average_memory_bytes < (kept + bytes) as f64); - } - - /// The KLL build of a `quantile_over_time` candidate, with its sketch - /// parameters, and the candidate. - fn kll_build() -> (PhysicalWorkloadPlan, PhysicalASAPNodeId, SketchParams) { - let (candidates, _) = per_series_kll(AccuracyScope::PerValue); - let candidate = candidates.into_iter().next().unwrap(); - let (build, sketch) = candidate - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some((n.id, kind.params().clone())), - _ => None, - }) - .unwrap(); - (candidate, build, sketch) - } - - /// An entry pricing `operator` at both phases on `sketch`. - fn sketch_entry( - operator: OperatorKind, - sketch: SketchParams, - ns_per_unit: f64, - ) -> OperatorCost { - OperatorCost { - operator, - phase: None, - sketch: Some(sketch), - exact: None, - measure: None, - ns_per_unit, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - } - } - - /// An entry pricing `operator` at `phase` with no family. - fn plain_entry( - operator: OperatorKind, - phase: Option, - ns_per_unit: f64, - ) -> OperatorCost { - OperatorCost { - operator, - phase, - sketch: None, - exact: None, - measure: None, - ns_per_unit, - ns_per_invocation: 0.0, - memory_bytes_per_state: None, - } - } - - /// Settings pricing from `operators`. - fn measured(operators: Vec) -> CostInputs { - CostInputs { - measured_operation_costs: MeasuredOperationCosts { - operators, - ..MeasuredOperationCosts::default() - }, - ..CostInputs::default() - } - } - - /// A measured update time for the candidate's KLL prices that build - /// from it (rows × measured ns, counted in operations of - /// `ms_per_cpu_op`) and reports `Measured`; every other node keeps its - /// formulas, reports `Analytical`, and has its CPU scaled by the - /// calibration factor, measured ÷ formula time of the build: 250 ns - /// against the formula's one 1-ns operation per row. - #[test] - fn a_measured_entry_prices_its_node_and_calibrates_the_others() { - let (candidate, build, sketch) = kll_build(); - // Scanning is free here, so each node's time is its CPU, which the - // factor scales (it leaves scanned bytes unscaled). - let cpu_time_only = |mut settings: CostInputs| { - settings.resource_speed.ms_per_scanned_byte = 0.0; - settings - }; - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &cpu_time_only(CostInputs::default()), - ) - .unwrap(); - let settings = cpu_time_only(measured(vec![sketch_entry( - OperatorKind::SummaryAgg, - sketch.clone(), - 250.0, - )])); - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - for (id, node) in &after.per_node { - let was = before.per_node[id].cpu_core_seconds_per_second; - if *id == build { - assert_eq!(node.price_source, NodePriceSource::Measured); - assert!(node.detail.contains("x 250.000 ns"), "{}", node.detail); - } else { - assert_eq!(node.price_source, NodePriceSource::Analytical, "node {id}"); - } - assert!( - close(node.cpu_core_seconds_per_second, 250.0 * was), - "node {id}: {} vs {was}", - node.cpu_core_seconds_per_second - ); - } - assert_eq!(before.source, COST_SOURCE); - assert_eq!(after.source, MIXED_COST_SOURCE); - // The candidate's CPU grows linearly in the measured time. - let cpu_at = |ns: f64| { - let settings = measured(vec![sketch_entry( - OperatorKind::SummaryAgg, - sketch.clone(), - ns, - )]); - price(&candidate.dag, &recurrences(1), &data(), &settings) - .unwrap() - .usage - .cpu_ops_per_second - }; - assert!(close( - (cpu_at(250.0) - cpu_at(100.0)) / (cpu_at(400.0) - cpu_at(100.0)), - 0.5 - )); - } - - /// A KLL with `k` = 469, as Pass 1 sizes it from the data, is priced - /// between the measured `k` = 200 and `k` = 1024, linearly in the log - /// of the state's size; a measured `k` matches exactly whatever failure - /// probability it was sized for, and a `k` beyond the measured ones - /// takes the nearest one's price. - #[test] - fn a_kll_between_two_measured_sizes_is_interpolated() { - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - let table = MeasuredOperationCosts { - operators: vec![ - sketch_entry(OperatorKind::SummaryAgg, kll(200, None), 100.0), - sketch_entry(OperatorKind::SummaryAgg, kll(1024, None), 200.0), - ], - ..MeasuredOperationCosts::default() - }; - let size = |params: &SketchParams| sketch_shape(params, StateInputRows::UNKNOWN).1 as f64; - let at = |k| { - lookup( - &table, - OperatorKind::SummaryAgg, - ExecutionTiming::IngestionTime, - StateFamily::Sketch(&kll(k, Some(0.001))), - size, - ) - .unwrap() - .ns_per_unit - }; - let expected = 100.0 - + 100.0 * (size(&kll(469, None)).ln() - size(&kll(200, None)).ln()) - / (size(&kll(1024, None)).ln() - size(&kll(200, None)).ln()); - assert!(close(at(469), expected), "{} vs {expected}", at(469)); - assert!(at(469) > 100.0 && at(469) < 200.0); - assert_eq!(at(200), 100.0); - assert_eq!(at(1024), 200.0); - assert_eq!(at(4096), 200.0); - assert_eq!(at(50), 100.0); - // Another sketch, operator or family has no entry. - assert!(lookup( - &table, - OperatorKind::SummaryMerge, - ExecutionTiming::QueryTime, - StateFamily::Sketch(&kll(469, None)), - size, - ) - .is_none()); - assert!(lookup( - &table, - OperatorKind::SummaryAgg, - ExecutionTiming::QueryTime, - StateFamily::Sketch(&SketchParams::DDSketch { alpha: 0.01 }), - size, - ) - .is_none()); - } - - /// An entry for the node's own phase wins over one for both phases. - #[test] - fn a_phase_entry_wins_over_one_for_both_phases() { - let table = MeasuredOperationCosts { - operators: vec![ - plain_entry(OperatorKind::Scan, None, 20.0), - plain_entry(OperatorKind::Scan, Some(ExecutionTiming::QueryTime), 150.0), - ], - ..MeasuredOperationCosts::default() - }; - let at = |phase| { - lookup(&table, OperatorKind::Scan, phase, StateFamily::None, |_| { - 1.0 - }) - .unwrap() - .ns_per_unit - }; - assert_eq!(at(ExecutionTiming::QueryTime), 150.0); - assert_eq!(at(ExecutionTiming::IngestionTime), 20.0); - } - - /// The physical candidates of a per-series p99 over 5 min, queried every - /// minute over data arriving at 10,000 rows per second: each Stage 1 - /// choice (the exact rebuild from raw data samples, KLL, DDSketch, ...) - /// with each Stage 2 materialization, ids `P-m`. - fn quantile_candidates() -> Vec { - let root = lower_promql("quantile_over_time(0.99, m[5m])", p99_target()); - let inventory = - asap_logical_asap_optimization::test_support::generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&[ - QueryRoot::Operator(root), - ]), - ) - .unwrap(); - let mut out = Vec::new(); - for choice in 0..inventory.targets[0].choice_count() { - let roots: Vec<_> = - asap_logical_asap_optimization::test_support::compose_logical_candidate( - &inventory, - &[choice], - ) - .unwrap(); - let roots = sized_as_pass1(&roots, &[p99_target()], &arriving(10_000.0)); - let candidates = - optimize_physical_asap("L", &roots, &[every_minute()], &arriving(10_000.0), false) - .unwrap(); - for (k, mut candidate) in candidates.into_iter().enumerate() { - candidate.id = format!("P{}-m{k}", choice + 1); - out.push(candidate); - } - } - out - } - - /// Whether `candidate` builds no summary: it rebuilds the quantile - /// exactly from raw data samples. - fn exact_rebuild(candidate: &PhysicalWorkloadPlan) -> bool { - !candidate.dag.nodes.iter().any(|n| { - matches!( - n.payload, - Payload::ASAP( - ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. } - ) - ) - }) - } - - /// The exact rebuild of [`quantile_candidates`] and the KLL tumbling window - /// candidates ([`tumbling_window_candidates`]: B2 rebuilt at query time, B1 - /// maintained at ingestion time), with the KLL's parameters. - fn exact_and_kll() -> (Vec, SketchParams) { - let mut candidates: Vec<_> = quantile_candidates() - .into_iter() - .filter(exact_rebuild) - .collect(); - assert_eq!(candidates.len(), 1); - candidates[0].id = "EXACT".to_string(); - candidates.extend(tumbling_window_candidates(10_000.0)); - let kll = candidates[2] - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .expect("a KLL tumbling-window chain"); - (candidates, kll) - } - - /// Cost model 1 pricing CPU only (1 per ms of one core): the tests - /// below compare plans by the CPU their tables price. - fn cpu_only(operators: Vec) -> CostInputs { - CostInputs { - cost_model: CostModel::UsageOverTime(ResourceUnitCost { - cost_per_cpu_op: 1e-6, - cost_per_scanned_byte: 1e-7, - cost_per_memory_byte_second: 0.0, - cost_per_provisioned_memory_byte_second: 0.0, - cost_per_provisioned_cpu_core_second: 0.0, - cost_per_disk_byte_second: 0.0, - }), - ..measured(operators) - } - } - - /// The candidate Stage 3 selects among `candidates` for one p99 every - /// minute over 10,000 rows per second. - fn selected(candidates: &[PhysicalWorkloadPlan], settings: &CostInputs) -> WholePlanSelection { - select_among_plans( - candidates, - &[every_minute()], - &arriving(10_000.0), - SelectionModels::builtin().with_cost_inputs(settings), - ) - .unwrap() - } - - /// The candidate cheapest when every node without an entry keeps its - /// formulas' price (no calibration), as Stage 3 priced before. - fn cheapest_uncalibrated(candidates: &[PhysicalWorkloadPlan], settings: &CostInputs) -> String { - let cost = |candidate: &PhysicalWorkloadPlan| { - plan_cost( - &candidate.dag, - &node_map(&candidate.dag), - &kept_tumbling_window_chains(&candidate.dag), - &[repeating(MINUTE)], - &[], - &arriving(10_000.0), - settings, - 1.0, - ) - .unwrap() - .0 - .cost_per_second - .unwrap() - }; - candidates - .iter() - .min_by(|a, b| cost(a).total_cmp(&cost(b))) - .unwrap() - .id - .clone() - } - - /// Calibration keeps the unmeasured nodes on the measured scale. A - /// table measuring the KLL as expensive (150 ns per row) and one exact - /// operator as cheap (the time range, 2 ns per row), but not the exact - /// plan's scan and aggregate, made the exact rebuild from raw data samples - /// look cheapest while the formulas kept its scan and aggregate at - /// their 1-ns scale; with the calibration factor the KLL maintained at - /// ingestion time (B1) is selected. With the full table, the exact - /// rebuild is selected when it really is cheaper (1 ns per scanned - /// sample and aggregated row), and B1 when raw reads and the exact - /// quantile cost 100 and 50 ns. - #[test] - fn calibration_keeps_unmeasured_nodes_on_the_measured_scale() { - let (candidates, kll) = exact_and_kll(); - let quantile = |ns_per_unit| OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, ns_per_unit) - }; - let kll_entries = || { - vec![ - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 150.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 2_000.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 5_000.0), - plain_entry(OperatorKind::TimeRange, None, 2.0), - plain_entry(OperatorKind::Project, None, 1.0), - ] - }; - let partial = cpu_only(kll_entries()); - assert_eq!(cheapest_uncalibrated(&candidates, &partial), "EXACT"); - let selection = selected(&candidates, &partial); - assert_eq!(selection.selected, "P1-m1", "{:?}", selection.rejected); - let full = |scan_ns, aggregate_ns| { - let mut entries = kll_entries(); - entries.push(plain_entry(OperatorKind::Scan, None, scan_ns)); - entries.push(quantile(aggregate_ns)); - cpu_only(entries) - }; - assert_eq!(selected(&candidates, &full(1.0, 1.0)).selected, "EXACT"); - assert_eq!(selected(&candidates, &full(100.0, 50.0)).selected, "P1-m1"); - } - - /// The calibration factor is one heuristic, so a partial table can - /// still mis-rank plans: if the exact plan's scan and quantile really - /// cost 40 and 20 ns, the full table selects the exact rebuild, while - /// the partial table of - /// [`calibration_keeps_unmeasured_nodes_on_the_measured_scale`] prices - /// them at the KLL's ratio and selects B1, which then costs more than - /// the exact rebuild under the full table. - #[test] - fn a_partial_table_can_still_mis_rank_plans() { - let (candidates, kll) = exact_and_kll(); - let partial = vec![ - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 150.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 2_000.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 5_000.0), - plain_entry(OperatorKind::TimeRange, None, 2.0), - plain_entry(OperatorKind::Project, None, 1.0), - ]; - let mut full = partial.clone(); - full.push(plain_entry(OperatorKind::Scan, None, 40.0)); - full.push(OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, 20.0) - }); - assert_eq!(selected(&candidates, &cpu_only(partial)).selected, "P1-m1"); - let full = selected(&candidates, &cpu_only(full)); - assert_eq!(full.selected, "EXACT"); - let cost = |id: &str| full.costs[id].cost_per_second.unwrap(); - assert!(cost("P1-m1") > cost("EXACT")); - } - - /// Reading raw data samples from the database holding them is the - /// deployment's cost. With every operator measured, the exact rebuild - /// from raw data samples at query time is the cheapest plan for a p99 every - /// minute while its raw reads are free (0 ns per sample fetched and - /// decoded at query time); priced at 200 ns per sample, it costs more - /// than the KLL maintained at ingestion time (B1), which is selected. - #[test] - fn external_raw_reads_make_the_query_time_raw_plan_costlier() { - let (candidates, kll) = exact_and_kll(); - let table = |raw_read_ns| { - cpu_only(vec![ - plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::IngestionTime), - 20.0, - ), - plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::QueryTime), - raw_read_ns, - ), - plain_entry(OperatorKind::TimeRange, None, 1.0), - plain_entry(OperatorKind::Project, None, 1.0), - OperatorCost { - measure: Some("quantile".to_string()), - ..plain_entry(OperatorKind::Aggregate, None, 10.0) - }, - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 30.0), - sketch_entry(OperatorKind::SummaryMerge, kll.clone(), 500.0), - sketch_entry(OperatorKind::SummaryEstimate, kll.clone(), 1_000.0), - ]) - }; - let free = selected(&candidates, &table(0.0)); - assert_eq!(free.selected, "EXACT", "{:?}", free.rejected); - let priced = selected(&candidates, &table(200.0)); - assert_eq!(priced.selected, "P1-m1", "{:?}", priced.rejected); - let cost = - |selection: &WholePlanSelection, id: &str| selection.costs[id].cost_per_second.unwrap(); - assert!(cost(&priced, "EXACT") > cost(&priced, "P1-m1")); - assert!(cost(&priced, "EXACT") > cost(&free, "EXACT")); - } - - /// A calibration factor below 1 (here a KLL build measured at 0.01 ns - /// per row against the formula's 1 ns) scales only the CPU of the - /// nodes without an entry, never their scanned bytes, so no node's CPU - /// goes negative: the query-time scan keeps its scanned bytes and - /// takes at least their scan time. - #[test] - fn a_calibration_factor_below_one_never_makes_cpu_negative() { - let (candidate, _, sketch) = kll_build(); - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &CostInputs::default(), - ) - .unwrap(); - let settings = measured(vec![sketch_entry(OperatorKind::SummaryAgg, sketch, 0.01)]); - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - assert!(after.usage.cpu_ops_per_second >= 0.0, "{:?}", after.usage); - for (id, node) in &after.per_node { - assert!( - node.cpu_core_seconds_per_second >= 0.0, - "node {id}: {}", - node.detail - ); - assert!(node.cost_per_second.unwrap() >= 0.0, "node {id}"); - } - assert_eq!( - after.usage.scanned_bytes_per_second, - before.usage.scanned_bytes_per_second - ); - } - - /// Each query request pays the table's `ns_per_evaluation` once: on - /// the node the query reads first, in its CPU and its latency. (Memory - /// is free here: the longer evaluation would hold its memory longer.) - #[test] - fn each_request_pays_the_per_evaluation_cost() { - let (candidate, _, _) = kll_build(); - let free_memory = unit_costs(|c| c.cost_per_memory_byte_second = 0.0); - let before = price(&candidate.dag, &recurrences(1), &data(), &free_memory).unwrap(); - let settings = CostInputs { - measured_operation_costs: MeasuredOperationCosts { - ns_per_evaluation: 1e6, - ..MeasuredOperationCosts::default() - }, - ..free_memory - }; - let after = price(&candidate.dag, &recurrences(1), &data(), &settings).unwrap(); - // 1 ms per request, a request every 10 s. - assert!(close( - after.usage.latency_per_query[0].query_ms - before.usage.latency_per_query[0].query_ms, - 1.0 - )); - let core_seconds = |cost: &PlanCost| { - cost.per_node - .values() - .map(|n| n.cpu_core_seconds_per_second) - .sum::() - }; - assert!(close( - core_seconds(&after) - core_seconds(&before), - 1e-3 / 10.0 - )); - assert!(close( - after.cost_per_second.unwrap() - before.cost_per_second.unwrap(), - 0.1 - )); - } - - // A literal query has no operator to own its fee, but still incurs one - // request's CPU, latency and price; range steps do not multiply that fee. - #[test] - fn literal_requests_pay_overhead_without_operator_nodes() { - use asap_types::workload::{DemandEstimate, Rate}; - let roots = [QueryRoot::Scalar(ScalarExpr::Literal( - ScalarValue::Float64(1.0), - ))]; - let recurrence = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { - expected_rate: Rate(2000.0), - observed_at: None, - valid_for: None, - })); - let entries = [entry_with(AccuracyTarget::exact(), recurrence.clone())]; - let candidate = optimize_physical_asap("literal", &roots, &entries, &data(), false) - .unwrap() - .remove(0); - assert!(candidate.dag.nodes.is_empty()); - let mut settings = CostInputs::default(); - settings.measured_operation_costs.ns_per_evaluation = 2_000_000.0; - for range in [ - None, - Some(RangeEvaluation { - step_ms: 1000, - span_ms: 3000, - }), - ] { - let cost = self_calibrated_cost( - &candidate.dag, - &node_map(&candidate.dag), - &kept_tumbling_window_chains(&candidate.dag), - std::slice::from_ref(&recurrence), - &[range], - &data(), - &settings, - ) - .unwrap(); - let CostModel::UsageOverTime(prices) = &settings.cost_model else { - unreachable!() - }; - assert!(close( - cost.usage.cpu_ops_per_second, - 2.0 / settings.resource_speed.ms_per_cpu_op * 2000.0 - )); - assert!(close( - cost.cost_per_second.unwrap(), - prices.cost_per_cpu_op * cost.usage.cpu_ops_per_second - )); - assert!(cost.cost_per_second.unwrap() > 0.0); - assert_eq!(cost.usage.latency_per_query[0].query_ms, 2.0); - assert_eq!(cost.usage.peak_cpu_cores, 4); - assert!(cost.per_node.is_empty()); - assert_eq!(cost.source, MEASURED_COST_SOURCE); - } - // Cluster capacity must enforce overhead's throughput requirement too. - settings.cost_model = CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: Some(3), - objective: ClusterObjective::LowestLatency, - }); - let facts = data(); - let models = SelectionModels { - cost_inputs: &settings, - ..SelectionModels::builtin() - }; - let mut selector = PlanSelector::new(&entries, &facts, models, false).unwrap(); - selector.set_calibration(CalibrationFit::default()); - selector.offer(&candidate).unwrap(); - assert!(matches!( - selector.finish(), - Err(ErrorSelectingPlan::NoValidCandidate(_)) - )); - // Finite input coefficients can still overflow when multiplied by demand. - settings.measured_operation_costs.ns_per_evaluation = f64::MAX; - assert!(matches!( - price(&candidate.dag, &[recurrence], &facts, &settings), - Err((_, AnalyticalCostError::Overflow)) - )); - } - - /// A measured `memory_bytes_per_state` sizes the state a build keeps, - /// in place of the bytes the formulas derive from its parameters. - #[test] - fn measured_state_bytes_size_the_kept_state() { - let candidates = tumbling_window_candidates(10_000.0); - let maintained = &candidates[1]; - let kll = maintained - .dag - .nodes - .iter() - .find_map(|n| match &n.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let memory = |memory_bytes_per_state| { - let settings = measured(vec![OperatorCost { - memory_bytes_per_state, - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 1.0) - }]); - price( - &maintained.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings, - ) - .unwrap() - .usage - .average_memory_bytes - }; - let formula = memory(None); - // The formula's state: each tumbling window sees its series' 60 samples (one - // minute of 10 000 rows per second over 10 000 series). - let samples = StateInputRows { - rows_per_state: Some(60), - non_negative: false, - }; - let half = memory(Some(sketch_shape(&kll, samples).1 as f64 / 2.0)); - assert!(half < formula && half > 0.0, "{half} vs {formula}"); - } - - // Oversized state configurations must be rejected as unrepresentable, - // never wrap to a cheap candidate or panic while pricing tumbling window choices. - #[test] - fn tumbling_window_state_byte_overflow_returns_a_cost_error() { - for candidate in tumbling_window_candidates(10_000.0) { - let kll = candidate - .dag - .nodes - .iter() - .find_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let settings = measured(vec![OperatorCost { - memory_bytes_per_state: Some(u64::MAX as f64), - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll, 1.0) - }]); - assert!(matches!( - price( - &candidate.dag, - &[repeating(MINUTE)], - &arriving(10_000.0), - &settings - ), - Err((_, AnalyticalCostError::Overflow)) - )); - } - } - - /// The reference pricer rejects a candidate it cannot price, as the - /// search does, rather than failing the whole selection. - #[test] - fn the_reference_pricer_rejects_an_unpriceable_candidate() { - let candidates = tumbling_window_candidates(10_000.0); - let kll = candidates[0] - .dag - .nodes - .iter() - .find_map(|node| match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - }) - .unwrap(); - let settings = measured(vec![OperatorCost { - memory_bytes_per_state: Some(u64::MAX as f64), - ..sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll, 1.0) - }]); - let entries = [entry_with(p99_target(), repeating(MINUTE))]; - let models = SelectionModels { - cost_inputs: &settings, - ..SelectionModels::builtin() - }; - let Err(ErrorSelectingPlan::NoValidCandidate(rejected)) = - select_among_plans(&candidates, &entries, &arriving(10_000.0), models) - else { - panic!("every candidate overflows, so none is valid"); - }; - assert_eq!(rejected.len(), candidates.len()); - assert!(rejected - .iter() - .all(|r| r.cause == RejectionCause::CostNotRepresentable)); - } - - /// The table of earlier versions, a JSON array of sketch operations, - /// still loads: each entry becomes its operator's entry at both phases, - /// an update both a `SummaryAgg` and a `SummaryTumblingWindowChain` entry. - #[test] - fn the_earlier_table_format_still_loads() { - let settings: CostInputs = serde_json::from_str( - r#"{"measured_operation_costs": [ - {"sketch": {"Kll": {"k": 200}}, "operation": "update", "cpu_ns_per_operation": 120.0}, - {"sketch": {"Kll": {"k": 200}}, "operation": "merge", "cpu_ns_per_operation": 900.0}]}"#, - ) - .unwrap(); - let kll = SketchParams::Kll { - k: 200, - failure_probability: None, - }; - assert_eq!( - settings.measured_operation_costs.operators, - vec![ - sketch_entry(OperatorKind::SummaryAgg, kll.clone(), 120.0), - sketch_entry(OperatorKind::SummaryTumblingWindowChain, kll.clone(), 120.0), - sketch_entry(OperatorKind::SummaryMerge, kll, 900.0), - ] - ); - let table: MeasuredOperationCosts = serde_json::from_str( - r#"{"operators": [{"operator": "Scan", "phase": "query_time", "ns_per_unit": 150.0}], - "ns_per_evaluation": 20000.0, - "provenance": {"executor_version": "0.1.0", "machine": "x86_64, 16 cores"}}"#, - ) - .unwrap(); - assert_eq!( - table.operators, - [plain_entry( - OperatorKind::Scan, - Some(ExecutionTiming::QueryTime), - 150.0 - )] - ); - let round_trip: MeasuredOperationCosts = - serde_json::from_str(&serde_json::to_string(&table).unwrap()).unwrap(); - assert_eq!(round_trip, table); - assert!(serde_json::from_str::( - r#"{"operators": [{"operator": "Scan", "ns": 1.0}]}"# - ) - .is_err()); - } - - /// With no entry for any node's operator and family (a CMS that no - /// node builds, a KLL merge in a plan without merges), every node is - /// priced from the formulas, as without a table. - #[test] - fn entries_for_other_operators_or_families_change_nothing() { - let (candidate, _, sketch) = kll_build(); - let before = price( - &candidate.dag, - &recurrences(1), - &data(), - &CostInputs::default(), - ) - .unwrap(); - let other = measured(vec![ - sketch_entry( - OperatorKind::SummaryAgg, - SketchParams::Cms { - width: 1024, - depth: 3, - }, - 50.0, - ), - sketch_entry(OperatorKind::SummaryMerge, sketch, 50.0), - ]); - let after = price(&candidate.dag, &recurrences(1), &data(), &other).unwrap(); - assert_eq!(after, before); - } - - /// Bad input: a negative or non-finite time, an entry naming two - /// families, two entries for one operator, phase and family, or a - /// table with `ms_per_cpu_op` 0. - #[test] - fn invalid_measured_tables_are_rejected() { - let cms = SketchParams::Cms { - width: 1024, - depth: 3, - }; - let entry = |ns| sketch_entry(OperatorKind::SummaryAgg, cms.clone(), ns); - let validate = |operators, ms_per_cpu_op| { - let mut settings = measured(operators); - settings.resource_speed.ms_per_cpu_op = ms_per_cpu_op; - settings.validate() - }; - for ns in [-1.0, f64::NAN] { - assert!(matches!( - validate(vec![entry(ns)], 1e-6), - Err(AnalyticalCostError::InvalidCalibration(..)) - )); - } - let duplicate = validate(vec![entry(1.0), entry(2.0)], 1e-6).unwrap_err(); - assert!( - matches!(duplicate, AnalyticalCostError::DuplicateOperatorCost(_)), - "{duplicate}" - ); - assert!( - duplicate - .to_string() - .contains("SummaryAgg at both phases on Cms"), - "{duplicate}" - ); - let two_families = OperatorCost { - exact: Some(asap_types::ir::schema::ExactKind::Sum), - ..entry(1.0) - }; - assert!(matches!( - validate(vec![two_families], 1e-6), - Err(AnalyticalCostError::AmbiguousOperatorCost(_)) - )); - assert_eq!( - validate(vec![entry(1.0)], 0.0), - Err(AnalyticalCostError::MeasuredCostsWithoutCpuSpeed(0.0)) - ); - let mut empty = CostInputs::default(); - empty.resource_speed.ms_per_cpu_op = 0.0; - assert!( - empty.validate().is_ok(), - "no table, no positive speed needed" - ); - } -} diff --git a/crates/plan-selection/src/milp/completion.rs b/crates/plan-selection/src/milp/completion.rs deleted file mode 100644 index 86bcc1f44..000000000 --- a/crates/plan-selection/src/milp/completion.rs +++ /dev/null @@ -1,306 +0,0 @@ -//! The cost of a plan given as values of the MILP's binaries: the cheapest -//! values of the continuous variables, found without a solver. -//! -//! Every continuous variable of the model is one of: -//! - a lower bound that minimization makes tight: `v ≥ f(others)` over one or -//! more rows (reach `a`, latency `L`, held time `G`/`H`/`W`, AND variables, -//! rate components, raw-retention levels, …). Its cheapest value is the -//! largest right-hand side, or 0; -//! - an equality `N = Σ a` (approximate values read); -//! - free below its upper rows only (`t`, reach through): its value is the -//! smallest upper bound, which leaves the most room to the rows it -//! relaxes. -//! -//! A row defines the continuous variable it bounds from below: the one with a -//! negative coefficient in `expression ≤ 0` (or any nonzero one in an -//! equality) that was created last, since the model creates each variable -//! after the variables its defining rows read. Rows of the families in -//! [`CHECK_ONLY_FAMILIES`] (upper bounds of reach, reader constraints, and -//! valid inequalities a plan's values meet anyway) define nothing. The -//! values are computed in creation order, repeated until no value changes, -//! then every row and bound is checked. A plan that fails a check is -//! infeasible as far as this completion goes (HiGHS's LP completion could -//! find another one; the warm start then just skips that plan). -//! -//! Each value is the smallest its defining rows allow, so at a plan's -//! binaries the values are the plan's own reach, latencies, held times and -//! rates, and the objective is what the model charges the plan: with the -//! exact cost terms of [`super::cost_terms`], what `asap_cost::plan_cost` -//! says (the warm start's test checks it). -use std::collections::HashMap; - -use good_lp::{IntoAffineExpression, Variable}; - -use super::model::SelectionMilp; - -/// Families whose rows only check values (upper bounds of reach, reader -/// constraints, rows tests add) and define no variable. -const CHECK_ONLY_FAMILIES: [&str; 4] = [ - "6 reach from above", - "9 reader constraints", - "V (slowest rate)", - "test", -]; - -/// The most passes over the variables before giving up on a fixpoint. -const MOST_PASSES: usize = 64; - -/// One row in `Σ coefficient · column + constant ≤ 0` (or `= 0`). -struct Row { - /// Range into `columns` and `coefficients`. - start: usize, - end: usize, - constant: f64, - equality: bool, -} - -/// The model's rows, compiled for repeated completions. -pub(crate) struct Completion { - column_of: HashMap, - /// The variables, by column. - variables: Vec, - binary: Vec, - max: Vec, - rows: Vec, - columns: Vec, - coefficients: Vec, - /// Per column, the rows defining it. - defining_rows: Vec>, - /// Per column with no defining row, the rows bounding it from above. - upper_rows: Vec>, - /// The columns with defining rows, in creation order. - defined: Vec, - /// The columns with neither (free continuous variables), in creation - /// order. - free: Vec, - objective: Vec<(usize, f64)>, - /// The family of each row. - families: Vec<&'static str>, -} - -/// A completed plan. -pub(crate) struct Completed { - /// Every variable's value, in creation order. - pub(crate) columns: Vec, - /// The objective (cost per second without the request fees). - pub(crate) cost_per_second: f64, -} - -impl Completion { - pub(crate) fn new(milp: &SelectionMilp<'_>) -> Self { - let builder = &milp.builder; - let mut column_of = HashMap::new(); - let mut variables = Vec::new(); - let mut binary = Vec::new(); - let mut max = Vec::new(); - for (column, (variable, definition)) in - builder.variables.iter_variables_with_def().enumerate() - { - column_of.insert(variable, column); - variables.push(variable); - binary.push(definition.is_integer()); - max.push(definition.get_max()); - } - let count = binary.len(); - let mut rows = Vec::with_capacity(builder.rows.len()); - let mut columns = Vec::new(); - let mut coefficients = Vec::new(); - let mut defining_rows: Vec> = vec![Vec::new(); count]; - let mut positive_rows: Vec> = vec![Vec::new(); count]; - for (index, (row, family)) in builder.rows.iter().zip(&builder.row_families).enumerate() { - let expression = row.expression(); - let start = columns.len(); - let mut defines: Option = None; - for (variable, coefficient) in expression.linear_coefficients() { - if coefficient == 0.0 { - continue; - } - let column = column_of[&variable]; - columns.push(column as u32); - coefficients.push(coefficient); - if binary[column] { - continue; - } - if coefficient > 0.0 { - positive_rows[column].push(index as u32); - } - if (coefficient < 0.0 || row.is_equality()) - && defines.is_none_or(|known| column > known) - { - defines = Some(column); - } - } - if !CHECK_ONLY_FAMILIES.contains(family) { - if let Some(column) = defines { - defining_rows[column].push(index as u32); - } - } - rows.push(Row { - start, - end: columns.len(), - constant: expression.constant(), - equality: row.is_equality(), - }); - } - let mut defined = Vec::new(); - let mut free = Vec::new(); - let mut upper_rows: Vec> = vec![Vec::new(); count]; - for column in 0..count { - if binary[column] { - continue; - } - if defining_rows[column].is_empty() { - upper_rows[column] = std::mem::take(&mut positive_rows[column]); - free.push(column); - } else { - defined.push(column); - } - } - let objective = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| (column_of[&variable], coefficient)) - .collect(); - Self { - column_of, - variables, - binary, - max, - rows, - columns, - coefficients, - defining_rows, - upper_rows, - defined, - free, - objective, - families: builder.row_families.clone(), - } - } - - /// The completion of a solver's columns (in creation order): its - /// binaries rounded, every continuous variable recomputed. - pub(crate) fn complete_columns(&self, columns: &[f64]) -> Result { - let binaries: Vec<(Variable, f64)> = self - .variables - .iter() - .zip(columns) - .zip(&self.binary) - .filter(|(_, &binary)| binary) - .map(|((&variable, &value), _)| (variable, if value > 0.5 { 1.0 } else { 0.0 })) - .collect(); - self.complete(&binaries) - } - - /// Every variable's value in `completed`. - pub(crate) fn values(&self, completed: &Completed) -> Vec<(Variable, f64)> { - self.variables - .iter() - .copied() - .zip(completed.columns.iter().copied()) - .collect() - } - - /// `Σ coefficient · value + constant` of row `row`, without column - /// `skip`, and the coefficient of `skip`. - fn row_value(&self, row: &Row, values: &[f64], skip: usize) -> (f64, f64) { - let mut sum = row.constant; - let mut own = 0.0; - for at in row.start..row.end { - let column = self.columns[at] as usize; - if column == skip { - own += self.coefficients[at]; - } else { - sum += self.coefficients[at] * values[column]; - } - } - (sum, own) - } - - /// The cheapest completion of `binaries` (every binary not listed is 0), - /// or why it has none here: the row or bound it breaks. - pub(crate) fn complete(&self, binaries: &[(Variable, f64)]) -> Result { - let mut values = vec![0.0; self.binary.len()]; - for &(variable, value) in binaries { - let column = self.column_of[&variable]; - if self.binary[column] { - values[column] = value; - } - } - // Defined columns: the largest lower bound, to a fixpoint. - let mut converged = false; - for _ in 0..MOST_PASSES { - let mut changed = false; - for &column in &self.defined { - let mut value: f64 = 0.0; - for &row in &self.defining_rows[column] { - let row = &self.rows[row as usize]; - let (rest, own) = self.row_value(row, &values, column); - // own · v + rest ≤ 0 (or = 0) with own < 0 for an inequality. - let bound = -rest / own; - if row.equality { - value = bound; - break; - } - value = value.max(bound); - } - if (value - values[column]).abs() > 1e-12 * value.abs().max(1.0) { - changed = true; - } - values[column] = value; - } - if !changed { - converged = true; - break; - } - } - if !converged { - return Err("no fixpoint of the lower bounds".into()); - } - // Free columns: the smallest upper bound. - for &column in &self.free { - let mut value = self.max[column]; - for &row in &self.upper_rows[column] { - let row = &self.rows[row as usize]; - let (rest, own) = self.row_value(row, &values, column); - value = value.min(-rest / own); - } - values[column] = if value.is_finite() { - value.max(0.0) - } else { - 0.0 - }; - } - // Every row and bound. - for (index, row) in self.rows.iter().enumerate() { - let mut sum = row.constant; - let mut magnitude = row.constant.abs(); - for at in row.start..row.end { - let term = self.coefficients[at] * values[self.columns[at] as usize]; - sum += term; - magnitude += term.abs(); - } - let tolerance = 1e-7 * magnitude.max(1.0); - if sum > tolerance || (row.equality && sum < -tolerance) { - return Err(format!( - "row {index} ({}) off by {sum:e}", - self.families[index] - )); - } - } - for (column, &value) in values.iter().enumerate() { - if value > self.max[column] + 1e-7 * self.max[column].abs().max(1.0) { - return Err(format!("column {column} at {value:e} above its bound")); - } - } - let cost_per_second = self - .objective - .iter() - .map(|&(column, coefficient)| coefficient * values[column]) - .sum(); - Ok(Completed { - columns: values, - cost_per_second, - }) - } -} diff --git a/crates/plan-selection/src/milp/cost_terms.rs b/crates/plan-selection/src/milp/cost_terms.rs deleted file mode 100644 index 7814f7b3c..000000000 --- a/crates/plan-selection/src/milp/cost_terms.rs +++ /dev/null @@ -1,1204 +0,0 @@ -//! #1095 §3: the cost model 1 objective of the group selection MILP. Every -//! number is `asap-cost`'s: a candidate's usage per execution and standing -//! cost ([`CandidateUsage`], from [`asap_cost::node_usage`] in -//! [`crate::groups`]), the schedule groups -//! ([`asap_cost::execution_demand::schedule_groups`]) and rates -//! ([`asap_cost::execution_demand::evaluation_rate`]) and the unit costs of -//! [`asap_cost::ResourceUnitCost`]. This module only turns them into -//! coefficients and the auxiliary rows that linearize them. -//! -//! A **run** is a candidate computed in an execution context (`p[ctx, c]`), -//! or a shareable candidate's own run (`x[m]`, keyed by its own context). -//! A run's usage is one or more [`RunTerm`]s, each an indicator and the -//! numbers that count when it is 1: -//! - one term on the run's own indicator when its usage is plan -//! independent; -//! - per producer choice, when its usage depends on which candidate of an -//! input's slot the context picks (a summary state's configuration, a -//! scan's window): `AND(run, p[ctx, c'], producer gives that output)`; -//! - for a query-time scan read at several windows in one context: one -//! term per set `S` of window kinds read, `δ_S`, the exact indicator that -//! exactly the kinds in `S` are read (from `σ_W`, the exact OR of the -//! reads at kind `W`), costed at the union of `S`'s windows -//! (`PhysicalASAPDAG::scan_window`). This needs no monotonicity: every -//! union is costed by `asap-cost` itself. -//! -//! **Executions per second** of a run: `rate(q)` in query context `q`; 1 in -//! each `I_r` and for an ingestion-time shareable candidate; for a query-time -//! shareable candidate `m` and the runs in its context, `Σ rate · z[m, G, -//! ℓ]` over its schedule groups and their levels, `z ≥ a[q, m]` for each -//! reader at that level or faster (a group's rate is its fastest reader's). -//! A context's runs other than `m`'s plain own term are summed once, `E[m] -//! ≥ Σ cost · indicator`, and charged per component, `V ≥ E[m] − Ē[m] (x[m] -//! − z)`, with `Σ rate · V ≥ (slowest rate) · E[m]`. -//! -//! **Held memory**: working memory of a run in query context `q` for `q`'s -//! latency at `q`'s rate (`H_query ≥ L[q] − L̄[q] (1 − indicator)`); `m`'s -//! `MatAtQuery(Memory)` result and the working memory of the runs in its -//! context until `m`'s slowest reader finishes (`H[m] ≥ L[q] − L̄[q] (1 − -//! a[q, m])`, then `W ≥ H[m] − H̄[m] (slack − Σ indicators)` per rate -//! component). -//! -//! **Raw retention**: per source, levels at each raw-data size a query-time -//! scan run may keep, `v[src, ℓ] ≥` each scan term keeping at least that -//! much; charged `Δbytes · unit disk cost`. -//! -//! **Pane retention**: a kept pane chain's usage at the fewest panes any -//! reader needs on `x[m]`, and each larger retention's increase on `π[m, -//! ℓ] ≥` the reach (as `plan_cost` counts it: through query-time runs) of -//! each query needing at least that many panes. -//! -//! Terms `plan_cost` defines differently from the above are counted in -//! [`SelectionMilp::inexact`] by reason. -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use asap_cost::execution_demand::{evaluation_rate, schedule_groups}; -use asap_cost::CostModel; -use asap_types::ir::physical_export::ScanWindow; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{NonASAPOp, Operator}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::PhysicalCandidateId; -use asap_types::workload::bounds::source_name; -use asap_types::workload::{QueryRecurrence, RepeatedDemand}; -use good_lp::{constraint, Expression, IntoAffineExpression, Variable}; - -use crate::groups::{ - CandidateUsage, ContextKind, ResultSlot, RunKind, UsageAndOutput, UsageContext, -}; - -use super::model::{Reader, Scope, SelectionMilp, SelectionMilpError}; - -/// The numbers of one run `asap-cost` computed, as the objective and the -/// latency rows read them. -#[derive(Debug, Clone, Copy, Default, PartialEq)] -pub(crate) struct NodeNumbers { - /// Cost of one execution. - pub(crate) execution_cost: f64, - /// Cost per second of what it keeps whatever runs it. - pub(crate) standing_cost: f64, - /// What it adds to a reaching query's latency, ms. - pub(crate) latency_ms: f64, - /// Query-time memory held while an evaluation reading it runs. - pub(crate) held_bytes: f64, - /// Raw data a query-time scan keeps on disk. - pub(crate) raw_kept_bytes: f64, -} - -impl NodeNumbers { - pub(crate) fn of(usage: &CandidateUsage) -> Self { - Self { - execution_cost: usage.cost_per_execution.unwrap_or(0.0), - standing_cost: usage.standing_cost_per_second.unwrap_or(0.0), - latency_ms: usage.own_latency_ms, - held_bytes: usage.held_bytes as f64, - raw_kept_bytes: usage.raw_data_kept_bytes as f64, - } - } - - /// `self − other`, each number at least 0, and whether one was below. - fn minus(self, other: Self) -> (Self, bool) { - let fields = [ - self.execution_cost - other.execution_cost, - self.standing_cost - other.standing_cost, - self.latency_ms - other.latency_ms, - self.held_bytes - other.held_bytes, - self.raw_kept_bytes - other.raw_kept_bytes, - ]; - let negative = fields - .iter() - .any(|&field| field < -1e-12 * field.abs().max(1.0)); - let [execution_cost, standing_cost, latency_ms, held_bytes, raw_kept_bytes] = - fields.map(|field| field.max(0.0)); - ( - Self { - execution_cost, - standing_cost, - latency_ms, - held_bytes, - raw_kept_bytes, - }, - negative, - ) - } -} - -/// One term of a run: its numbers count, and it gives its readers output -/// `output`, when `indicator` is 1. At integral binaries exactly one term -/// of a used run is 1, except a kept pane chain's retention increases, -/// which add to its base term. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct RunTerm { - pub(crate) indicator: Expression, - pub(crate) numbers: NodeNumbers, - /// Whether `indicator` is the run's own variable (`p[ctx, c]` or - /// `x[m]`). - pub(crate) plain: bool, - /// Its output (an index below `CandidateData::output_count`). - pub(crate) output: usize, -} - -/// Run `(context, candidate)`'s terms (module documentation). Empty when -/// the run does not exist. -pub(crate) fn run_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - candidate: PhysicalCandidateId, -) -> Vec { - let Some(run) = milp.run_indicator(context, candidate) else { - return Vec::new(); - }; - let data = &milp.inputs.candidate_data[candidate.0]; - let Ok(base) = &data.usage else { - return Vec::new(); - }; - let base = *base; - for usage_context in data.usage_depends_on.clone() { - match usage_context { - UsageContext::ProducerOnACycle => milp.note_inexact("usage: a producer on a cycle"), - UsageContext::ProducerOutput - | UsageContext::ScanWindowOfReader - | UsageContext::PaneRetentionOfReachingQueries - | UsageContext::UncostableContext => {} - } - } - if milp.inputs.candidate_data[candidate.0].producer_usage_ambiguous { - milp.note_inexact("usage: a producer's output depends on its own producers"); - } - if !milp.inputs.candidate_data[candidate.0] - .usage_per_window - .is_empty() - { - if let Some(terms) = window_terms(milp, context, candidate, run) { - return terms; - } - } - if !milp.inputs.candidate_data[candidate.0] - .inputs_changing_usage - .is_empty() - { - return producer_terms(milp, context, candidate, run); - } - let mut terms = vec![RunTerm { - indicator: run.into(), - numbers: NodeNumbers::of(&base), - plain: true, - output: 0, - }]; - if !milp.inputs.candidate_data[candidate.0] - .usage_per_retained_panes - .is_empty() - { - pane_retention_terms(milp, context, candidate, &mut terms); - } - terms -} - -/// The indicator that, in `context`, the producer `producer` of a reader is -/// read and gives one of `outputs` (`None`: any output): `p[ctx, c']`, or -/// `AND(p[ctx, c'], t)` with `t ≥` the producer run's terms giving them. -/// `None` when the producer never gives them. -fn producer_gives( - milp: &mut SelectionMilp<'_>, - context: usize, - producer: PhysicalCandidateId, - outputs: Option<&BTreeSet>, -) -> Option { - let read = *milp.choice[context].get(&producer)?; - let Some(outputs) = outputs else { - return Some(read.into()); - }; - // The producer's run: computed here, its own run, or its run in an - // `I_r`. - let producer_run = match milp.kind(producer) { - _ if milp.computes(context, producer) => context, - RunKind::Shareable => milp.context_of(producer)?, - RunKind::Ingestion => milp.ingestion_context_read_by(context)?, - RunKind::PerContext => return None, - }; - if milp.terms_in_progress.contains(&(producer_run, producer)) { - milp.note_inexact("usage: a producer's output on a cycle of links (every output kept)"); - return Some(read.into()); - } - let producer_terms = milp.run_terms(producer_run, producer); - if producer_terms - .iter() - .all(|term| outputs.contains(&term.output)) - { - return Some(read.into()); - } - let gives: Vec<&RunTerm> = producer_terms - .iter() - .filter(|term| outputs.contains(&term.output)) - .collect(); - if gives.is_empty() { - return None; - } - let key = ( - producer_run, - producer, - outputs.iter().copied().collect::>(), - ); - let gives = match milp.producer_gives.get(&key) { - Some(&gives) => gives, - None => { - let sum: Expression = gives.iter().map(|term| term.indicator.clone()).sum(); - let variable = milp - .builder - .continuous("t_outputs (a producer gives outputs)", Some(1.0)); - milp.builder.row( - "t_outputs (a producer gives outputs)", - constraint!(variable >= sum), - ); - milp.producer_gives.insert(key, variable); - variable - } - }; - if milp.computes(context, producer) { - return Some(gives.into()); - } - Some( - milp.builder - .and_at_least("t_producers (AND)", &[read.into(), gives.into()]) - .into(), - ) -} - -/// A producer choice with the outputs it stands for (`None`: any output). -type OutputsOfProducer = (PhysicalCandidateId, Option>); - -/// Terms of a run whose usage or output depends on its producers (and -/// their outputs): one per choice, on `AND(run, producer gives)` over the -/// changing inputs. -fn producer_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - candidate: PhysicalCandidateId, - run: Variable, -) -> Vec { - milp.terms_in_progress.insert((context, candidate)); - let data = &milp.inputs.candidate_data[candidate.0]; - let changing = data.inputs_changing_usage.clone(); - let base = *data.usage.as_ref().expect("costed"); - let same = |known: &Option, gives: &Option| match (known, gives) - { - (Some(known), Some(gives)) => known == gives, - (None, None) => true, - _ => false, - }; - let mut table: Vec<(Vec, Option)> = Vec::new(); - for (choices, gives) in &data.usage_per_producers { - let choices: Vec = choices - .iter() - .map(|choice| { - ( - choice.class, - choice.output.map(|output| BTreeSet::from([output])), - ) - }) - .collect(); - if changing.len() == 1 { - if let Some((known, _)) = table - .iter_mut() - .find(|(known, known_gives)| known[0].0 == choices[0].0 && same(known_gives, gives)) - { - if let (Some(outputs), Some(more)) = (&mut known[0].1, &choices[0].1) { - outputs.extend(more); - } - continue; - } - } - table.push((choices, *gives)); - } - let producers_here: Vec> = changing - .iter() - .map(|&input| { - milp.inputs.producer_candidates_per_input[candidate.0][input] - .iter() - .copied() - .filter(|producer| milp.choice[context].contains_key(producer)) - .filter(|producer| milp.slot(*producer) != milp.slot(candidate)) - .collect() - }) - .collect(); - let mut covered: Vec> = vec![BTreeSet::new(); changing.len()]; - let mut terms = Vec::new(); - for (choices, gives) in &table { - let mut per_input: Vec = Vec::with_capacity(choices.len()); - for (position, (producer, outputs)) in choices.iter().enumerate() { - if !producers_here[position].contains(producer) { - break; - } - covered[position].insert(*producer); - match producer_gives(milp, context, *producer, outputs.as_ref()) { - Some(indicator) => per_input.push(indicator), - None => break, - } - } - if per_input.len() < choices.len() { - continue; - } - let mut all: Vec = vec![run.into()]; - all.extend(per_input); - // A multi-slot input reads the choice's slot only through its link. - for (position, (producer, _)) in choices.iter().enumerate() { - if let Some(links) = milp - .slot_links - .get(&(context, candidate, changing[position])) - { - let slot = milp.slot(*producer); - if let Some(&(_, link)) = links.iter().find(|(at, _)| *at == slot) { - all.push(link.into()); - } - } - } - let count = all.len() as f64; - match gives { - None => { - let sum: Expression = all.into_iter().sum(); - milp.builder - .row("t_producers (AND)", constraint!(sum <= count - 1.0)); - } - Some((usage, output)) => { - let indicator = milp.builder.and_at_least("t_producers (AND)", &all); - terms.push(RunTerm { - indicator: indicator.into(), - numbers: NodeNumbers::of(usage), - plain: false, - output: *output, - }); - } - } - } - // A producer missing from the table: its reads carry the first costed - // usage. - let mut missing = false; - for (position, producers) in producers_here.iter().enumerate() { - for &producer in producers { - if covered[position].contains(&producer) { - continue; - } - missing = true; - let read = milp.choice[context][&producer]; - let indicator = milp - .builder - .and_at_least("t_producers (AND)", &[run.into(), read.into()]); - terms.push(RunTerm { - indicator: indicator.into(), - numbers: NodeNumbers::of(&base), - plain: false, - output: 0, - }); - } - } - if missing { - milp.note_inexact("usage: a producer missing from its usage table"); - } - if terms.is_empty() { - milp.builder - .row("t_producers (AND)", constraint!(run <= 0.0)); - } - milp.terms_in_progress.remove(&(context, candidate)); - terms -} - -/// The union of `windows`, as [`PhysicalASAPDAG::scan_window`] computes it -/// for one scan read at each: the nearest end and the farthest start. -pub(crate) fn window_union(windows: impl IntoIterator) -> ScanWindow { - let mut near = i64::MAX; - let mut far = i64::MIN; - for window in windows { - match window { - ScanWindow::Unbounded => return ScanWindow::Unbounded, - ScanWindow::Within { - end_offset_ms, - width_ms, - } => { - near = near.min(end_offset_ms); - far = far.max(end_offset_ms.saturating_add(width_ms)); - } - } - } - if near > far { - return ScanWindow::Unbounded; - } - ScanWindow::Within { - end_offset_ms: near, - width_ms: far.saturating_sub(near), - } -} - -/// The window a scan run read at `kinds` scans, as `asap_cost::plan_cost` -/// prices it: their union ([`window_union`]) when every kind is bounded; -/// with an unbounded read (a reader that is not a time range, a pane chain -/// or a time shift, or a read as an output), the trailing window of the -/// widest bounded kind (`plan_cost`'s `widest_window_read_ms`), or -/// `Unbounded` when there is none. -pub(crate) fn scan_window_read_at( - kinds: impl IntoIterator + Clone, -) -> ScanWindow { - let unbounded = kinds - .clone() - .into_iter() - .any(|kind| kind == ScanWindow::Unbounded); - if !unbounded { - return window_union(kinds); - } - let widest = kinds - .into_iter() - .filter_map(|kind| match kind { - ScanWindow::Within { width_ms, .. } => Some(width_ms), - ScanWindow::Unbounded => None, - }) - .max(); - match widest { - Some(width_ms) => ScanWindow::Within { - end_offset_ms: 0, - width_ms, - }, - None => ScanWindow::Unbounded, - } -} - -/// The windows reader `reader` reads scan `scan` at (one per path through -/// time shifts); `Unbounded` when unknown. -fn reader_windows( - milp: &SelectionMilp<'_>, - scan: PhysicalCandidateId, - reader: PhysicalCandidateId, -) -> Vec { - milp.inputs.candidate_data[scan.0] - .windows_per_reader - .iter() - .find(|(known, _)| *known == reader) - .map(|(_, read)| read.clone()) - .unwrap_or_else(|| vec![ScanWindow::Unbounded]) -} - -/// Terms of query-time scan run `(context, scan)` whose usage depends on -/// the window its readers read (module documentation). `None` when its -/// readers read one window kind and it has no other dependence handled -/// here: the caller then uses its other terms. -fn window_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - scan: PhysicalCandidateId, - run: Variable, -) -> Option> { - if let Some(cached) = milp.window_cache.get(&(context, scan)) { - return cached.clone(); - } - // Each read: (reading context, window kind, reader's indicator). - let mut reads: Vec<(usize, ScanWindow, Expression)> = Vec::new(); - let reading_contexts: Vec = if milp.kind(scan) == RunKind::Shareable { - (0..milp.choice.len()) - .filter(|&other| milp.choice[other].contains_key(&scan)) - .collect() - } else { - vec![context] - }; - for &reading in &reading_contexts { - for reader in milp.readers[reading] - .get(&scan) - .cloned() - .unwrap_or_default() - { - let (window, indicator) = match reader { - Reader::Candidate(reader, _) => { - let windows = reader_windows(milp, scan, reader); - if windows.len() > 1 { - milp.note_inexact("a scan read at several windows by one reader (union)"); - } - ( - window_union(windows), - Expression::from(milp.choice[reading][&reader]), - ) - } - Reader::Output(_) => (ScanWindow::Unbounded, Expression::from(1.0)), - Reader::OwnInput(_) => { - let ContextKind::Shared(own) = milp.inputs.contexts[reading].kind else { - continue; - }; - // `m` itself reads the scan: at its own window. - ( - window_union(reader_windows(milp, scan, own)), - Expression::from(milp.selected[&own]), - ) - } - }; - reads.push((reading, window, indicator)); - } - } - let mut kinds: Vec = Vec::new(); - for (_, window, _) in &reads { - if !kinds.contains(window) { - kinds.push(*window); - } - } - let usage_at = |milp: &SelectionMilp<'_>, window: ScanWindow| { - let found = milp.inputs.candidate_data[scan.0] - .usage_per_window - .iter() - .find(|(known, _, _)| *known == window) - .map(|&(_, usage, output)| (NodeNumbers::of(&usage), output)); - if found.is_none() { - super::warm_start::debug(|| { - format!( - "scan {} has no usage at {window:?}; costed at {:?}", - scan.0, - milp.inputs.candidate_data[scan.0] - .usage_per_window - .iter() - .map(|(w, _, _)| *w) - .collect::>() - ) - }); - } - found - }; - let terms = match kinds.len() { - 0 => None, - 1 => match usage_at(milp, scan_window_read_at([kinds[0]])) { - Some((numbers, output)) => Some(vec![RunTerm { - indicator: run.into(), - numbers, - plain: true, - output, - }]), - None => { - // `asap-cost` cannot cost the scan so: no plan reads it so. - milp.builder - .row("δ (scan window sets)", constraint!(run <= 0.0)); - Some(Vec::new()) - } - }, - count => { - // `σ_W`: the exact OR, over the reading contexts, of "the - // context reads the scan and a reader at kind `W` is used" - // (`w`, exact from both sides at integral binaries). - let mut sigma = Vec::with_capacity(count); - for kind in &kinds { - let mut per_context: Vec = Vec::new(); - for &reading in &reading_contexts { - let at_kind: Vec<&Expression> = reads - .iter() - .filter(|(at, window, _)| *at == reading && window == kind) - .map(|(_, _, indicator)| indicator) - .collect(); - if at_kind.is_empty() { - continue; - } - let read_here = milp.choice[reading][&scan]; - let used = milp.builder.continuous("σ (scan window kinds)", Some(1.0)); - let mut upper = Expression::from(0.0); - for indicator in at_kind { - let indicator = indicator.clone(); - upper += indicator.clone(); - milp.builder.row( - "σ (scan window kinds)", - constraint!(used >= indicator + read_here - 1.0), - ); - } - milp.builder - .row("σ (scan window kinds)", constraint!(used <= read_here)); - milp.builder - .row("σ (scan window kinds)", constraint!(used <= upper)); - per_context.push(used); - } - let or = match per_context.as_slice() { - [one] => *one, - many => { - let or = milp.builder.continuous("σ (scan window kinds)", Some(1.0)); - let mut upper = Expression::from(0.0); - for &used in many { - milp.builder - .row("σ (scan window kinds)", constraint!(or >= used)); - upper += used; - } - milp.builder - .row("σ (scan window kinds)", constraint!(or <= upper)); - or - } - }; - sigma.push(or); - } - let mut terms = Vec::new(); - assert!( - count < usize::BITS as usize, - "a scan run with {count} window kinds has too many unions to encode" - ); - for set in 1usize..(1 << count) { - let window = scan_window_read_at( - (0..count) - .filter(|bit| set & (1 << bit) != 0) - .map(|bit| kinds[bit]), - ); - let members = set.count_ones() as f64; - let mut sum = Expression::from(0.0); - for (bit, &or) in sigma.iter().enumerate() { - if set & (1 << bit) != 0 { - sum += or; - } else { - sum -= or; - } - } - let Some((numbers, output)) = usage_at(milp, window) else { - // `asap-cost` cannot cost the scan at this union: no - // plan reads exactly these kinds. - milp.builder - .row("δ (scan window sets)", constraint!(sum <= members - 1.0)); - continue; - }; - let exactly = milp.builder.continuous("δ (scan window sets)", Some(1.0)); - milp.builder.row( - "δ (scan window sets)", - constraint!(exactly >= sum - (members - 1.0)), - ); - terms.push(RunTerm { - indicator: exactly.into(), - numbers, - plain: false, - output, - }); - } - Some(terms) - } - }; - milp.window_cache.insert((context, scan), terms.clone()); - terms -} - -/// For a shareable kept pane chain with usage depending on the panes it -/// retains: its base term at the fewest panes, plus one term per larger -/// retention, `π[m, ℓ] ≥` the reach of each query needing at least `ℓ`. -fn pane_retention_terms( - milp: &mut SelectionMilp<'_>, - context: usize, - chain: PhysicalCandidateId, - terms: &mut Vec, -) { - let data = &milp.inputs.candidate_data[chain.0]; - let mut levels = data.usage_per_retained_panes.clone(); - levels.sort_by_key(|(panes, _)| *panes); - let per_query = data.retained_panes_per_query.clone(); - let Some(&(_, base)) = levels.first() else { - return; - }; - terms[0].numbers = NodeNumbers::of(&base); - let at_query_time = milp.runs_at_query_time(context); - // Per query, the indicator that it reaches the chain as `plan_cost` - // counts reach (through query-time runs). - let mut reached: Vec<(usize, Vec)> = Vec::new(); - for &(query, _) in &per_query { - let mut indicators = Vec::new(); - if let Some(&reach) = milp.reach.get(&(query, chain)) { - if at_query_time - || milp.physical.candidates[chain.0].timing == ExecutionTiming::QueryTime - { - indicators.push(Expression::from(reach)); - reached.push((query, indicators)); - continue; - } - } - if let Some(&direct) = milp.choice[query].get(&chain) { - indicators.push(direct.into()); - } - for &reader in milp - .shared_readers - .get(&chain) - .cloned() - .unwrap_or_default() - .iter() - { - let reader_context = milp.inputs.context_of_shareable[&reader]; - if !milp.runs_at_query_time(reader_context) { - continue; - } - if let Some(&above) = milp.reach.get(&(query, reader)) { - let read = milp.choice[reader_context][&chain]; - let and = milp - .builder - .and_at_least("π (pane retention levels)", &[above.into(), read.into()]); - indicators.push(and.into()); - } - } - reached.push((query, indicators)); - } - let mut below = NodeNumbers::of(&base); - for &(panes, usage) in &levels[1..] { - let numbers = NodeNumbers::of(&usage); - let (increase, negative) = numbers.minus(below); - below = numbers; - if negative { - milp.note_inexact("a longer pane retention with a smaller number"); - } - if increase.execution_cost > 0.0 && at_query_time { - milp.note_inexact("pane retention changing a query-time execution cost"); - } - if increase.latency_ms > 0.0 || increase.held_bytes > 0.0 { - milp.note_inexact("pane retention changing latency or held memory (not encoded)"); - } - let level = milp - .builder - .continuous("π (pane retention levels)", Some(1.0)); - for (query, indicators) in &reached { - let needs = per_query - .iter() - .any(|&(q, retained)| q == *query && retained >= panes); - if !needs { - continue; - } - for indicator in indicators { - let indicator = indicator.clone(); - milp.builder - .row("π (pane retention levels)", constraint!(level >= indicator)); - } - } - terms.push(RunTerm { - indicator: level.into(), - numbers: NodeNumbers { - latency_ms: 0.0, - held_bytes: 0.0, - ..increase - }, - plain: false, - output: terms[0].output, - }); - } -} - -/// Each ingestion-time shareable candidate's compaction term for its -/// readers' latency, ms, where `plan_cost` charges one: `(k − 1) · states · -/// merge time`, `k` the cores its ingestion work takes. With at most one -/// second of ingestion work per second below it, `k = 1` and the term is 0; -/// a larger subtree is not encoded (counted as inexact). -pub(crate) fn compaction_ms(milp: &mut SelectionMilp<'_>) -> HashMap { - let inputs = milp.inputs; - let mut compaction = HashMap::new(); - for &shareable in inputs.context_of_shareable.keys() { - if milp.physical.candidates[shareable.0].timing != ExecutionTiming::IngestionTime { - continue; - } - let mut seen = BTreeSet::new(); - let mut stack = vec![shareable]; - let mut most_work_ms = 0.0; - while let Some(class) = stack.pop() { - if !seen.insert(class) { - continue; - } - if let Ok(usage) = &inputs.candidate_data[class.0].usage { - most_work_ms += usage.work_ms_per_execution; - } - stack.extend( - inputs.producer_candidates_per_input[class.0] - .iter() - .flatten() - .copied(), - ); - } - if most_work_ms > 1_000.0 { - milp.note_inexact("compaction with more than one ingest worker (not encoded)"); - } - compaction.insert(shareable, 0.0); - } - compaction -} - -/// A schedule group's readers with their own rates, and its rate levels. -type GroupLevels = (Vec<(usize, f64)>, Vec); - -/// How the runs of a context execute per second. -#[derive(Debug, Clone)] -enum ContextRate { - /// A fixed rate (a query's, 1 at ingestion time, or a query-time - /// shareable candidate whose possible readers run it at one rate). - Fixed(f64), - /// `Σ rate · z` over a query-time shareable candidate's schedule-group - /// levels. - Components(Vec<(Variable, f64)>), -} - -/// Each context's rate (module documentation), and the queries' own -/// rates. -fn context_rates( - milp: &mut SelectionMilp<'_>, -) -> Result<(Vec, Vec), SelectionMilpError> { - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let cost = |error: asap_cost::analytical::AnalyticalCostError| { - SelectionMilpError::Cost(error.to_string()) - }; - let query_rate: Vec = milp - .entries - .iter() - .map(|entry| evaluation_rate([(&entry.recurrence, entry.range_evaluation)], horizon)) - .collect::>() - .map_err(cost)?; - let mut rates = Vec::with_capacity(milp.inputs.contexts.len()); - milp.builder.scope = Scope::SharedContexts; - for context in 0..milp.inputs.contexts.len() { - let rate = match milp.inputs.contexts[context].kind { - ContextKind::Query(query) => ContextRate::Fixed(query_rate[query]), - ContextKind::Ingestion(_) => ContextRate::Fixed(1.0), - ContextKind::Shared(m) => { - let per_request = match milp.physical.candidates[m.0].materialization { - Materialization::MatAtIngestion(_) | Materialization::NotMaterialized => { - rates.push(ContextRate::Fixed(1.0)); - continue; - } - Materialization::MatAtQuery(_) => false, - Materialization::MatAtQueryKeptAcrossEvaluations(_) => true, - }; - let readers = milp.queries_reaching(m); - let demand = |query: usize| { - let entry = &milp.entries[query]; - ( - &entry.recurrence, - if per_request { - None - } else { - entry.range_evaluation - }, - ) - }; - if unknown_phase_joins_several( - readers.iter().map(|&query| &milp.entries[query].recurrence), - ) { - milp.note_inexact("schedule: an unknown phase among several known ones"); - } - let groups = schedule_groups(readers.iter().map(|&query| demand(query)), horizon) - .map_err(cost)?; - let mut levels_per_group: Vec = Vec::new(); - for group in groups { - let members: Vec = group - .readers - .iter() - .map(|&position| readers[position]) - .collect(); - let alone: Vec = members - .iter() - .map(|&query| evaluation_rate([demand(query)], horizon)) - .collect::>() - .map_err(cost)?; - let fastest = alone.iter().copied().fold(0.0, f64::max); - let full = group.evaluations_per_second; - let mut levels: Vec = alone.clone(); - if (full - fastest).abs() > 1e-9 * full.abs().max(1e-12) { - milp.note_inexact("schedule: a group's rate is not its fastest reader's"); - levels = vec![full]; - } - levels.sort_by(f64::total_cmp); - levels.dedup(); - levels_per_group.push((members.into_iter().zip(alone).collect(), levels)); - } - if let [(_, levels)] = levels_per_group.as_slice() { - if let [level] = levels.as_slice() { - rates.push(ContextRate::Fixed(*level)); - continue; - } - } - let mut components = Vec::new(); - for (members, levels) in levels_per_group { - let mut below = 0.0; - for &level in &levels { - let z = milp.builder.continuous("z (schedule groups)", Some(1.0)); - for &(query, own) in &members { - if own >= level || levels.len() == 1 { - let reach = milp.reach[&(query, m)]; - milp.builder - .row("z (schedule groups)", constraint!(z >= reach)); - } - } - components.push((z, level - below)); - below = level; - } - } - ContextRate::Components(components) - } - }; - rates.push(rate); - } - milp.builder.scope = Scope::Global; - Ok((rates, query_rate)) -} - -/// Whether some interval has readers of unknown phase and readers of two -/// or more known phases. -fn unknown_phase_joins_several<'r>(recurrences: impl Iterator) -> bool { - let mut phases: BTreeMap)> = BTreeMap::new(); - for recurrence in recurrences { - match recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - phases.entry(interval.0).or_default().0 = true; - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => { - phases - .entry(interval.0) - .or_default() - .1 - .insert(evaluation_phase.0 % u64::from(interval.0.max(1))); - } - _ => {} - } - } - phases - .values() - .any(|(unknown, known)| *unknown && known.len() > 1) -} - -/// The objective (cost model 1, module documentation) and its rows. -pub(crate) fn encode_objective(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { - let CostModel::UsageOverTime(unit_cost) = milp.cost_inputs.cost_model.clone() else { - return Err(SelectionMilpError::UnsupportedCostModel("not cost model 1")); - }; - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let cost = |error: asap_cost::analytical::AnalyticalCostError| { - SelectionMilpError::Cost(error.to_string()) - }; - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - if request_ms > 0.0 { - for entry in milp.entries { - let requests = evaluation_rate([(&entry.recurrence, None)], horizon).map_err(cost)?; - milp.request_cost_per_second += unit_cost.cost_per_cpu_op * request_ms - / milp.cost_inputs.resource_speed.ms_per_cpu_op - * requests; - } - } - let (rates, query_rate) = context_rates(milp)?; - let memory = unit_cost.cost_per_memory_byte_second; - let mut objective: Vec<(Variable, f64)> = Vec::new(); - let mut charge = |expression: &Expression, coefficient: f64| { - if coefficient != 0.0 { - for (variable, factor) in expression.linear_coefficients() { - objective.push((variable, factor * coefficient)); - } - } - }; - // Per query-time shareable context: the execution cost of its runs - // other than `m`'s plain own term, and per slot (`None`: `m`'s own run) - // the most it can be. - let mut context_execution: BTreeMap, f64>)> = - BTreeMap::new(); - // `H[m]` per query-time shareable context. - let mut held_time: HashMap = HashMap::new(); - let runs = milp.runs(); - for (context, candidate) in runs { - let terms = milp.run_terms(context, candidate); - milp.builder.scope = milp.scope_of(context); - let candidate_data = &milp.physical.candidates[candidate.0]; - let (timing, materialization) = (candidate_data.timing, candidate_data.materialization); - let own = milp.inputs.contexts[context].kind == ContextKind::Shared(candidate); - // The most this run adds to its context's execution cost: all its - // terms at 1. - let mut run_execution = 0.0; - for term in &terms { - let numbers = term.numbers; - charge(&term.indicator, numbers.standing_cost); - match &rates[context] { - ContextRate::Fixed(rate) => charge(&term.indicator, numbers.execution_cost * rate), - ContextRate::Components(components) => { - if numbers.execution_cost == 0.0 { - } else if term.plain && own { - for &(component, rate) in components { - charge(&component.into(), numbers.execution_cost * rate); - } - } else { - let (sum, _) = context_execution - .entry(context) - .or_insert_with(|| (Expression::from(0.0), BTreeMap::new())); - *sum += numbers.execution_cost * term.indicator.clone(); - run_execution += numbers.execution_cost; - } - } - } - // Held memory. - if numbers.held_bytes <= 0.0 || timing != ExecutionTiming::QueryTime { - continue; - } - // Per byte held per latency unit. - let per_byte_ms = memory * numbers.held_bytes / 1_000.0 * milp.ms_per_latency_unit; - match milp.inputs.contexts[context].kind { - ContextKind::Query(query) => { - let latency = milp.latency[query]; - let most = milp.latency_max[query]; - let held = milp - .builder - .continuous("H_query (held for a query's latency)", None); - let indicator = term.indicator.clone(); - milp.builder.row( - "H_query (held for a query's latency)", - constraint!(held >= latency - most * (1.0 - indicator)), - ); - charge(&held.into(), per_byte_ms * query_rate[query]); - } - ContextKind::Shared(m) => { - if own && materialization != Materialization::MatAtQuery(Medium::Memory) { - milp.note_inexact("held bytes of a shareable candidate not in memory"); - } - let (longest, most_held) = *held_time.entry(context).or_insert_with(|| { - let longest = milp - .builder - .continuous("H (held until m's slowest reader)", None); - let mut most_held: f64 = 0.0; - for query in 0..milp.latency.len() { - if let Some(&reach) = milp.reach.get(&(query, m)) { - let latency = milp.latency[query]; - let most = milp.latency_max[query]; - most_held = most_held.max(most); - milp.builder.row( - "H (held until m's slowest reader)", - constraint!(longest >= latency - most * (1.0 - reach)), - ); - } - } - (longest, most_held) - }); - let components: Vec<(Option, f64)> = match &rates[context] { - ContextRate::Fixed(rate) => vec![(None, *rate)], - ContextRate::Components(components) => components - .iter() - .map(|&(component, rate)| (Some(component), rate)) - .collect(), - }; - for (component, rate) in components { - let mut both = Expression::from(0.0); - let mut slack = 0.0; - if let Some(component) = component { - both += component; - slack += 1.0; - } - if !(term.plain && own) { - both += term.indicator.clone(); - slack += 1.0; - } - if slack == 0.0 { - charge(&longest.into(), per_byte_ms * rate); - continue; - } - let held = milp - .builder - .continuous("W (held time × schedule group)", None); - milp.builder.row( - "W (held time × schedule group)", - constraint!(held >= longest - most_held * (slack - both)), - ); - charge(&held.into(), per_byte_ms * rate); - } - } - ContextKind::Ingestion(_) => {} - } - } - if let Some((_, most_per_slot)) = context_execution.get_mut(&context) { - let slot = (!own).then(|| milp.inputs.slot[candidate.0]); - let entry = most_per_slot.entry(slot).or_default(); - *entry = entry.max(run_execution); - } - } - // A context's execution at its varying rate: `V ≥ E − Ē (x[m] − z)` per - // component `z`, `Ē` the most `E` can be: per slot, its costliest run. - milp.builder.scope = Scope::SharedContexts; - for (context, (sum, most_per_slot)) in context_execution { - let most: f64 = if super::model::tests_loose_big_m() { - // Before: every term's cost. - sum.clone() - .linear_coefficients() - .map(|(_, cost)| cost) - .sum() - } else { - most_per_slot.values().sum() - }; - let ContextRate::Components(components) = &rates[context] else { - continue; - }; - let ContextKind::Shared(m) = milp.inputs.contexts[context].kind else { - continue; - }; - let region = milp - .builder - .continuous("V (context execution at a rate component)", None); - milp.builder.row( - "V (context execution at a rate component)", - constraint!(region >= sum), - ); - let active = milp.selected[&m]; - let slowest = components - .iter() - .map(|&(_, rate)| rate) - .fold(f64::INFINITY, f64::min); - let mut at_rates = Expression::from(0.0); - for &(component, rate) in components { - let at_rate = milp - .builder - .continuous("V (context execution at a rate component)", None); - milp.builder.row( - "V (context execution at a rate component)", - constraint!(at_rate >= region - most * (active - component)), - ); - charge(&at_rate.into(), rate); - at_rates += rate * at_rate; - } - if slowest.is_finite() && slowest > 0.0 { - milp.builder.row( - "V (slowest rate)", - constraint!(at_rates >= slowest * region), - ); - } - } - milp.builder.scope = Scope::Global; - raw_retention(milp, unit_cost.cost_per_disk_byte_second, &mut charge); - milp.objective_terms = objective; - Ok(()) -} - -/// Raw retention: per source, the largest raw data a query-time scan run of -/// the plan keeps, by levels. -fn raw_retention( - milp: &mut SelectionMilp<'_>, - disk: f64, - charge: &mut impl FnMut(&Expression, f64), -) { - let mut kept: BTreeMap> = BTreeMap::new(); - for (context, candidate) in milp.runs() { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = - &milp.physical.physical_node(candidate).operator - else { - continue; - }; - if milp.physical.candidates[candidate.0].timing != ExecutionTiming::QueryTime { - continue; - } - let source = source_name(source).to_string(); - for term in milp.run_terms(context, candidate) { - let bytes = term.numbers.raw_kept_bytes; - if bytes > 0.0 { - kept.entry(source.clone()) - .or_default() - .push((bytes, term.indicator)); - } - } - } - for (_, scans) in kept { - let mut levels: Vec = scans.iter().map(|(bytes, _)| *bytes).collect(); - levels.sort_by(f64::total_cmp); - levels.dedup(); - let mut below = 0.0; - for level in levels { - let at_least = milp - .builder - .continuous("v (raw retention levels)", Some(1.0)); - for (bytes, indicator) in &scans { - if *bytes >= level { - let indicator = indicator.clone(); - milp.builder.row( - "v (raw retention levels)", - constraint!(at_least >= indicator), - ); - } - } - charge(&at_least.into(), disk * (level - below)); - below = level; - } - } -} diff --git a/crates/plan-selection/src/milp/decode.rs b/crates/plan-selection/src/milp/decode.rs deleted file mode 100644 index 5b8b53466..000000000 --- a/crates/plan-selection/src/milp/decode.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! #1095 Step 6 (§2.0, "Decode and check"): the group selection MILP's raw -//! solution as a [`DagOfSelectedCandidates`], one node per candidate run. -//! -//! - A candidate computed in a context (`p[ctx, c] = 1`, a `NotMaterialized` -//! query-time candidate in a query's or a shareable candidate's context, -//! or a `NotMaterialized` ingestion-time candidate in an `I_r`) is one node per -//! context: selected in two contexts, it is two nodes; read by two -//! readers in one context, it is one. -//! - A selected shareable candidate (`x[m] = 1`) is one node, whoever reads -//! it. -//! - In each context, input `n` of a computed node (or of `m`'s own node in -//! `m`'s context) reads the context's selected candidate among its -//! producers: the node computed there, the shareable candidate's node, or -//! the candidate's node in `I_r(m)` (read in ingestion-time `m`'s context). -//! - Output reference `r` is the selected output candidate in `r`'s query -//! context. -//! -//! Only the nodes the outputs reach are kept. The MILP never selects an -//! unread candidate (row 4), so this drops nothing at a solution whose -//! objective equals `plan_cost`. - -use std::collections::{BTreeMap, BTreeSet}; - -use asap_types::stages::{ - output_references, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, PhysicalCandidateId, - SelectedEdge, SelectedNodeId, -}; - -use crate::groups::{MilpInputs, ResultSlot, RunKind}; - -/// One run of a candidate in the solution. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -enum Run { - /// Candidate `.1` computed in context `.0`. - Computed(usize, PhysicalCandidateId), - /// Shareable candidate `.0`. - Shared(PhysicalCandidateId), -} - -/// The selected candidates of a solution: per context, `p[ctx, c] = 1`, and -/// the shareable candidates with `x[m] = 1`. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub(crate) struct ChosenCandidates { - pub(crate) chosen: Vec>, - pub(crate) selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)`, the slot it reads. - pub(crate) slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, -} - -/// The selection as a DAG of candidate runs over `physical` (module -/// documentation), or why it is not one (an output or an input with no -/// selected producer). The caller validates it -/// ([`DagOfSelectedCandidates::validate`]). -pub(crate) fn decode( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, - chosen: &ChosenCandidates, -) -> Result { - // The node a read of `producer` in `context` resolves to. - let run_of = |context: usize, producer: PhysicalCandidateId| -> Option { - if !chosen.chosen[context].contains(&producer) { - return None; - } - let kind = inputs.kind[producer.0]; - if inputs.contexts[context].computes(kind) { - return Some(Run::Computed(context, producer)); - } - match kind { - RunKind::Shareable => chosen - .selected - .contains(&producer) - .then_some(Run::Shared(producer)), - RunKind::Ingestion => inputs - .ingestion_context_read_by(context) - .filter(|&ingestion| chosen.chosen[ingestion].contains(&producer)) - .map(|ingestion| Run::Computed(ingestion, producer)), - RunKind::PerContext => None, - } - }; - // The run's context for its inputs, and its candidate. - let inputs_of = |run: Run| -> Result, String> { - let (context, candidate) = match run { - Run::Computed(context, candidate) => (context, candidate), - Run::Shared(candidate) => ( - *inputs - .context_of_shareable - .get(&candidate) - .ok_or_else(|| format!("shareable candidate {} has no context", candidate.0))?, - candidate, - ), - }; - let mut reads = Vec::new(); - for (input, producers) in inputs.producer_candidates_per_input[candidate.0] - .iter() - .enumerate() - { - let slot_read = match run { - Run::Computed(..) => chosen.slot_reads.get(&(context, candidate, input)), - Run::Shared(_) => None, - }; - let producer = producers - .iter() - .filter(|producer| { - matches!(run, Run::Shared(_)) - || inputs.slot[producer.0] != inputs.slot[candidate.0] - }) - .filter(|producer| slot_read.is_none_or(|&slot| inputs.slot[producer.0] == slot)) - .find_map(|&producer| run_of(context, producer)) - .ok_or_else(|| { - format!( - "input {input} of candidate {} in context {context} has no selected producer", - candidate.0 - ) - })?; - reads.push((input, producer)); - } - Ok(reads) - }; - let outputs: Vec = output_references(&physical.query_roots) - .into_iter() - .enumerate() - .map(|(reference, (query, _))| { - inputs.output_candidates[reference] - .iter() - .find_map(|&candidate| run_of(query, candidate)) - .ok_or_else(|| format!("output reference {reference} has no selected candidate")) - }) - .collect::>()?; - // Number the runs the outputs reach, depth first from the outputs. - let mut node_of: BTreeMap = BTreeMap::new(); - let mut reads_of: BTreeMap> = BTreeMap::new(); - let mut candidate_of_node = Vec::new(); - let mut stack: Vec = outputs.iter().rev().copied().collect(); - while let Some(run) = stack.pop() { - if node_of.contains_key(&run) { - continue; - } - node_of.insert(run, SelectedNodeId(candidate_of_node.len())); - candidate_of_node.push(match run { - Run::Computed(_, candidate) | Run::Shared(candidate) => candidate, - }); - let reads = inputs_of(run)?; - stack.extend(reads.iter().map(|&(_, producer)| producer)); - reads_of.insert(run, reads); - } - let mut edges = Vec::new(); - for (run, &consumer) in &node_of { - for &(input, producer) in &reads_of[run] { - edges.push(SelectedEdge { - producer: node_of[&producer], - consumer, - input, - }); - } - } - edges.sort_by_key(|edge| (edge.consumer.0, edge.input)); - Ok(DagOfSelectedCandidates { - candidate_of_node, - edges, - node_of_output_reference: outputs.iter().map(|run| node_of[run]).collect(), - }) -} diff --git a/crates/plan-selection/src/milp/lp_diagnosis.rs b/crates/plan-selection/src/milp/lp_diagnosis.rs deleted file mode 100644 index d8bdc113f..000000000 --- a/crates/plan-selection/src/milp/lp_diagnosis.rs +++ /dev/null @@ -1,302 +0,0 @@ -//! #1095 §6.1: where the selection MILP's LP relaxation loses its bound -//! (measurement only, behind the `lp-diagnosis` feature). With -//! `ASAP_STAGE3_LP_DIAGNOSIS=`, each solve appends one JSON line: -//! - `lp`: the LP relaxation of the model as built, and `root`: HiGHS's -//! bound after its root node (presolve, cuts), both from the warm start; -//! - `lp_without`: the LP with one row family dropped, per family: how much -//! of the LP bound that family's rows hold up; -//! - `lp_tightened`: the LP with valid rows added: `Σ terms ≥ run` for each -//! run charged only on its configuration or window terms (at integral -//! binaries one of them is 1); -//! - `objective_by_family`: the objective split by the family of each -//! charged variable, at the LP's solution, at the warm start and at the -//! incumbent, so `incumbent − lp` splits into one share per family; -//! - `runs`: the charged `p` and `x` variables with the largest difference -//! between the incumbent and the LP, with their values in both. -//! -//! A diagnosed run's wall time is not comparable with an undiagnosed one. -//! The LPs and the root node run without a time limit: an LP solve and a -//! one-node MIP solve end on their own. -use std::collections::BTreeMap; -use std::time::Instant; - -use good_lp::solvers::highs::highs; -use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, SolverModel}; - -use super::model::{objective_scale, SelectionMilp}; - -/// The environment variable naming the file the diagnosis appends to. -const DIAGNOSIS_ENV: &str = "ASAP_STAGE3_LP_DIAGNOSIS"; - -/// The runs listed per solve. -const RUNS_SHOWN: usize = 40; - -/// One LP (or root node) solved. -#[derive(Debug, Clone, serde::Serialize)] -struct Bound { - /// The bound, cost per second (`None` when HiGHS gave none). - per_second: Option, - status: String, - ms: u64, -} - -/// The LP solves, before the MILP's. -pub(crate) struct Diagnosis { - path: String, - lp: Bound, - root: Bound, - lp_without: BTreeMap<&'static str, Bound>, - lp_tightened: Bound, - tightening_rows: usize, - /// The LP's columns. - lp_columns: Vec, - /// The objective, by column. - objective: Vec<(usize, f64)>, - column_families: Vec<&'static str>, - /// Per `p` and `x` column, what it stands for. - labels: BTreeMap, -} - -impl Diagnosis { - /// The LP solves of `milp` from `start` (columns), when the diagnosis is - /// asked for. - pub(crate) fn run(milp: &SelectionMilp<'_>, start: Option<&[f64]>) -> Option { - let path = std::env::var(DIAGNOSIS_ENV).ok()?; - let definitions: Vec<_> = milp - .builder - .variables - .iter_variables_with_def() - .map(|(variable, definition)| (variable, definition.clone())) - .collect(); - let column_of: std::collections::HashMap<_, usize> = definitions - .iter() - .enumerate() - .map(|(column, (variable, _))| (*variable, column)) - .collect(); - let objective: Vec<(usize, f64)> = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| (column_of[&variable], coefficient)) - .collect(); - let start_values: Option> = start.map(|start| { - definitions - .iter() - .map(|(variable, _)| *variable) - .zip(start.iter().copied()) - .collect() - }); - let scale = objective_scale(&milp.objective_terms, start_values.as_deref()); - // The relaxation: the same variables (same indices), none integral. - let relaxed = || { - let mut variables = ProblemVariables::new(); - for (_, definition) in &definitions { - variables.add( - variable() - .min(definition.get_min()) - .max(definition.get_max()), - ); - } - variables - }; - let objective_expression: Expression = milp - .objective_terms - .iter() - .map(|&(variable, coefficient)| coefficient * scale * variable) - .sum(); - let solve = |variables: ProblemVariables, rows: Vec, root: bool| { - let started = Instant::now(); - let problem = variables - .minimise(objective_expression.clone()) - .using(highs) - .with_all(rows); - let mut model = problem.try_into_inner().expect("a HiGHS model"); - model.set_option("output_flag", false); - model.set_option("threads", milp.highs_threads); - if root { - // The root node, then stop: its bound is the root bound. - model.set_option("mip_max_nodes", 1); - model.set_option("mip_rel_gap", 0.0); - model.set_option("mip_abs_gap", 0.0); - if let Some(start) = start { - let _ = model.try_set_solution(Some(start), None, None, None); - } - } - let Ok(solved) = model.try_solve() else { - return ( - Bound { - per_second: None, - status: "solver error".into(), - ms: started.elapsed().as_millis() as u64, - }, - Vec::new(), - ); - }; - let value = if root { - solved.double_info_value(c"mip_dual_bound").ok() - } else if solved.status() == highs::HighsModelStatus::Optimal { - Some(solved.objective_value()) - } else { - None - }; - let columns = if root { - Vec::new() - } else { - solved.get_solution().columns().to_vec() - }; - ( - Bound { - per_second: value.filter(|v| v.is_finite()).map(|v| v / scale), - status: format!("{:?}", solved.status()), - ms: started.elapsed().as_millis() as u64, - }, - columns, - ) - }; - let rows = &milp.builder.rows; - let families = &milp.builder.row_families; - let (lp, lp_columns) = solve(relaxed(), rows.clone(), false); - let (root, _) = solve(milp.builder.variables.clone(), rows.clone(), true); - let mut lp_without = BTreeMap::new(); - let present: std::collections::BTreeSet<&'static str> = families.iter().copied().collect(); - for family in present { - let kept: Vec = rows - .iter() - .zip(families) - .filter(|(_, &f)| f != family) - .map(|(row, _)| row.clone()) - .collect(); - lp_without.insert(family, solve(relaxed(), kept, false).0); - } - let tightening = cover_rows(milp); - let tightening_rows = tightening.len(); - let mut tightened = rows.clone(); - tightened.extend(tightening); - let (lp_tightened, _) = solve(relaxed(), tightened, false); - let candidate = |c: asap_types::stages::PhysicalCandidateId| { - let data = &milp.physical.candidates[c.0]; - format!( - "c{} {} {:?} {:?}", - c.0, - milp.physical.physical_node(c).operator.kind_name(), - data.timing, - data.materialization - ) - }; - let mut labels = BTreeMap::new(); - for (context, choices) in milp.choice.iter().enumerate() { - let place = match milp.inputs.contexts[context].kind { - crate::groups::ContextKind::Query(query) => format!("q{query}"), - crate::groups::ContextKind::Ingestion(pipeline) => match pipeline { - crate::groups::IngestionPipeline::Window(window) => { - format!("I[{}ms@{}]", window.width_ms, window.phase_ms) - } - crate::groups::IngestionPipeline::Alone(m) => format!("I[alone m{}]", m.0), - }, - crate::groups::ContextKind::Shared(m) => format!("m{}", m.0), - }; - for (&c, variable) in choices { - labels.insert(column_of[variable], format!("p[{place}] {}", candidate(c))); - } - } - for (&m, variable) in &milp.selected { - labels.insert(column_of[variable], format!("x {}", candidate(m))); - } - Some(Self { - labels, - path, - lp, - root, - lp_without, - lp_tightened, - tightening_rows, - lp_columns, - objective, - column_families: milp.builder.column_families.clone(), - }) - } - - /// Appends the line, with the warm start's and the incumbent's columns - /// (completed) and the MILP's results. - pub(crate) fn write( - self, - start: Option<&[f64]>, - incumbent: Option<&[f64]>, - objective_per_second: f64, - dual_bound_per_second: f64, - ) { - let by_family = |columns: Option<&[f64]>| -> Option> { - let columns = columns.filter(|columns| !columns.is_empty())?; - let mut split: BTreeMap<&'static str, f64> = BTreeMap::new(); - for &(column, coefficient) in &self.objective { - *split.entry(self.column_families[column]).or_default() += - coefficient * columns[column]; - } - Some(split) - }; - // Per labeled column: (LP charge, incumbent charge, LP value, - // incumbent value). - let mut runs: Vec<(String, f64, f64, f64, f64)> = Vec::new(); - if let Some(incumbent) = incumbent.filter(|columns| !columns.is_empty()) { - let mut charge: BTreeMap = BTreeMap::new(); - for &(column, coefficient) in &self.objective { - if self.labels.contains_key(&column) { - *charge.entry(column).or_default() += coefficient; - } - } - for (column, coefficient) in charge { - let (lp, at) = (self.lp_columns[column], incumbent[column]); - runs.push(( - self.labels[&column].clone(), - coefficient * lp, - coefficient * at, - lp, - at, - )); - } - runs.sort_by(|a, b| (b.2 - b.1).abs().total_cmp(&(a.2 - a.1).abs())); - runs.truncate(RUNS_SHOWN); - } - let line = serde_json::json!({ - "runs": runs, - "variables": self.column_families.len(), - "objective_per_second": objective_per_second, - "dual_bound_per_second": dual_bound_per_second, - "lp": self.lp, - "root": self.root, - "lp_without": self.lp_without, - "lp_tightened": self.lp_tightened, - "tightening_rows": self.tightening_rows, - "objective_by_family": { - "lp": by_family(Some(&self.lp_columns)), - "warm_start": by_family(start), - "incumbent": by_family(incumbent), - }, - }); - use std::io::Write; - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&self.path) - { - let _ = writeln!(file, "{line}"); - } - } -} - -/// `Σ terms ≥ run` for each run whose terms are all configuration or -/// window terms (none on the run's own indicator): at integral binaries a -/// used run has one term at 1. -fn cover_rows(milp: &SelectionMilp<'_>) -> Vec { - let mut rows = Vec::new(); - for (&(context, candidate), terms) in &milp.run_terms_cache { - if terms.is_empty() || terms.iter().any(|term| term.plain) { - continue; - } - let Some(run) = milp.run_indicator(context, candidate) else { - continue; - }; - let sum: Expression = terms.iter().map(|term| term.indicator.clone()).sum(); - rows.push(constraint!(sum >= run)); - } - rows -} diff --git a/crates/plan-selection/src/milp/mod.rs b/crates/plan-selection/src/milp/mod.rs index 7c0e9ab2f..9a2d7f7e3 100644 --- a/crates/plan-selection/src/milp/mod.rs +++ b/crates/plan-selection/src/milp/mod.rs @@ -1,26 +1,3 @@ -//! #1095 Steps 3–5: the group selection MILP (§2.0) over -//! [`MilpInputs`](crate::groups::MilpInputs): one candidate per result slot -//! per execution context, solved once with HiGHS from a warm start. -//! -//! - [`model`]: the variables and the structural rows (coverage, slots, -//! validity, used only if read, sharing, reach, reader constraints, -//! latency, accuracy), the solve and the raw solution; -//! - [`cost_terms`]: the cost model 1 objective, every coefficient a -//! number `asap-cost` computed (§3); -//! - [`warm_start`]: each query's cheapest plan, then greedy sharing, -//! HiGHS's first incumbent; -//! - [`completion`]: the exact objective of a plan given as binaries, the -//! cheapest values of the continuous variables found without a solver; -//! - [`decode`]: the solution as a `DagOfSelectedCandidates`, one node per -//! candidate run; -//! - `lp_diagnosis` (feature `lp-diagnosis`, measurement only): where the -//! LP relaxation loses its bound. -//! -//! [`crate::select`] runs them, realizes the plan once and checks it. -pub(crate) mod completion; -pub(crate) mod cost_terms; -pub(crate) mod decode; -#[cfg(feature = "lp-diagnosis")] -pub(crate) mod lp_diagnosis; +//! Stage 3's selection MILP (concepts doc, sections 3–5): its variables, +//! one function per constraint family, and the execution contexts. pub(crate) mod model; -pub(crate) mod warm_start; diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index ffd268768..772384a6f 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -1,2185 +1,162 @@ -//! #1095 §2.0: the group selection MILP's variables and rows over execution -//! contexts ([`crate::groups`]), one HiGHS solve, and the raw solution. +//! Stage 3's selection MILP, named after the formulation (concepts doc, +//! sections 3–5). //! -//! ## Variables +//! | Family | Variables | Constraints | +//! |---|---|---| +//! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) [`utilization_constraints`]; (G), (H): $y$ exists only on admissible edges | +//! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ ([`ExecutionContextVariables`]) | (R) [`retention_constraints`] | +//! | accuracy (section 4) | $E_c$, $F_{c,i}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | //! -//! | Family | Meaning | -//! |---|---| -//! | `p[ctx, c]` | in context `ctx`, candidate `c` gives its result slot (binary): computed there, or read across the boundary (a shareable candidate, or a candidate of an ingestion pipeline `I_r`) | -//! | `x[m]` | shareable candidate `m` is selected (binary) | -//! | `a[q, m]` | query `q` depends on shareable `m` (continuous, exact at integral binaries) | -//! | `y[q, m, c]` | `AND(a[q, m], p[m, c])`, for reach from above and accuracy (continuous) | -//! | `L[ctx]`, `T[q, m]`, `K[q]` | latency of a query or of a query-time shareable candidate's run and context; `m`'s latency in `q`'s; compaction (continuous) | -//! | `N[q]`, `A[q, parts]` | approximate values `q` depends on; whether it depends on one sized for `parts` shares (continuous) | -//! | `g[m, c]` | highest shared materialization: `c` is read in context `m`, which at most one query reaches (binary) | -//! | `rank` | order ranks on a context's cyclic links (continuous) | -//! -//! The cost families are [`super::cost_terms`]'s. -//! -//! ## Rows (numbered as in #1095 §2.0) -//! -//! 1. Coverage: `Σ_{c ∈ Out(r)} p[q(r), c] = 1` per output reference. -//! 2. One candidate per result slot per context: `Σ_{c ∈ slot} p[ctx, c] ≤ 1`. -//! 3. Validity: `Σ_{c' ∈ compat(c, n)} p[ctx, c'] ≥ p[ctx, c]` for each `c` -//! computed in `ctx`; `≥ x[m]` for `m`'s own inputs in its context; -//! `p[ctx, m] ≤ x[m]`, `p[m, c] ≤ p[I_r(m), c]` for reads across a -//! boundary (`r(m)` the refresh window of ingestion-time `m`). -//! 4. Used only if read, per context; a candidate of `I_r` is also read -//! by `p[m, c]` for each `m` with `r(m) = r`. -//! 5. Sharing: `x[m] ≤ Σ_ctx p[ctx, m]`. -//! 6. Reach: `a[q, m] ≥ p[q, m]`, `a[q, c] ≥ a[q, m] + p[m, c] − 1`; from -//! above, where a reader constraint counts queries. -//! 7. Latency (`plan_cost`'s: each query-time run it reaches counts once). -//! 8. Accuracy (A1), per run a query depends on. -//! 9. Reader constraints. -//! -//! A link from a candidate to one of its own result slot is dropped: with -//! one candidate per slot per context it can only be a self-loop (a -//! widened scan's `Project` lists its own memo group). Links inside a -//! larger cyclic component of a context get order ranks. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::time::Instant; - -use asap_cost::{CostInputs, CostModel}; -use asap_types::stages::{ - output_references, PhysicalASAPMemoCandidates, PhysicalCandidateId, ReaderConstraint, -}; -use asap_types::workload::QueryWorkloadEntry; -use good_lp::solvers::highs::highs; -use good_lp::{ - constraint, variable, Constraint, Expression, IntoAffineExpression, ProblemVariables, - SolverModel, Variable, -}; - -use crate::groups::{AccuracyWeight, ContextKind, MilpInputs, ResultSlot, RunKind}; +//! Every function body is `todo!()` until #1207 PRs 6 (baseline), 7 +//! (contexts) and 8 (accuracy) fill them in; until then nothing calls +//! them, hence the `dead_code` allowance below. +#![allow(dead_code)] -use super::cost_terms::{self, RunTerm}; -use super::warm_start; +use std::collections::HashMap; -/// Why the selection MILP was not built or solved. -#[derive(Debug, Clone, PartialEq)] -pub(crate) enum SelectionMilpError { - /// The MILP implements cost model 1 only (§3); the name of the model given. - UnsupportedCostModel(&'static str), - /// `entries` and the query roots differ in number. - EntryCount { roots: usize, entries: usize }, - /// No plan meets every row. - Infeasible, - /// A rate or another number `asap-cost` could not compute. - Cost(String), - /// HiGHS failed. - Solver(String), -} - -/// How the MILP is built. -#[derive(Debug, Clone, Copy)] -pub(crate) struct SelectionMilpOptions { - /// Write the accuracy rows (row 8); off when the facade ignores - /// accuracy guarantees. - pub(crate) enforce_accuracy: bool, - /// Solve each independent part alone ([`independent_parts`]); off only - /// to compare with one solve in tests. - pub(crate) split_into_parts: bool, - /// HiGHS's threads (its `threads` option). One thread makes ties among - /// plans of equal cost break the same way on every run. - pub(crate) highs_threads: i32, -} +use good_lp::{Constraint, Variable}; -impl Default for SelectionMilpOptions { - fn default() -> Self { - Self { - enforce_accuracy: true, - split_into_parts: true, - highs_threads: 1, - } - } -} +use asap_types::physical::EvaluationSchedule; +use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; -/// Which part of the model a variable or row belongs to, for the size -/// report: the query contexts, the `I_r`, the shareable candidates' contexts -/// (with their `x`, reach and schedule terms), or the rest. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)] -pub(crate) enum Scope { - Queries, - Ingestion, - SharedContexts, - Global, -} - -/// The size of one family (or scope) of variables and rows. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)] -pub(crate) struct FamilySize { - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - /// Nonzero coefficients of its rows. - pub(crate) nonzeros: usize, -} - -impl FamilySize { - fn add(&mut self, other: FamilySize) { - self.variables += other.variables; - self.binaries += other.binaries; - self.rows += other.rows; - self.nonzeros += other.nonzeros; - } -} - -/// Variables and rows, counted per family and per scope. -pub(crate) struct ModelBuilder { - pub(crate) variables: ProblemVariables, - pub(crate) rows: Vec, - /// The family of each row of `rows`. - pub(crate) row_families: Vec<&'static str>, - pub(crate) families: BTreeMap<&'static str, FamilySize>, - pub(crate) scopes: BTreeMap, - /// Where new variables and rows are counted. - pub(crate) scope: Scope, - /// The family of each variable, in creation order. - #[cfg(feature = "lp-diagnosis")] - pub(crate) column_families: Vec<&'static str>, -} - -impl ModelBuilder { - fn new() -> Self { - Self { - variables: ProblemVariables::new(), - rows: Vec::new(), - row_families: Vec::new(), - families: BTreeMap::new(), - scopes: BTreeMap::new(), - scope: Scope::Global, - #[cfg(feature = "lp-diagnosis")] - column_families: Vec::new(), - } - } - - fn count(&mut self, family: &'static str, size: FamilySize) { - #[cfg(feature = "lp-diagnosis")] - if size.variables > 0 { - self.column_families.push(family); - } - self.families.entry(family).or_default().add(size); - self.scopes.entry(self.scope).or_default().add(size); - } - - pub(crate) fn binary(&mut self, family: &'static str) -> Variable { - self.count( - family, - FamilySize { - variables: 1, - binaries: 1, - ..FamilySize::default() - }, - ); - self.variables.add(variable().binary()) - } - - /// A continuous variable in `[0, max]` (unbounded above for `None`). - pub(crate) fn continuous(&mut self, family: &'static str, max: Option) -> Variable { - self.count( - family, - FamilySize { - variables: 1, - ..FamilySize::default() - }, - ); - let definition = variable().min(0.0); - self.variables.add(match max { - Some(max) => definition.max(max), - None => definition, - }) - } - - pub(crate) fn row(&mut self, family: &'static str, row: Constraint) { - let nonzeros = row.expression().linear_coefficients().count(); - self.count( - family, - FamilySize { - rows: 1, - nonzeros, - ..FamilySize::default() - }, - ); - self.rows.push(row); - self.row_families.push(family); - } - - /// `and ≥ Σ terms − (n − 1)`, `and ∈ [0, 1]`: the AND of `n` - /// indicators, tight under a cost or a `≤` row that charges it. - pub(crate) fn and_at_least(&mut self, family: &'static str, terms: &[Expression]) -> Variable { - let and = self.continuous(family, Some(1.0)); - let sum: Expression = terms.iter().cloned().sum(); - let count = terms.len() as f64; - self.row(family, constraint!(and >= sum - (count - 1.0))); - and - } -} - -/// One way a candidate's input is read in a context: `(reader, input)` -/// with the reader computed in that context, the query's output reference, -/// or the shareable candidate's own input. +/// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub(crate) enum Reader { - Candidate(PhysicalCandidateId, usize), - Output(usize), - OwnInput(usize), -} - -/// One input of a candidate: `(reader, input)`. -type InputOf = (PhysicalCandidateId, usize); - -/// The slot links of one multi-slot input: `(slot, e)`. -type SlotLinks = Vec<(ResultSlot, Variable)>; - -/// The built model, before the solve. -pub(crate) struct SelectionMilp<'a> { - pub(crate) builder: ModelBuilder, - pub(crate) physical: &'a PhysicalASAPMemoCandidates, - pub(crate) inputs: &'a MilpInputs, - pub(crate) entries: &'a [QueryWorkloadEntry], - pub(crate) cost_inputs: &'a CostInputs, - /// Per context, `p[ctx, c]` per option. - pub(crate) choice: Vec>, - /// Per context, per option, the reads of it in that context. - pub(crate) readers: Vec>>, - /// `x[m]` per shareable candidate with a context. - pub(crate) selected: BTreeMap, - /// `a[q, m]`, for each query that may depend on shareable `m`. - pub(crate) reach: HashMap<(usize, PhysicalCandidateId), Variable>, - /// Per shareable candidate, the shareable candidates whose contexts may - /// read it. - pub(crate) shared_readers: BTreeMap>, - /// `L[q]`. - pub(crate) latency: Vec, - /// An upper bound of each `L[q]` (its bound, or the most it can be), in - /// latency units. - pub(crate) latency_max: Vec, - /// Milliseconds per latency unit: every latency variable (`L`, `T`, `K` - /// and the held times) counts in these units, chosen so the largest - /// possible latency is about 1,000 units (with sub-microsecond work per - /// execution, milliseconds fall below HiGHS's absolute tolerances). - pub(crate) ms_per_latency_unit: f64, - /// The cost model 1 objective, in cost per second, without the request - /// fees (constant for every plan, added after selection). - pub(crate) objective_terms: Vec<(Variable, f64)>, - /// The request fees per second (the same for every plan). - pub(crate) request_cost_per_second: f64, - /// Terms encoded inexactly, by reason. - pub(crate) inexact: BTreeMap<&'static str, usize>, - /// The terms of each run `(context, candidate)`, cached; a shareable - /// candidate's own run is keyed by its own context. - pub(crate) run_terms_cache: HashMap<(usize, PhysicalCandidateId), Vec>, - /// Runs whose terms are being built (cycles of links). - pub(crate) terms_in_progress: HashSet<(usize, PhysicalCandidateId)>, - /// Per run and outputs, the variable `≥` its terms giving them. - pub(crate) producer_gives: HashMap<(usize, PhysicalCandidateId, Vec), Variable>, - /// Per query-time scan run, its window terms - /// ([`cost_terms::window_terms`]). - pub(crate) window_cache: HashMap<(usize, PhysicalCandidateId), Option>>, - /// For each input whose producers lie in several result slots (a - /// factored root over the private nodes of several implementations), - /// per context: `e[ctx, r, n, slot]`, the slot input `n` of computed - /// reader `r` reads (binary). Row 4 counts these reads, so a selected - /// producer in the other slot is not justified by `r`. - pub(crate) slot_links: - BTreeMap<(usize, PhysicalCandidateId, usize), Vec<(ResultSlot, Variable)>>, - /// [`SelectionMilpOptions::split_into_parts`]. - pub(crate) split_into_parts: bool, - /// [`SelectionMilpOptions::highs_threads`]. - pub(crate) highs_threads: i32, -} - -/// Variables and rows of one model. -#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] -pub(crate) struct SelectionMilpSize { - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - pub(crate) nonzeros: usize, - pub(crate) families: BTreeMap<&'static str, FamilySize>, - pub(crate) scopes: BTreeMap, - pub(crate) inexact: BTreeMap<&'static str, usize>, -} - -/// The raw solution of a solve run to proven optimality, ready to decode. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct SelectionMilpSolution { - /// The objective: cost per second without the request fees. - pub(crate) cost_per_second: f64, - /// The request fees per second, the same for every plan. - pub(crate) request_cost_per_second: f64, - /// HiGHS's dual bound on `cost_per_second`. - pub(crate) cost_lower_bound_per_second: f64, - /// Per context, the options with `p[ctx, c] = 1`. - pub(crate) chosen: Vec>, - /// The shareable candidates with `x[m] = 1`. - pub(crate) selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)` of a used reader, the - /// slot it reads. - pub(crate) slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, - pub(crate) size: SelectionMilpSize, - /// Per independent part, how its solve went, in part order. - pub(crate) parts: Vec, - pub(crate) build_ms: u64, - /// Building the warm start, HiGHS's first incumbent. - pub(crate) warm_start_ms: u64, - /// The warm start's objective (like `cost_per_second`), when its - /// completion costed it. - pub(crate) warm_start_cost_per_second: Option, - pub(crate) solve_ms: u64, -} - -impl<'a> SelectionMilp<'a> { - pub(crate) fn query_count(&self) -> usize { - self.physical.query_roots.len() - } - - /// The context where `context` reads a candidate of an `I_r` - /// ([`MilpInputs::ingestion_context_read_by`]). - pub(crate) fn ingestion_context_read_by(&self, context: usize) -> Option { - self.inputs.ingestion_context_read_by(context) - } - - /// The context of shareable candidate `m`. - pub(crate) fn context_of(&self, shareable: PhysicalCandidateId) -> Option { - self.inputs.context_of_shareable.get(&shareable).copied() - } - - pub(crate) fn kind(&self, candidate: PhysicalCandidateId) -> RunKind { - self.inputs.kind[candidate.0] - } - - pub(crate) fn slot(&self, candidate: PhysicalCandidateId) -> ResultSlot { - self.inputs.slot[candidate.0] - } - - /// Whether `candidate` is computed in context `context`. - pub(crate) fn computes(&self, context: usize, candidate: PhysicalCandidateId) -> bool { - self.inputs.contexts[context].computes(self.kind(candidate)) - } - - /// The indicator of run `(context, candidate)`: `p[ctx, c]` for a - /// candidate computed there, `x[m]` for a shareable candidate's own run - /// in its own context. - pub(crate) fn run_indicator( - &self, - context: usize, - candidate: PhysicalCandidateId, - ) -> Option { - if self.inputs.contexts[context].kind == ContextKind::Shared(candidate) { - return self.selected.get(&candidate).copied(); - } - if !self.computes(context, candidate) { - return None; - } - self.choice[context].get(&candidate).copied() - } - - /// The context where run `(context, candidate)` reads its inputs: its - /// own (a shareable candidate's own run reads in its context too). - pub(crate) fn runs_at_query_time(&self, context: usize) -> bool { - self.inputs.contexts[context].at_query_time - } - - pub(crate) fn size(&self) -> SelectionMilpSize { - let families = self.builder.families.clone(); - SelectionMilpSize { - variables: families.values().map(|f| f.variables).sum(), - binaries: families.values().map(|f| f.binaries).sum(), - rows: families.values().map(|f| f.rows).sum(), - nonzeros: families.values().map(|f| f.nonzeros).sum(), - families, - scopes: self.builder.scopes.clone(), - inexact: self.inexact.clone(), - } - } - - pub(crate) fn note_inexact(&mut self, reason: &'static str) { - *self.inexact.entry(reason).or_default() += 1; - } - - /// The scope of context `context`. - pub(crate) fn scope_of(&self, context: usize) -> Scope { - match self.inputs.contexts[context].kind { - ContextKind::Query(_) => Scope::Queries, - ContextKind::Ingestion(_) => Scope::Ingestion, - ContextKind::Shared(_) => Scope::SharedContexts, - } - } - - /// Run `(context, candidate)`'s terms ([`cost_terms::run_terms`]), - /// cached. - pub(crate) fn run_terms( - &mut self, - context: usize, - candidate: PhysicalCandidateId, - ) -> Vec { - if let Some(terms) = self.run_terms_cache.get(&(context, candidate)) { - return terms.clone(); - } - let scope_here = self.scope_of(context); - let scope = std::mem::replace(&mut self.builder.scope, scope_here); - let terms = cost_terms::run_terms(self, context, candidate); - self.builder.scope = scope; - self.run_terms_cache - .insert((context, candidate), terms.clone()); - terms - } - - /// Every run of the model: `(context, candidate)` for each candidate - /// computed in a context, and each shareable candidate's own run. - pub(crate) fn runs(&self) -> Vec<(usize, PhysicalCandidateId)> { - let mut runs = Vec::new(); - for (context, options) in self.choice.iter().enumerate() { - for &candidate in options.keys() { - if self.computes(context, candidate) { - runs.push((context, candidate)); - } - } - if let ContextKind::Shared(own) = self.inputs.contexts[context].kind { - if self.selected.contains_key(&own) { - runs.push((context, own)); - } - } - } - runs - } - - /// The query contexts that may depend on shareable `m`, ascending. - pub(crate) fn queries_reaching(&self, shareable: PhysicalCandidateId) -> Vec { - let mut queries: Vec = (0..self.query_count()) - .filter(|&query| self.reach.contains_key(&(query, shareable))) - .collect(); - queries.sort_unstable(); - queries - } -} - -/// Builds the MILP over `inputs` (§2.0): `entries[q]` is query `q`'s -/// workload entry. -pub(crate) fn build_selection_milp<'a>( - physical: &'a PhysicalASAPMemoCandidates, - inputs: &'a MilpInputs, - entries: &'a [QueryWorkloadEntry], - cost_inputs: &'a CostInputs, - options: SelectionMilpOptions, -) -> Result, SelectionMilpError> { - match cost_inputs.cost_model { - CostModel::UsageOverTime(_) => {} - CostModel::ProvisionedCapacity(_) => { - return Err(SelectionMilpError::UnsupportedCostModel( - "ProvisionedCapacity (cost model 2)", - )) - } - CostModel::ClusterCapacity(_) => { - return Err(SelectionMilpError::UnsupportedCostModel( - "ClusterCapacity (cost model 3)", - )) - } - } - let query_count = physical.query_roots.len(); - if entries.len() != query_count { - return Err(SelectionMilpError::EntryCount { - roots: query_count, - entries: entries.len(), - }); - } - let mut milp = SelectionMilp { - builder: ModelBuilder::new(), - physical, - inputs, - entries, - cost_inputs, - choice: Vec::new(), - readers: Vec::new(), - selected: BTreeMap::new(), - reach: HashMap::new(), - shared_readers: BTreeMap::new(), - latency: Vec::new(), - latency_max: Vec::new(), - ms_per_latency_unit: 1.0, - objective_terms: Vec::new(), - request_cost_per_second: 0.0, - inexact: BTreeMap::new(), - run_terms_cache: HashMap::new(), - terms_in_progress: HashSet::new(), - producer_gives: HashMap::new(), - window_cache: HashMap::new(), - slot_links: BTreeMap::new(), - split_into_parts: options.split_into_parts, - highs_threads: options.highs_threads, - }; - choice_variables(&mut milp); - structural_rows(&mut milp); - acyclic_rows(&mut milp); - acyclic_context_rows(&mut milp); - reach_rows(&mut milp); - reader_constraint_rows(&mut milp); - latency_rows(&mut milp)?; - if options.enforce_accuracy { - accuracy_rows(&mut milp); - } - cost_terms::encode_objective(&mut milp)?; - Ok(milp) -} - -/// `p`, `x` and, per context, the reads of each option. -fn choice_variables(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let output_refs = output_references(&milp.physical.query_roots); - for (context, data) in inputs.contexts.iter().enumerate() { - milp.builder.scope = milp.scope_of(context); - let options: BTreeSet = data.options.iter().copied().collect(); - let mut readers: BTreeMap> = BTreeMap::new(); - let mut choices = BTreeMap::new(); - for &option in &options { - choices.insert(option, milp.builder.binary("p (choices)")); - } - for &reader in &options { - if !data.computes(inputs.kind[reader.0]) { - continue; - } - for (input, producers) in inputs.producer_candidates_per_input[reader.0] - .iter() - .enumerate() - { - for &producer in producers { - // A read of its own result slot is a self-loop. - if options.contains(&producer) - && inputs.slot[producer.0] != inputs.slot[reader.0] - { - readers - .entry(producer) - .or_default() - .push(Reader::Candidate(reader, input)); - } - } - } - } - match data.kind { - ContextKind::Query(query) => { - for (reference, (q, _)) in output_refs.iter().enumerate() { - if *q != query { - continue; - } - for &out in &inputs.output_candidates[reference] { - if options.contains(&out) { - readers - .entry(out) - .or_default() - .push(Reader::Output(reference)); - } - } - } - } - ContextKind::Shared(own) => { - for (input, producers) in inputs.producer_candidates_per_input[own.0] - .iter() - .enumerate() - { - for &producer in producers { - if options.contains(&producer) { - readers - .entry(producer) - .or_default() - .push(Reader::OwnInput(input)); - } - } - } - } - ContextKind::Ingestion(_) => {} - } - for &reader in &options { - if !data.computes(inputs.kind[reader.0]) { - continue; - } - for (input, producers) in inputs.producer_candidates_per_input[reader.0] - .iter() - .enumerate() - { - let slots: BTreeSet = producers - .iter() - .filter(|producer| options.contains(producer)) - .map(|producer| inputs.slot[producer.0]) - .filter(|&slot| slot != inputs.slot[reader.0]) - .collect(); - if slots.len() > 1 { - let links = slots - .into_iter() - .map(|slot| (slot, milp.builder.binary("e (slot an input reads)"))) - .collect(); - milp.slot_links.insert((context, reader, input), links); - } - } - } - milp.choice.push(choices); - milp.readers.push(readers); - } - milp.builder.scope = Scope::SharedContexts; - for &shareable in inputs.context_of_shareable.keys() { - let selected = milp.builder.binary("x (shareable candidates)"); - milp.selected.insert(shareable, selected); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(reader) = data.kind { - for &option in &data.options { - if inputs.kind[option.0] == RunKind::Shareable { - milp.shared_readers.entry(option).or_default().push(reader); - } - } - let _ = context; - } - } - milp.builder.scope = Scope::Global; -} - -/// Rows 1–5. -fn structural_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let output_refs = output_references(&milp.physical.query_roots); - // Per `I_r`, the contexts of the ingestion-time shareable candidates of - // refresh window `r`. - let mut readers_of_ingestion: BTreeMap> = BTreeMap::new(); - for (m, &at) in &inputs.ingestion_context_of_shareable { - readers_of_ingestion - .entry(at) - .or_default() - .push(inputs.context_of_shareable[m]); - } - // 1. Coverage. - milp.builder.scope = Scope::Queries; - for (reference, (query, _)) in output_refs.iter().enumerate() { - let sum: Expression = inputs.output_candidates[reference] - .iter() - .filter_map(|out| milp.choice[*query].get(out).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("1 coverage", constraint!(sum == 1.0)); - } - for context in 0..inputs.contexts.len() { - milp.builder.scope = milp.scope_of(context); - let data = &inputs.contexts[context]; - // 2. One candidate per slot. - let mut per_slot: BTreeMap> = BTreeMap::new(); - for (&option, &variable) in &milp.choice[context] { - per_slot - .entry(inputs.slot[option.0]) - .or_default() - .push(variable); - } - for variables in per_slot.values().filter(|v| v.len() > 1) { - let sum: Expression = variables.iter().copied().map(Expression::from).sum(); - milp.builder - .row("2 one candidate per slot", constraint!(sum <= 1.0)); - } - // 3. Validity. - let choices = milp.choice[context].clone(); - let producers_sum = |reader: PhysicalCandidateId, input: usize| -> Expression { - inputs.producer_candidates_per_input[reader.0][input] - .iter() - .filter(|producer| inputs.slot[producer.0] != inputs.slot[reader.0]) - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum() - }; - for (&option, &variable) in &choices { - let kind = inputs.kind[option.0]; - if data.computes(kind) { - for input in 0..inputs.producer_candidates_per_input[option.0].len() { - let sum = producers_sum(option, input); - milp.builder.row("3 validity", constraint!(sum >= variable)); - } - } else { - match kind { - RunKind::Shareable => { - let selected = milp.selected[&option]; - milp.builder - .row("3 validity (boundary)", constraint!(variable <= selected)); - } - RunKind::Ingestion => { - let source = milp - .ingestion_context_read_by(context) - .and_then(|ingestion| milp.choice[ingestion].get(&option).copied()); - match source { - Some(source) => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= source)), - None => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= 0.0)), - } - } - RunKind::PerContext => milp - .builder - .row("3 validity (boundary)", constraint!(variable <= 0.0)), - } - } - } - if let ContextKind::Shared(own) = data.kind { - let selected = milp.selected[&own]; - for input in 0..inputs.producer_candidates_per_input[own.0].len() { - let sum: Expression = inputs.producer_candidates_per_input[own.0][input] - .iter() - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("3 validity", constraint!(sum >= selected)); - } - } - // The slot each multi-slot input reads: one, and only a slot the - // context gives. - let links: Vec<(InputOf, SlotLinks)> = milp - .slot_links - .range((context, PhysicalCandidateId(0), 0)..(context + 1, PhysicalCandidateId(0), 0)) - .map(|(&(_, reader, input), links)| ((reader, input), links.clone())) - .collect(); - for ((reader, input), links) in links { - let used = choices[&reader]; - let sum: Expression = links.iter().map(|&(_, link)| Expression::from(link)).sum(); - milp.builder - .row("3 validity (slot links)", constraint!(sum == used)); - for (slot, link) in links { - let given: Expression = inputs.producer_candidates_per_input[reader.0][input] - .iter() - .filter(|producer| inputs.slot[producer.0] == slot) - .filter_map(|producer| choices.get(producer).copied()) - .map(Expression::from) - .sum(); - milp.builder - .row("3 validity (slot links)", constraint!(link <= given)); - } - } - // 4. Used only if read. - for (&option, &variable) in &choices { - let mut reads = Expression::from(0.0); - for reader in milp.readers[context].get(&option).into_iter().flatten() { - match *reader { - Reader::Candidate(reader, input) => { - match milp.slot_links.get(&(context, reader, input)) { - Some(links) => { - let slot = inputs.slot[option.0]; - for &(at, link) in links { - if at == slot { - reads += link; - } - } - } - None => reads += choices[&reader], - } - } - Reader::Output(_) => reads += 1.0, - Reader::OwnInput(_) => { - if let ContextKind::Shared(own) = data.kind { - reads += milp.selected[&own]; - } - } - } - } - // A candidate of `I_r` is also read by the contexts of the - // ingestion-time shareable candidates of refresh window `r`. - for &other in readers_of_ingestion.get(&context).into_iter().flatten() { - if let Some(&read) = milp.choice[other].get(&option) { - reads += read; - } - } - milp.builder - .row("4 used only if read", constraint!(variable <= reads)); - } - } - // 5. Sharing. - milp.builder.scope = Scope::SharedContexts; - let mut reads_of: BTreeMap = BTreeMap::new(); - for choices in &milp.choice { - for (&option, &variable) in choices { - if inputs.kind[option.0] == RunKind::Shareable { - *reads_of.entry(option).or_default() += variable; - } - } - } - let selected = milp.selected.clone(); - for (shareable, selected) in selected { - let reads = reads_of.remove(&shareable).unwrap_or_default(); - milp.builder - .row("5 sharing", constraint!(selected <= reads)); - } - milp.builder.scope = Scope::Global; -} - -/// Order ranks on the links inside each cyclic component of a context's -/// links between candidates it computes (none when the slot graph is -/// acyclic). -fn acyclic_rows(milp: &mut SelectionMilp<'_>) { - for context in 0..milp.choice.len() { - milp.builder.scope = milp.scope_of(context); - let computed: Vec = milp.choice[context] - .keys() - .copied() - .filter(|&c| milp.computes(context, c)) - .collect(); - let index: HashMap = - computed.iter().enumerate().map(|(i, &c)| (c, i)).collect(); - let mut edges: Vec<(usize, usize)> = Vec::new(); - for (&producer, readers) in &milp.readers[context] { - let Some(&from) = index.get(&producer) else { - continue; - }; - for reader in readers { - if let Reader::Candidate(reader, _) = reader { - edges.push((from, index[reader])); - } - } - } - let components = strongly_connected_components(computed.len(), &edges); - let mut rank: HashMap = HashMap::new(); - for &(producer, reader) in &edges { - let size = components.size_of(reader); - if components.of[producer] != components.of[reader] || size < 2 { - continue; - } - let n = size as f64; - let [producer_rank, reader_rank] = [producer, reader].map(|node| { - *rank - .entry(node) - .or_insert_with(|| milp.builder.continuous("acyclic ranks", Some(n - 1.0))) - }); - // Both used: the reader ranks above the producer. (Within one - // context a used reader reads the used candidate of the slot.) - let (p, r) = ( - milp.choice[context][&computed[producer]], - milp.choice[context][&computed[reader]], - ); - milp.builder.row( - "acyclic ranks", - constraint!(producer_rank + 1.0 <= reader_rank + n * (2.0 - p - r)), - ); - } - } - milp.builder.scope = Scope::Global; -} - -/// Order ranks across contexts: on the graph whose nodes are the shareable -/// candidates' own runs and the runs of each `I_r`, with an edge from a run -/// to each shareable candidate or candidate of an `I_r` its context may -/// read (for a run of `I_r`, its producers there). Rows 3–5 alone admit a -/// cycle of reads across contexts that justifies itself with no output -/// above it (two shareable candidates of one memo group, each listed among -/// the other's producers); only edges inside a cyclic component get a row. -fn acyclic_context_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - // Nodes: (context of the run, candidate). - let mut nodes: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (&m, &context) in &inputs.context_of_shareable { - nodes.push((context, m)); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Ingestion(_) = data.kind { - for &c in milp.choice[context].keys() { - if milp.computes(context, c) { - nodes.push((context, c)); - } - } - } - } - let index: HashMap<(usize, PhysicalCandidateId), usize> = - nodes.iter().enumerate().map(|(i, &run)| (run, i)).collect(); - // The node a read of `option` in `context` resolves to. - let node_read = |context: usize, option: PhysicalCandidateId| -> Option { - let run_context = match inputs.kind[option.0] { - RunKind::Shareable => milp.context_of(option)?, - RunKind::Ingestion => milp.ingestion_context_read_by(context)?, - RunKind::PerContext => return None, - }; - index.get(&(run_context, option)).copied() - }; - // Edges with their activity: `(from, to, indicators that must all be 1)`. - let mut edges: Vec<(usize, usize, Vec)> = Vec::new(); - for (from, &(context, candidate)) in nodes.iter().enumerate() { - if let ContextKind::Ingestion(_) = inputs.contexts[context].kind { - let reader = milp.choice[context][&candidate]; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - for &producer in producers { - if let (Some(to), Some(&read)) = ( - node_read(context, producer), - milp.choice[context].get(&producer), - ) { - if to != from { - edges.push((from, to, vec![reader, read])); - } - } - } - } - } else { - for (&option, &read) in &milp.choice[context] { - if let Some(to) = node_read(context, option) { - edges.push((from, to, vec![read])); - } - } - } - } - let plain: Vec<(usize, usize)> = edges.iter().map(|&(from, to, _)| (from, to)).collect(); - let components = strongly_connected_components(nodes.len(), &plain); - milp.builder.scope = Scope::SharedContexts; - let mut rank: HashMap = HashMap::new(); - for (from, to, active) in edges { - let size = components.size_of(from); - if components.of[from] != components.of[to] || size < 2 { - continue; - } - let n = size as f64; - let [reader_rank, read_rank] = [from, to].map(|node| { - *rank.entry(node).or_insert_with(|| { - milp.builder - .continuous("acyclic ranks (contexts)", Some(n - 1.0)) - }) - }); - let slack = active.len() as f64; - let on: Expression = active.into_iter().map(Expression::from).sum(); - milp.builder.row( - "acyclic ranks (contexts)", - constraint!(read_rank + 1.0 <= reader_rank + n * (slack - on)), - ); - } - milp.builder.scope = Scope::Global; -} - -/// The strongly connected components of a directed graph. -pub(crate) struct Components { - /// Per node, its component. - pub(crate) of: Vec, - /// Per component, its number of nodes. - pub(crate) sizes: Vec, -} - -impl Components { - pub(crate) fn size_of(&self, node: usize) -> usize { - self.sizes[self.of[node]] - } -} - -/// The strongly connected components of the graph on nodes `0..count` with -/// `edges` `(from, to)` (Tarjan's algorithm, without recursion). -pub(crate) fn strongly_connected_components(count: usize, edges: &[(usize, usize)]) -> Components { - let mut successors = vec![Vec::new(); count]; - for &(from, to) in edges { - successors[from].push(to); - } - const UNVISITED: usize = usize::MAX; - let mut order = vec![UNVISITED; count]; - let mut lowest = vec![0; count]; - let mut on_stack = vec![false; count]; - let mut stack = Vec::new(); - let mut of = vec![UNVISITED; count]; - let mut sizes = Vec::new(); - let mut next = 0; - for root in 0..count { - if order[root] != UNVISITED { - continue; - } - let mut path = vec![(root, 0)]; - order[root] = next; - lowest[root] = next; - next += 1; - stack.push(root); - on_stack[root] = true; - while let Some(&mut (node, ref mut successor)) = path.last_mut() { - if let Some(&to) = successors[node].get(*successor) { - *successor += 1; - if order[to] == UNVISITED { - order[to] = next; - lowest[to] = next; - next += 1; - stack.push(to); - on_stack[to] = true; - path.push((to, 0)); - } else if on_stack[to] { - lowest[node] = lowest[node].min(order[to]); - } - continue; - } - path.pop(); - if let Some(&(parent, _)) = path.last() { - lowest[parent] = lowest[parent].min(lowest[node]); - } - if lowest[node] == order[node] { - let component = sizes.len(); - let mut size = 0; - loop { - let member = stack.pop().expect("the node is on the stack"); - on_stack[member] = false; - of[member] = component; - size += 1; - if member == node { - break; - } - } - sizes.push(size); - } - } - } - Components { of, sizes } -} - -/// The shareable candidates in order: each before every shareable -/// candidate its context may read (readers first), so `a[q, m]` exists -/// before the reach rows of what `m` reads. A cycle among contexts is -/// broken at the first candidate met. -pub(crate) fn shareables_readers_first(milp: &SelectionMilp<'_>) -> Vec { - let inputs = milp.inputs; - let mut order = Vec::new(); - let mut state: BTreeMap = BTreeMap::new(); - // Post-order over "m reads c" (children = what m reads), reversed. - for &start in inputs.context_of_shareable.keys() { - if state.contains_key(&start) { - continue; - } - let mut stack: Vec<(PhysicalCandidateId, bool)> = vec![(start, false)]; - while let Some((node, done)) = stack.pop() { - if done { - state.insert(node, 2); - order.push(node); - continue; - } - if state.contains_key(&node) { - continue; - } - state.insert(node, 1); - stack.push((node, true)); - let context = inputs.context_of_shareable[&node]; - for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable && !state.contains_key(&option) { - stack.push((option, false)); - } - } - } - } - order.reverse(); - order -} - -/// Row 6: reach, from below for every `(q, m)`, from above where a reader -/// constraint counts queries. -fn reach_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - milp.builder.scope = Scope::SharedContexts; - let order = shareables_readers_first(milp); - // Which queries may depend on each shareable candidate: read in their - // context, or in the context of one they may depend on. - let mut may_reach: BTreeMap> = BTreeMap::new(); - for query in 0..milp.query_count() { - for &option in &inputs.contexts[query].options { - if inputs.kind[option.0] == RunKind::Shareable { - may_reach.entry(option).or_default().insert(query); - } - } - } - for &shareable in &order { - let queries = may_reach.get(&shareable).cloned().unwrap_or_default(); - let context = inputs.context_of_shareable[&shareable]; - for &option in &inputs.contexts[context].options { - if inputs.kind[option.0] == RunKind::Shareable { - may_reach - .entry(option) - .or_default() - .extend(queries.iter().copied()); - } - } - } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.builder.continuous("a (reach)", Some(1.0)); - milp.reach.insert((query, shareable), reach); - } - } - for &shareable in &order { - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - if let Some(&direct) = milp.choice[query].get(&shareable) { - milp.builder - .row("6 reach from below", constraint!(reach >= direct)); - } - for &reader in milp.shared_readers.get(&shareable).into_iter().flatten() { - let Some(&above) = milp.reach.get(&(query, reader)) else { - continue; - }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; - milp.builder.row( - "6 reach from below", - constraint!(reach >= above + read - 1.0), - ); - } - } - } - // From above, where needed. - let exact = shareables_needing_exact_reach(milp); - for &shareable in &order { - if !exact.contains(&shareable) { - continue; - } - for &query in may_reach.get(&shareable).into_iter().flatten() { - let reach = milp.reach[&(query, shareable)]; - let mut sources = Expression::from(0.0); - if let Some(&direct) = milp.choice[query].get(&shareable) { - sources += direct; - } - for &reader in milp - .shared_readers - .get(&shareable) - .cloned() - .unwrap_or_default() - .iter() - { - let Some(&above) = milp.reach.get(&(query, reader)) else { - continue; - }; - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&shareable]; - let through = milp.builder.continuous("y (reach through)", Some(1.0)); - milp.builder - .row("6 reach from above", constraint!(through <= above)); - milp.builder - .row("6 reach from above", constraint!(through <= read)); - sources += through; - } - milp.builder - .row("6 reach from above", constraint!(reach <= sources)); - } - } - milp.builder.scope = Scope::Global; -} - -/// The reader constraints of each candidate. -pub(crate) fn reader_constraints_of_candidates<'a>( - milp: &SelectionMilp<'a>, -) -> BTreeMap> { - let physical: &'a PhysicalASAPMemoCandidates = milp.physical; - let mut of_candidate: BTreeMap> = - BTreeMap::new(); - for constraint in &physical.reader_constraints { - of_candidate - .entry(constraint.candidate()) - .or_default() - .push(constraint); - } - of_candidate -} - -/// The shareable candidates whose `a` must be exact from above: those a -/// counting reader constraint (minimum readers, highest shared -/// materialization) names or whose context holds a constrained candidate, -/// and transitively every shareable candidate whose context reads one. -fn shareables_needing_exact_reach(milp: &SelectionMilp<'_>) -> BTreeSet { - let inputs = milp.inputs; - let constraints = reader_constraints_of_candidates(milp); - let mut stack: Vec = Vec::new(); - for (candidate, constraints) in &constraints { - let counted = constraints.iter().any(|constraint| { - matches!( - constraint, - ReaderConstraint::MinimumQueryReaders { .. } - | ReaderConstraint::HighestSharedQueryMaterialization { .. } - ) - }); - if !counted { - continue; - } - if inputs.kind[candidate.0] == RunKind::Shareable { - stack.push(*candidate); - // Highest shared materialization reads the reach of its - // reading contexts. - stack.extend(milp.shared_readers.get(candidate).into_iter().flatten()); - } - for (context, data) in inputs.contexts.iter().enumerate() { - if let ContextKind::Shared(own) = data.kind { - if milp.choice[context].contains_key(candidate) { - stack.push(own); - } - } - } - } - let mut exact = BTreeSet::new(); - while let Some(shareable) = stack.pop() { - if exact.insert(shareable) { - stack.extend(milp.shared_readers.get(&shareable).into_iter().flatten()); - } - } - exact -} - -/// The queries of `query_outputs` buckets (one per query, each its output -/// references). -fn bucket_queries(query_of_reference: &[usize], query_outputs: &[Vec]) -> Vec { - query_outputs - .iter() - .filter_map(|bucket| { - bucket - .first() - .map(|&reference| query_of_reference[reference]) - }) - .collect() -} - -/// Row 9: reader constraints, on every node of the constrained candidate. -/// A node in a query's context is reached by that query alone; a node in -/// shareable `m`'s context, and `m` itself, by the queries that depend on -/// `m`. Reads into an `I_r` and per-output-reference reach inside one query -/// are not modeled: the constraint is then applied to every reference of -/// the query (conservative, counted as inexact). -fn reader_constraint_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - let query_of_reference: Vec = output_references(&milp.physical.query_roots) - .into_iter() - .map(|(query, _)| query) - .collect(); - let constraints = reader_constraints_of_candidates(milp); - let query_count = milp.query_count(); - milp.builder.scope = Scope::Global; - let reach_count = |milp: &SelectionMilp<'_>, queries: &[usize], m: PhysicalCandidateId| { - let mut count = Expression::from(0.0); - let mut most = 0.0f64; - for &query in queries { - if let Some(&reach) = milp.reach.get(&(query, m)) { - count += reach; - most += 1.0; - } - } - (count, most) - }; - for (candidate, constraints) in constraints { - let shareable = inputs.kind[candidate.0] == RunKind::Shareable; - // Its uses: (context, p) per context holding it as an option it - // computes. - let uses: Vec<(usize, Variable)> = (0..milp.choice.len()) - .filter(|&context| milp.computes(context, candidate)) - .filter_map(|context| { - milp.choice[context] - .get(&candidate) - .map(|&variable| (context, variable)) - }) - .collect(); - for constraint in constraints { - match constraint { - ReaderConstraint::MinimumQueryReaders { - query_outputs, - minimum, - .. - } => { - let queries = bucket_queries(&query_of_reference, query_outputs); - let minimum = *minimum as f64; - if shareable { - if let Some(&selected) = milp.selected.get(&candidate) { - let (count, _) = reach_count(milp, &queries, candidate); - milp.builder.row( - "9 reader constraints", - constraint!(count >= minimum * selected), - ); - } - } - for &(context, used) in &uses { - let count = match inputs.contexts[context].kind { - ContextKind::Query(_) => Expression::from(1.0), - ContextKind::Shared(m) => reach_count(milp, &queries, m).0, - ContextKind::Ingestion(_) => { - milp.note_inexact( - "minimum readers of a candidate of an I_r (forbidden)", - ); - Expression::from(0.0) - } - }; - milp.builder - .row("9 reader constraints", constraint!(count >= minimum * used)); - } - } - ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - let queries = bucket_queries(&query_of_reference, query_outputs); - if shareable { - let Some(&selected) = milp.selected.get(&candidate) else { - continue; - }; - // Allowed: a read in a query's context (its readers - // there are reached by that query alone), or a read - // in the context of `m` that at most one query - // reaches. - let mut allowed = Expression::from(0.0); - for query in 0..query_count { - if let Some(&read) = milp.choice[query].get(&candidate) { - allowed += read; - } - } - for &reader in milp - .shared_readers - .get(&candidate) - .cloned() - .unwrap_or_default() - .iter() - { - let context = inputs.context_of_shareable[&reader]; - let read = milp.choice[context][&candidate]; - let (count, most) = reach_count(milp, &queries, reader); - let single = milp.builder.binary("g (single-query readers)"); - milp.builder - .row("9 reader constraints", constraint!(single <= read)); - let slack = (most - 1.0).max(0.0); - milp.builder.row( - "9 reader constraints", - constraint!(count <= 1.0 + slack * (1.0 - single)), - ); - allowed += single; - } - milp.builder - .row("9 reader constraints", constraint!(selected <= allowed)); - } - for &(context, used) in &uses { - match inputs.contexts[context].kind { - ContextKind::Query(_) => {} - ContextKind::Shared(m) => { - let (count, most) = reach_count(milp, &queries, m); - let slack = (most - 1.0).max(0.0); - milp.builder.row( - "9 reader constraints", - constraint!(count <= 1.0 + slack * (1.0 - used)), - ); - } - ContextKind::Ingestion(_) => { - milp.note_inexact( - "highest shared materialization in an I_r (forbidden)", - ); - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - } - } - } - ReaderConstraint::AllowedOutputs { outputs, .. } => { - for query in 0..query_count { - let references: Vec = (0..query_of_reference.len()) - .filter(|&reference| query_of_reference[reference] == query) - .collect(); - let disallowed = references - .iter() - .filter(|reference| !outputs.contains(reference)) - .count(); - if disallowed == 0 { - continue; - } - if disallowed < references.len() { - milp.note_inexact("AllowedOutputs over part of a query's outputs"); - } - forbid_reach(milp, query, candidate, &uses); - } - } - ReaderConstraint::IncompatibleOutputs { left, right, .. } => { - let (left, right) = (query_of_reference[*left], query_of_reference[*right]); - if left == right { - milp.note_inexact("IncompatibleOutputs within one query (forbidden)"); - forbid_reach(milp, left, candidate, &uses); - continue; - } - if shareable { - if let (Some(&l), Some(&r)) = ( - milp.reach.get(&(left, candidate)), - milp.reach.get(&(right, candidate)), - ) { - milp.builder - .row("9 reader constraints", constraint!(l + r <= 1.0)); - } - } - for &(context, used) in &uses { - if let ContextKind::Shared(m) = inputs.contexts[context].kind { - if let (Some(&l), Some(&r)) = - (milp.reach.get(&(left, m)), milp.reach.get(&(right, m))) - { - milp.builder - .row("9 reader constraints", constraint!(l + r + used <= 2.0)); - } - } - } - } - } - } - } -} - -/// Query `query` reaches no node of `candidate`. -fn forbid_reach( - milp: &mut SelectionMilp<'_>, - query: usize, - candidate: PhysicalCandidateId, - uses: &[(usize, Variable)], -) { - if let Some(&reach) = milp.reach.get(&(query, candidate)) { - milp.builder - .row("9 reader constraints", constraint!(reach <= 0.0)); - } else if let Some(&read) = milp.choice[query].get(&candidate) { - milp.builder - .row("9 reader constraints", constraint!(read <= 0.0)); - } - for &(context, used) in uses { - match milp.inputs.contexts[context].kind { - ContextKind::Query(q) if q == query => { - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - ContextKind::Query(_) => {} - ContextKind::Shared(m) => { - if let Some(&reach) = milp.reach.get(&(query, m)) { - milp.builder - .row("9 reader constraints", constraint!(reach + used <= 1.0)); - } - } - ContextKind::Ingestion(_) => { - milp.note_inexact("a reader constraint on a candidate of an I_r (forbidden)"); - milp.builder - .row("9 reader constraints", constraint!(used <= 0.0)); - } - } - } -} - -/// Query `query`'s latency bound: the smaller of its `max_latency_ms` and -/// its fixed repeat interval (`checks::latency_violation`). -pub(crate) fn latency_bound_ms(entry: &QueryWorkloadEntry) -> Option { - let interval = entry.recurrence.fixed_interval_ms().map(|ms| ms as f64); - match (entry.requirements.max_latency_ms, interval) { - (Some(bound), Some(interval)) => Some(bound.min(interval)), - (bound, interval) => bound.or(interval), - } -} +pub struct QueryId(pub usize); -/// The latency terms of context `context`'s computed runs (and of `m`'s -/// own run in its own context): `Σ d · indicator`, and the most it can be. -fn context_work(milp: &mut SelectionMilp<'_>, context: usize) -> (Expression, f64) { - let mut work = Expression::from(0.0); - let mut most = 0.0; - let mut runs: Vec = milp.choice[context] - .keys() - .copied() - .filter(|&c| milp.computes(context, c)) - .collect(); - if let ContextKind::Shared(own) = milp.inputs.contexts[context].kind { - runs.push(own); - } - // Per slot, only the slowest candidate can count (one per slot). - let mut most_per_slot: BTreeMap = BTreeMap::new(); - for candidate in runs { - let mut run_most = 0.0; - for term in milp.run_terms(context, candidate) { - if term.numbers.latency_ms > 0.0 { - let units = term.numbers.latency_ms / milp.ms_per_latency_unit; - work += units * term.indicator.clone(); - run_most = f64::max(run_most, units); - } - } - let slot = milp.slot(candidate); - let entry = most_per_slot.entry(slot).or_default(); - *entry = entry.max(run_most); - } - most += most_per_slot.values().sum::(); - (work, most) -} - -/// Row 7: each query's latency, `asap_cost::plan_cost`'s definition: its -/// request time, plus the work of every query-time run it reaches, each -/// once (its own context's, and each query-time shareable candidate's run -/// and context through `T[q, m]`), plus the longest compaction among the -/// ingestion-time states it reaches. -fn latency_rows(milp: &mut SelectionMilp<'_>) -> Result<(), SelectionMilpError> { - let inputs = milp.inputs; - let query_count = milp.query_count(); - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - // The latency unit: the most any query's latency can be (every - // candidate's work once, plus the request) is about 1,000 units. - let most_ms = request_ms - + inputs - .candidate_data - .iter() - .filter_map(|data| data.usage.as_ref().ok()) - .map(|usage| usage.own_latency_ms) - .sum::(); - if most_ms.is_finite() && most_ms > 0.0 { - milp.ms_per_latency_unit = most_ms / 1_000.0; - } - let unit = milp.ms_per_latency_unit; - let request = request_ms / unit; - // `L[m]` per query-time shareable candidate. - milp.builder.scope = Scope::SharedContexts; - let mut shared_latency: BTreeMap = BTreeMap::new(); - let shareables: Vec<(PhysicalCandidateId, usize)> = inputs - .context_of_shareable - .iter() - .map(|(&m, &context)| (m, context)) - .collect(); - for (m, context) in shareables { - if !milp.runs_at_query_time(context) { - continue; - } - let (work, most) = context_work(milp, context); - milp.builder.scope = Scope::SharedContexts; - if most <= 0.0 { - continue; - } - let latency = milp.builder.continuous("L (latency of m)", None); - milp.builder.row("7 latency", constraint!(latency >= work)); - shared_latency.insert(m, (latency, most)); - } - let compaction = cost_terms::compaction_ms(milp); - for query in 0..query_count { - let (mut work, mut most) = context_work(milp, query); - milp.builder.scope = Scope::Queries; - work += request; - most += request; - let mut compactions = Vec::new(); - let mut longest_compaction = 0.0f64; - for (&m, &(shared, shared_most)) in &shared_latency { - let Some(&reach) = milp.reach.get(&(query, m)) else { - continue; - }; - let through = milp.builder.continuous("T (latency of m in q)", None); - milp.builder.row( - "7 latency", - constraint!(through >= shared - shared_most * (1.0 - reach)), - ); - work += through; - most += shared_most; - } - for (&m, &ms) in &compaction { - if let Some(&reach) = milp.reach.get(&(query, m)) { - if ms > 0.0 { - compactions.push((ms / unit, reach)); - longest_compaction = longest_compaction.max(ms / unit); - } - } - } - if !compactions.is_empty() { - let longest = milp.builder.continuous("K (compaction)", None); - for (ms, reach) in compactions { - milp.builder - .row("7 latency", constraint!(longest >= ms * reach)); - } - work += longest; - most += longest_compaction; - } - let latency = milp.builder.continuous("L (latency of q)", None); - let bound = latency_bound_ms(&milp.entries[query]) - .map(|bound| bound / unit) - .filter(|&bound| bound < most); - // A binding bound's rows are divided by it, so HiGHS's absolute - // feasibility tolerance is relative to the bound. - let per_ms = bound.map_or(1.0, |bound| 1.0 / bound.max(1e-9)); - milp.builder - .row("7 latency", constraint!(per_ms * latency >= per_ms * work)); - if let Some(bound) = bound { - milp.builder - .row("7 latency", constraint!(per_ms * latency <= per_ms * bound)); - } - let bound = latency_bound_ms(&milp.entries[query]).map(|bound| bound / unit); - milp.latency.push(latency); - milp.latency_max - .push(bound.map_or(most, |bound| bound.min(most))); - } - milp.builder.scope = Scope::Global; - Ok(()) -} - -/// Row 8: accuracy (A1, revised 2026-10-31), per run `q` depends on: a run -/// unusable for `q` is never depended on, two whose pair is excluded are -/// not depended on together, and the shares depended on are at most each -/// such run's `parts`. -fn accuracy_rows(milp: &mut SelectionMilp<'_>) { - let inputs = milp.inputs; - milp.builder.scope = Scope::Global; - let approximate_runs: Vec<(usize, PhysicalCandidateId)> = milp - .runs() - .into_iter() - .filter(|&(_, candidate)| inputs.candidate_data[candidate.0].approximate) - .collect(); - for query in 0..milp.query_count() { - let exact_target = milp.entries[query].requirements.accuracy.is_exact(); - // (candidate, dep, parts) per run `q` may depend on. - let mut counted: Vec<(PhysicalCandidateId, Expression, usize)> = Vec::new(); - // Where each counted run gives its value: (context, slot), `None` - // for a shareable candidate's own run. One value per place. - let mut places: BTreeSet<(usize, Option)> = BTreeSet::new(); - for &(context, candidate) in &approximate_runs { - let Some(weight) = inputs.accuracy_weights[query].get(&candidate).copied() else { - continue; - }; - let dep: Option = match inputs.contexts[context].kind { - ContextKind::Query(q) if q == query => { - milp.choice[context].get(&candidate).map(|&p| p.into()) - } - ContextKind::Query(_) => None, - ContextKind::Shared(own) if own == candidate => { - milp.reach.get(&(query, candidate)).map(|&a| a.into()) - } - ContextKind::Shared(own) => match milp.reach.get(&(query, own)) { - Some(&above) => { - let p = milp.choice[context][&candidate]; - Some( - milp.builder - .and_at_least( - "y (depends on a run in m)", - &[above.into(), p.into()], - ) - .into(), - ) - } - None => None, - }, - ContextKind::Ingestion(_) => { - milp.note_inexact("an approximate candidate of an I_r (forbidden)"); - let p = milp.choice[context][&candidate]; - milp.builder.row("8 accuracy", constraint!(p <= 0.0)); - None - } - }; - let Some(dep) = dep else { - continue; - }; - let usable = match weight { - _ if exact_target => false, - AccuracyWeight::One => true, - AccuracyWeight::Unusable(_) => false, - }; - if usable { - places.insert(match inputs.contexts[context].kind { - ContextKind::Shared(own) if own == candidate => (context, None), - _ => (context, Some(inputs.slot[candidate.0])), - }); - counted.push(( - candidate, - dep, - inputs.candidate_data[candidate.0].accuracy_budget_parts, - )); - } else { - milp.builder.row("8 accuracy", constraint!(dep <= 0.0)); - } - } - let mut pairs: Vec<(Expression, Expression)> = Vec::new(); - for &(first, second) in &inputs.accuracy_excluded_pairs[query] { - for (_, one, _) in counted.iter().filter(|(c, _, _)| *c == first) { - for (_, other, _) in counted.iter().filter(|(c, _, _)| *c == second) { - pairs.push((one.clone(), other.clone())); - } - } - } - for (one, other) in pairs { - milp.builder - .row("8 accuracy (pair)", constraint!(one + other <= 1.0)); - } - // The most values `q` can depend on together. - let most = if tests_loose_big_m() { - counted.len() as f64 - } else { - places.len() as f64 - }; - if counted.iter().all(|&(_, _, parts)| parts as f64 >= most) { - continue; - } - let shares = milp.builder.continuous("N (approximate values)", None); - let sum: Expression = counted.iter().map(|(_, dep, _)| dep.clone()).sum(); - milp.builder.row("8 accuracy", constraint!(shares == sum)); - let mut by_parts: BTreeMap> = BTreeMap::new(); - for (_, dep, parts) in &counted { - if (*parts as f64) < most { - by_parts.entry(*parts).or_default().push(dep.clone()); - } - } - for (parts, deps) in by_parts { - let reads = milp - .builder - .continuous("A (depends at a parts value)", Some(1.0)); - for dep in deps { - milp.builder.row("8 accuracy", constraint!(reads >= dep)); - } - let slack = most - parts as f64; - milp.builder.row( - "8 accuracy", - constraint!(shares <= parts as f64 + slack * (1.0 - reads)), - ); - } - } -} - -/// One independent part of the model (#1095 §5, 3a): a connected component -/// of the graph joining the variables of each row, or all components -/// without a query together. Parts share no variable and no row, and the -/// objective is a sum over variables, so solving each alone and joining -/// their solutions solves the whole model. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct IndependentPart { - /// The queries whose output references it covers, ascending. - pub(crate) queries: Vec, - /// Its columns (variables in creation order), ascending. - pub(crate) columns: Vec, - /// Its rows, ascending. - pub(crate) rows: Vec, -} - -/// The independent parts of `milp`, in order of their first query (the -/// part without a query last). Queries fall into one part when they may -/// share a shareable candidate, a run of an `I_r`, a source's raw retention or -/// anything else a row joins. -pub(crate) fn independent_parts(milp: &SelectionMilp<'_>) -> Vec { - let column_of: HashMap = milp - .builder - .variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let count = column_of.len(); - // Union-find over the columns, joined along each row. - let mut parent: Vec = (0..count).collect(); - fn root(parent: &mut [usize], mut node: usize) -> usize { - while parent[node] != node { - parent[node] = parent[parent[node]]; - node = parent[node]; - } - node - } - for row in &milp.builder.rows { - let mut first: Option = None; - for (variable, _) in row.expression().linear_coefficients() { - let column = root(&mut parent, column_of[&variable]); - match first { - None => first = Some(column), - Some(known) if known != column => parent[column] = known, - Some(_) => {} - } - } - } - let output_refs = output_references(&milp.physical.query_roots); - let mut query_of_component: BTreeMap = BTreeMap::new(); - for (reference, &(query, _)) in output_refs.iter().enumerate() { - for out in &milp.inputs.output_candidates[reference] { - if let Some(variable) = milp.choice[query].get(out) { - let component = root(&mut parent, column_of[variable]); - query_of_component.entry(component).or_insert(query); - } - } - } - // The part of each component: its first query's, or the last. - let part_of = |component: usize| query_of_component.get(&component).copied(); - let mut parts: BTreeMap, IndependentPart> = BTreeMap::new(); - let mut part_of_column = Vec::with_capacity(count); - for column in 0..count { - let key = part_of(root(&mut parent, column)); - part_of_column.push(key); - parts - .entry(key) - .or_insert_with(|| IndependentPart { - queries: Vec::new(), - columns: Vec::new(), - rows: Vec::new(), - }) - .columns - .push(column); - } - for (reference, &(query, _)) in output_refs.iter().enumerate() { - let Some(variable) = milp.inputs.output_candidates[reference] - .iter() - .find_map(|out| milp.choice[query].get(out)) - else { - continue; - }; - let key = part_of_column[column_of[variable]]; - let queries = &mut parts.get_mut(&key).expect("its column's part").queries; - if !queries.contains(&query) { - queries.push(query); - } - } - for (index, row) in milp.builder.rows.iter().enumerate() { - // A row without a variable is a constant check: kept in the part - // without a query when there is one (`None` sorts first), else in - // the first query's part. - let key = row - .expression() - .linear_coefficients() - .next() - .map(|(variable, _)| part_of_column[column_of[&variable]]) - .unwrap_or_else(|| *parts.keys().next().expect("a part")); - parts.get_mut(&key).expect("a part").rows.push(index); - } - // `None` sorts first; the part without a query goes last. - let mut parts: Vec<(Option, IndependentPart)> = parts.into_iter().collect(); - parts.sort_by_key(|(key, _)| key.unwrap_or(usize::MAX)); - parts - .into_iter() - .map(|(_, mut part)| { - part.queries.sort_unstable(); - part - }) - .collect() -} - -/// How one part's solve went, for the report. -#[derive(Debug, Clone, PartialEq, serde::Serialize)] -pub(crate) struct PartSolve { - pub(crate) queries: usize, - pub(crate) variables: usize, - pub(crate) binaries: usize, - pub(crate) rows: usize, - pub(crate) solve_ms: u64, - /// HiGHS's dual bound on the part's objective (cost per second). - pub(crate) dual_bound_per_second: f64, -} - -/// One part's model ready for HiGHS: its variables, rows and objective -/// terms, renamed to its own columns when the model has several parts. -struct PartModel { - variables: ProblemVariables, - rows: Vec, - objective_terms: Vec<(Variable, f64)>, - /// Per column of the part, the column in the whole model. - columns: Vec, - queries: usize, -} - -/// A part solved to proven optimality: its columns' values, and the HiGHS -/// objective and dual bound (cost per second). -struct SolvedPart { - columns: Vec, - objective_per_second: f64, - dual_bound_per_second: f64, - report: PartSolve, -} - -/// Solves `milp` with HiGHS, one solve per independent part -/// ([`independent_parts`]) when `split_into_parts`, each part from the warm -/// start and run to proven optimality (no time limit). -pub(crate) fn solve_selection_milp( - milp: SelectionMilp<'_>, - build_ms: u64, -) -> Result { - let size = milp.size(); - let highs_threads = milp.highs_threads; - let warm_started = Instant::now(); - let completion = super::completion::Completion::new(&milp); - let start = warm_start::warm_start(&milp, &completion); - let warm_start_ms = warm_started.elapsed().as_millis() as u64; - let warm_start_cost_per_second = start.as_ref().map(|start| start.cost_per_second); - #[cfg(feature = "lp-diagnosis")] - let diagnosis = { - let column_of: HashMap = milp - .builder - .variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let start_columns = start.as_ref().map(|start| { - let mut columns = vec![0.0; column_of.len()]; - for &(variable, value) in &start.values { - columns[column_of[&variable]] = value; - } - columns - }); - super::lp_diagnosis::Diagnosis::run(&milp, start_columns.as_deref()) - .map(|diagnosis| (diagnosis, start_columns)) - }; - let parts = if milp.split_into_parts { - independent_parts(&milp) - } else { - Vec::new() - }; - let SelectionMilp { - builder, - choice, - selected, - slot_links, - request_cost_per_second, - objective_terms, - .. - } = milp; - let ModelBuilder { - variables, rows, .. - } = builder; - let column_of: HashMap = variables - .iter_variables_with_def() - .enumerate() - .map(|(column, (variable, _))| (variable, column)) - .collect(); - let column_count = column_of.len(); - let start_columns: Option> = start.map(|start| { - let mut columns = vec![0.0; column_count]; - for (variable, value) in start.values { - columns[column_of[&variable]] = value; - } - columns - }); - let models: Vec = if parts.len() <= 1 { - vec![PartModel { - variables, - rows, - objective_terms, - columns: (0..column_count).collect(), - queries: parts.first().map_or(0, |part| part.queries.len()), - }] - } else { - let definitions: Vec<_> = variables - .iter_variables_with_def() - .map(|(_, definition)| definition.clone()) - .collect(); - let mut part_of_column = vec![0; column_count]; - let mut renamed: Vec> = vec![None; column_count]; - let mut models: Vec = parts - .iter() - .enumerate() - .map(|(index, part)| { - let mut variables = ProblemVariables::new(); - for &column in &part.columns { - part_of_column[column] = index; - renamed[column] = Some(variables.add(definitions[column].clone())); - } - PartModel { - variables, - rows: Vec::with_capacity(part.rows.len()), - objective_terms: Vec::new(), - columns: part.columns.clone(), - queries: part.queries.len(), - } - }) - .collect(); - let rename = |variable: Variable| renamed[column_of[&variable]].expect("in a part"); - for (index, part) in parts.iter().enumerate() { - for &row in &part.rows { - let row = &rows[row]; - let mut expression = Expression::from(row.expression().constant()); - for (variable, coefficient) in row.expression().linear_coefficients() { - expression.add_mul(coefficient, rename(variable)); - } - models[index].rows.push(if row.is_equality() { - constraint!(expression == 0.0) - } else { - constraint!(expression <= 0.0) - }); - } - } - for (variable, coefficient) in objective_terms { - let column = column_of[&variable]; - models[part_of_column[column]] - .objective_terms - .push((rename(variable), coefficient)); - } - models - }; - let mut columns = vec![0.0; column_count]; - let mut solved_parts: Vec = Vec::with_capacity(models.len()); - let started = Instant::now(); - for model in models { - let start = start_columns.as_ref().map(|start| { - model - .columns - .iter() - .map(|&column| start[column]) - .collect::>() - }); - let part_columns = model.columns.clone(); - let solved = solve_part(model, start, highs_threads)?; - for (&column, &value) in part_columns.iter().zip(&solved.columns) { - columns[column] = value; - } - solved_parts.push(solved); - } - let solve_ms = started.elapsed().as_millis() as u64; - let highs_objective: f64 = solved_parts - .iter() - .map(|part| part.objective_per_second) - .sum(); - let dual_bound: f64 = solved_parts - .iter() - .map(|part| part.dual_bound_per_second) - .sum(); - // The objective of the integral plan: the incumbent's binaries, - // rounded, with every continuous variable at its cheapest value (the - // solver's own value carries its tolerances, relative to the scaled - // objective). HiGHS's objective when the completion does not apply. - let objective_value = match completion.complete_columns(&columns) { - Ok(completed) => completed.cost_per_second, - Err(reason) => { - warm_start::debug(|| format!("completion of the incumbent failed: {reason}")); - highs_objective - } - }; - warm_start::debug(|| { - format!("objective: HiGHS {highs_objective:e}, completed {objective_value:e}") - }); - #[cfg(feature = "lp-diagnosis")] - if let Some((diagnosis, start_columns)) = diagnosis { - let incumbent = completion.complete_columns(&columns).ok(); - diagnosis.write( - start_columns.as_deref(), - incumbent - .as_ref() - .map(|completed| completed.columns.as_slice()), - objective_value, - dual_bound, - ); - } - let on = |variable: Variable| columns[column_of[&variable]] > 0.5; - Ok(SelectionMilpSolution { - cost_per_second: objective_value, - request_cost_per_second, - cost_lower_bound_per_second: dual_bound, - chosen: choice - .iter() - .map(|options| { - options - .iter() - .filter(|(_, &variable)| on(variable)) - .map(|(&candidate, _)| candidate) - .collect() - }) - .collect(), - selected: selected - .iter() - .filter(|(_, &variable)| on(variable)) - .map(|(&candidate, _)| candidate) - .collect(), - slot_reads: slot_links - .iter() - .filter_map(|(&key, links)| { - links - .iter() - .find(|(_, link)| on(*link)) - .map(|&(slot, _)| (key, slot)) - }) - .collect(), - size, - parts: solved_parts.into_iter().map(|part| part.report).collect(), - build_ms, - warm_start_ms, - warm_start_cost_per_second, - solve_ms, - }) -} - -/// Solves one part once with HiGHS to proven optimality, from `start` (its -/// columns' values) when given, on `highs_threads` threads. -fn solve_part( - model: PartModel, - start: Option>, - highs_threads: i32, -) -> Result { - let PartModel { - variables, - rows, - objective_terms, - columns: part_columns, - queries, - } = model; - let start_values: Option> = start.as_ref().map(|start| { - variables - .iter_variables_with_def() - .map(|(variable, _)| variable) - .zip(start.iter().copied()) - .collect() - }); - let scale = objective_scale(&objective_terms, start_values.as_deref()); - let objective: Expression = objective_terms - .iter() - .map(|&(variable, coefficient)| coefficient * scale * variable) - .sum(); - let binaries = variables - .iter_variables_with_def() - .filter(|(_, definition)| definition.is_integer()) - .count(); - let row_count = rows.len(); - let problem = variables.minimise(objective).using(highs).with_all(rows); - let mut model = problem - .try_into_inner() - .map_err(|error| SelectionMilpError::Solver(error.to_string()))?; - model.set_option("mip_rel_gap", 0.0); - model.set_option("mip_abs_gap", 0.0); - model.set_option("output_flag", false); - // HiGHS's MIP search is mostly serial; a thread per core costs - // gigabytes of address space on a large host (#1036 under 8 GB). - model.set_option("threads", highs_threads); - if let Some(start) = &start { - model - .try_set_solution(Some(start), None, None, None) - .map_err(|status| SelectionMilpError::Solver(format!("warm start: {status:?}")))?; - } - let started = Instant::now(); - let solved = model - .try_solve() - .map_err(|status| SelectionMilpError::Solver(format!("{status:?}")))?; - let solve_ms = started.elapsed().as_millis() as u64; - use highs::{HighsModelStatus, HighsSolutionStatus}; - // No limit is set, so HiGHS stops only at proven optimality or - // infeasibility; any other status is a solver failure. - match solved.status() { - HighsModelStatus::Optimal => {} - HighsModelStatus::Infeasible | HighsModelStatus::UnboundedOrInfeasible => { - return Err(SelectionMilpError::Infeasible) - } - other => return Err(SelectionMilpError::Solver(format!("{other:?}"))), - } - if solved.primal_solution_status() != HighsSolutionStatus::Feasible { - return Err(SelectionMilpError::Solver( - "HiGHS reported an optimal status without a feasible solution".into(), - )); - } - // HiGHS leaves `mip_dual_bound` at 0 for a part without binaries (an - // LP): its optimal objective is its bound. - let dual_bound = if binaries > 0 { - solved - .double_info_value(c"mip_dual_bound") - .unwrap_or(f64::NAN) - } else { - solved.objective_value() - } / scale; - Ok(SolvedPart { - columns: solved.get_solution().columns().to_vec(), - objective_per_second: solved.objective_value() / scale, - dual_bound_per_second: dual_bound, - report: PartSolve { - queries, - variables: part_columns.len(), - binaries, - rows: row_count, - solve_ms, - dual_bound_per_second: dual_bound, - }, - }) -} - -/// The factor the objective is multiplied by for HiGHS: the warm start's -/// cost becomes about 1,000, its largest coefficient at most 10⁷; without -/// a warm start cost, the largest coefficient becomes 1,000. -pub(crate) fn objective_scale(terms: &[(Variable, f64)], start: Option<&[(Variable, f64)]>) -> f64 { - let largest = terms - .iter() - .map(|(_, coefficient)| coefficient.abs()) - .fold(0.0, f64::max); - if !(largest > 0.0 && largest.is_finite()) { - return 1.0; - } - let started: HashMap = start.into_iter().flatten().copied().collect(); - let start_cost: f64 = terms - .iter() - .map(|(variable, coefficient)| coefficient * started.get(variable).copied().unwrap_or(0.0)) - .sum(); - if start_cost > 0.0 && start_cost.is_finite() { - (1_000.0 / start_cost).min(1e7 / largest) - } else { - 1_000.0 / largest - } -} - -/// Whether this thread builds the big-M bounds as before #1095 step 1 -/// (tests only: to compare the tightened bounds with the looser ones). -#[cfg(test)] -pub(crate) fn tests_loose_big_m() -> bool { - tests::LOOSE_BIG_M.with(|loose| loose.get()) -} +/// A refresh window of the workload's ingestion pipelines, by index. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RefreshWindowId(pub usize); -#[cfg(not(test))] -pub(crate) fn tests_loose_big_m() -> bool { - false +/// Index of a [`Context`] among the workload's contexts. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ContextId(pub usize); + +/// $\kappa$: a class of executions with a common owner and schedule, where +/// an unmaterialized candidate runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Context { + /// $\kappa_q$: an evaluation of a query. + QueryEvaluation(QueryId), + /// $I_r$: the ingestion pipeline of a refresh window. + IngestionPipeline(RefreshWindowId), + /// $\kappa_m$: inside a materialized candidate's own computation. + Materialized(PhysicalCandidateId), +} + +impl Context { + /// $T_\kappa$: the context's periodic evaluation schedule when it has + /// one; `None` (a non-periodic query) is not an implicit compatibility + /// pass. + pub fn evaluation_schedule(&self) -> Option { + todo!("#1207 PR 7: each context's schedule from its query's 𝒯_q") + } +} + +/// The baseline variables (section 3). +struct SelectionVariables { + /// $x_v$, by [`PhysicalCandidateId`]. + x: Vec, + /// $y_{u,c,i}$, on admissible edges only. + y: HashMap, + /// $z_{q,v}$, for $v \in G_q$. + z: HashMap<(QueryId, PhysicalCandidateId), Variable>, + /// $t_v$, by [`PhysicalCandidateId`]. + t: Vec, +} + +/// The execution-context variables (section 5), which replace $x$ and the +/// unindexed $y$ of the baseline. +struct ExecutionContextVariables { + /// $x^{\mathrm{mat}}_v$, for a materialized candidate. + x_materialized: HashMap, + /// $x_{\kappa,v}$, for an unmaterialized candidate. + x_in_context: HashMap<(ContextId, PhysicalCandidateId), Variable>, + /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. + y_in_context: HashMap<(ContextId, PhysicalEdgeId), Variable>, + /// $h_v$, for a materialized producer. + retention_amount: HashMap, +} + +/// The accuracy variables of constraint (K) (section 4), all continuous. +struct AccuracyVariables { + /// $E_c$, by [`PhysicalCandidateId`]. + error_bound: Vec, + /// $F_{c,i}$: the chosen producer's error bound on input $i$ of $c$. + error_bound_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, + /// $D_c$, by [`PhysicalCandidateId`]. + failure_probability: Vec, + /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. + failure_probability_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, +} + +/// (A) $\sum_{u \in P_{c,i}} y_{u,c,i} = x_c$. +fn input_satisfaction_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (A)") +} + +/// (B) $y_{u,c,i} \le x_u$. +fn producer_selection_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (B)") +} + +/// (C) $\sum_{v \in G_q} z_{q,v} = 1$; (D) $z_{q,v} \le x_v$. +fn query_coverage_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraints (C) and (D)") +} + +/// (E) $t_u + 1 \le t_c + |\mathcal C|(1 - y_{u,c,i})$. +fn acyclicity_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (E)") +} + +/// (F) $x_v \le \sum_{(v,c,i)} y_{v,c,i} + \sum_q z_{q,v}$. +fn utilization_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &SelectionVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 6: constraint (F)") +} + +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$. +/// +/// Assumptions (decided 2026-11-06): an edge's need is the time horizon +/// the queries it serves read, a constant per edge (a 5m range over 10 s +/// windows needs 30 windows), so (R) is linear; query latency is ignored +/// until the latency design: a result counts as read when its evaluation +/// starts. +fn retention_constraints( + physical: &PhysicalASAPMemoCandidates, + variables: &ExecutionContextVariables, +) -> Vec { + let _ = (physical, variables); + todo!("#1207 PR 7: constraint (R)") +} + +/// (K): $E_c$ and $D_c$ per candidate, big-M on selected edges, and the +/// root checks $E_v \le \varepsilon_q + M_q(1 - z_{q,v})$, +/// $D_v \le \delta_q + (1 - z_{q,v})$. +fn accuracy_constraints( + physical: &PhysicalASAPMemoCandidates, + selection: &SelectionVariables, + accuracy: &AccuracyVariables, +) -> Vec { + let _ = (physical, selection, accuracy); + todo!("#1207 PR 8: constraint (K)") } - -#[cfg(test)] -pub(crate) mod tests; diff --git a/crates/plan-selection/src/milp/model/tests.rs b/crates/plan-selection/src/milp/model/tests.rs deleted file mode 100644 index a8ec9f4a0..000000000 --- a/crates/plan-selection/src/milp/model/tests.rs +++ /dev/null @@ -1,1229 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; - -use super::*; -use crate::groups::{prepare_milp_inputs, AccuracyWeight, IngestionPipeline}; -use crate::milp::decode::{decode, ChosenCandidates}; -use asap_cost::{node_map, plan_cost, PlanCost, ResourceUnitCost}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::{AssignmentRealizer, DagOfSelectedCandidates, PhysicalCandidate}; -use asap_types::workload::{DataWorkload, DurationMs, RangeEvaluation}; - -// --------------------------------------------------------------------------- -// Through Stages 1 and 2 -// --------------------------------------------------------------------------- - -/// A workload through the real Stages 1 and 2, with the roots the -/// realizer rebuilds plans from. -pub(crate) struct Fixture { - pub(crate) physical: PhysicalASAPMemoCandidates, - pub(crate) roots: Vec, - pub(crate) entries: Vec, - pub(crate) data: DataWorkload, - pub(crate) query_time_results_retained: bool, -} - -/// One query of a fixture: PromQL over metric `m`, its repeat interval in -/// ms, its relative error target (`None`: exact), its latency bound and -/// its range evaluation. -#[derive(Clone, Copy)] -pub(crate) struct Query<'a> { - promql: &'a str, - interval_ms: u32, - max_error: Option, - max_latency_ms: Option, - range_evaluation: Option, -} - -pub(crate) fn query(promql: &str, interval_ms: u32) -> Query<'_> { - Query { - promql, - interval_ms, - max_error: None, - max_latency_ms: None, - range_evaluation: None, - } -} - -pub(crate) fn approximate(promql: &str, interval_ms: u32, max_error: f64) -> Query<'_> { - Query { - max_error: Some(max_error), - ..query(promql, interval_ms) - } -} - -/// `queries` through Stage 1 and Stage 2 as the facade runs them, over the -/// complete pane-width domain, metric `m` with 100 series sampled every -/// second. -pub(crate) fn through_stages(queries: &[Query<'_>], raw_data_retained: bool) -> Fixture { - through_stages_with(queries, raw_data_retained, false) -} - -/// [`through_stages`] over metric `m` declared non-negative (so relative -/// errors compose through quotients, and Pass 1 keeps DDSketch). -pub(crate) fn through_stages_non_negative(queries: &[Query<'_>]) -> Fixture { - through_stages_over(queries, &["m"], true, false, true) -} - -/// [`through_stages`], with rows arriving (100 per second) when -/// `ingesting`, so Stage 2 offers ingestion-time candidates. -pub(crate) fn through_stages_with( - queries: &[Query<'_>], - raw_data_retained: bool, - ingesting: bool, -) -> Fixture { - through_stages_over(queries, &["m"], raw_data_retained, ingesting, false) -} - -/// The fixture builder behind [`through_stages`], over the metrics -/// `metrics`, each with 100 series sampled every second. -fn through_stages_over( - queries: &[Query<'_>], - metrics: &[&str], - raw_data_retained: bool, - ingesting: bool, - values_non_negative: bool, -) -> Fixture { - use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; - use asap_logical_asap_optimization::generate_per_computation_logical_memo_candidates; - use asap_logical_asap_optimization::test_support::{entry_with, lower_promql}; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::workload::{ - Predictability, QueryRecurrence, RepeatedDemand, RepetitionInterval, - }; - let target = |query: &Query<'_>| match query.max_error { - Some(error) => AccuracyTarget::max_error(error).failure_probability(0.01), - None => AccuracyTarget::exact(), - }; - let roots: Vec = queries - .iter() - .map(|query| QueryRoot::Operator(lower_promql(query.promql, target(query)))) - .collect(); - let entries: Vec = queries - .iter() - .map(|query| { - let mut entry = entry_with( - target(query), - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( - query.interval_ms, - ))), - ); - entry.predictability = Predictability::Predictable { known_at: None }; - entry.requirements.max_latency_ms = query.max_latency_ms; - entry.range_evaluation = query.range_evaluation; - entry - }) - .collect(); - let mut data = DataWorkload::sampled_every( - metrics.iter().map(|metric| metric.to_string()), - DurationMs(1_000), - ); - for metric in metrics { - let source = data.sources.get_mut(*metric).expect("declared"); - source.series_count = Some(100); - source.values_non_negative = values_non_negative; - } - if ingesting { - data.arrival = asap_types::workload::DataArrival::Ingesting { - rows_per_second: 100.0, - }; - } - let targets: Vec = queries.iter().map(target).collect(); - let pass1 = generate_per_computation_logical_memo_candidates( - asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - &targets, - &asap_accuracy::BuiltinAccuracyModel, - &asap_accuracy::NoAccuracyEvidence, - &data, - None, - ) - .unwrap(); - let logical = generate_logical_asap_memo_candidates(pass1, &entries).unwrap(); - let physical = asap_physical_asap_optimization::generate_physical_asap_memo_candidates( - &logical, - &entries, - &data, - false, - raw_data_retained, - ) - .unwrap(); - Fixture { - physical, - roots, - entries, - data, - query_time_results_retained: false, - } -} - -/// What one checked solve gives. -pub(crate) struct Checked { - pub(crate) solution: SelectionMilpSolution, - pub(crate) plan: PlanCost, - pub(crate) dag: DagOfSelectedCandidates, -} - -/// `fixture`'s MILP under `cost_inputs`, with `restrict` adding rows first: -/// solved, decoded, realized by Stage 2 and costed by -/// `asap_cost::plan_cost`. The objective plus the request fees must equal -/// `plan_cost`'s cost. -pub(crate) fn solve_and_check( - fixture: &Fixture, - cost_inputs: &CostInputs, - restrict: impl FnOnce(&mut SelectionMilp<'_>), -) -> Result { - solve_and_check_by(fixture, cost_inputs, restrict, true) -} - -/// [`solve_and_check`] with the independent parts solved apart when -/// `split_into_parts`. -fn solve_and_check_by( - fixture: &Fixture, - cost_inputs: &CostInputs, - restrict: impl FnOnce(&mut SelectionMilp<'_>), - split_into_parts: bool, -) -> Result { - let inputs = prepare_milp_inputs( - &fixture.physical, - &fixture.entries, - &fixture.data, - cost_inputs, - ); - let mut milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - cost_inputs, - SelectionMilpOptions { - split_into_parts, - ..SelectionMilpOptions::default() - }, - )?; - assert!(milp.inexact.is_empty(), "inexact terms: {:?}", milp.inexact); - restrict(&mut milp); - let solution = solve_selection_milp(milp, 0)?; - let dag = decode( - &fixture.physical, - &inputs, - &ChosenCandidates { - chosen: solution.chosen.clone(), - selected: solution.selected.clone(), - slot_reads: solution.slot_reads.clone(), - }, - ) - .expect("the solution decodes"); - dag.validate(&fixture.physical) - .expect("the decoded selection is valid"); - let realizer = asap_physical_asap_optimization::Stage2AssignmentRealizer { - original_roots: &fixture.roots, - entries: &fixture.entries, - data: &fixture.data, - query_time_results_retained: fixture.query_time_results_retained, - }; - let realized = realizer - .realize(&fixture.physical, &dag) - .expect("the realizer accepts the decoded plan"); - let plan_dag = &realized.plan.dag; - let recurrence: Vec<_> = fixture - .entries - .iter() - .map(|entry| entry.recurrence.clone()) - .collect(); - let ranges: Vec<_> = fixture - .entries - .iter() - .map(|entry| entry.range_evaluation) - .collect(); - let (plan, _) = plan_cost( - plan_dag, - &node_map(plan_dag), - &asap_cost::kept_tumbling_window_chains(plan_dag), - &recurrence, - &ranges, - &fixture.data, - cost_inputs, - cost_inputs.frozen_calibration_factor.unwrap_or(1.0), - ) - .expect("plan_cost costs the plan"); - let expected = plan.cost_per_second.expect("cost model 1") - solution.request_cost_per_second; - let tolerance = 1e-6 * expected.abs().max(1e-12); - assert!( - (solution.cost_per_second - expected).abs() <= tolerance, - "MILP objective {:e} != plan_cost {:e} (without {:e} of request fees); per node: {:#?}", - solution.cost_per_second, - expected, - solution.request_cost_per_second, - plan.per_node - ); - Ok(Checked { - solution, - plan, - dag, - }) -} - -pub(crate) fn cost_inputs() -> CostInputs { - CostInputs { - frozen_calibration_factor: Some(1.0), - ..CostInputs::default() - } -} - -/// The `aws-fargate-us-east-1` cost model of the memo examples. -fn aws_fargate_cost_inputs() -> CostInputs { - let defaults = cost_inputs(); - CostInputs { - cost_model: asap_cost::CostModel::UsageOverTime(ResourceUnitCost::aws_fargate_us_east_1( - defaults.resource_speed, - )), - ..defaults - } -} - -/// The candidates of `milp` that satisfy `wanted`. -fn candidates_where( - milp: &SelectionMilp<'_>, - wanted: impl Fn(&PhysicalCandidate, &OperatorNode) -> bool, -) -> Vec { - (0..milp.physical.candidates.len()) - .map(PhysicalCandidateId) - .filter(|&candidate| { - wanted( - &milp.physical.candidates[candidate.0], - milp.physical.physical_node(candidate), - ) - }) - .collect() -} - -/// Adds `Σ x[m] ≥ 1` over the shareable `candidates`: one is selected. -fn require_one_of(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { - let sum: Expression = candidates - .iter() - .filter_map(|candidate| milp.selected.get(candidate).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 1.0)); -} - -/// Adds `x[m] = 0` and `p[ctx, c] = 0` everywhere for `candidates`. -fn forbid(milp: &mut SelectionMilp<'_>, candidates: &[PhysicalCandidateId]) { - for candidate in candidates { - if let Some(&selected) = milp.selected.get(candidate) { - milp.builder.row("test", constraint!(selected <= 0.0)); - } - for context in 0..milp.choice.len() { - if let Some(&choice) = milp.choice[context].get(candidate) { - milp.builder.row("test", constraint!(choice <= 0.0)); - } - } - } -} - -/// The candidates that are pane chains or run at ingestion time. -fn pane_chains_and_ingestion(milp: &SelectionMilp<'_>) -> Vec { - candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::IngestionTime - || matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) -} - -/// How many nodes of `dag` run `candidate`. -fn runs_of(dag: &DagOfSelectedCandidates, candidate: PhysicalCandidateId) -> usize { - dag.candidate_of_node - .iter() - .filter(|&&c| c == candidate) - .count() -} - -/// `sum(m)` repeating every minute and `sum(m) + max(m)` every 30 s: the -/// second query does more work, so its evaluations take longer. -pub(crate) fn shared_sum() -> Fixture { - through_stages( - &[query("sum(m)", 60_000), query("sum(m) + max(m)", 30_000)], - true, - ) -} - -/// `sum_over_time(m[5m])` and `sum_over_time(m[1h])` over raw data. -pub(crate) fn two_windows() -> Fixture { - through_stages( - &[ - query("sum_over_time(m[5m])", 60_000), - query("sum_over_time(m[1h])", 60_000), - ], - true, - ) -} - -// --------------------------------------------------------------------------- -// Cost terms equal `plan_cost` (§3) -// --------------------------------------------------------------------------- - -// The optimum's objective equals `plan_cost` of the plan it decodes to. -#[test] -fn the_optimum_costs_what_plan_cost_says() { - for costs in [cost_inputs(), aws_fargate_cost_inputs()] { - solve_and_check(&shared_sum(), &costs, |_| {}).unwrap(); - } -} - -// Held memory and schedule groups: a `MatAtQuery(Memory)` result both -// queries read (every 60 s and every 30 s) is held until the slower reader -// finishes, once per execution at the schedule groups' rates. -#[test] -fn held_memory_of_a_result_readers_of_different_latency_share() { - let fixture = shared_sum(); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let shared: Vec = candidates_where(milp, |candidate, _| { - candidate.materialization == Materialization::MatAtQuery(Medium::Memory) - }) - .into_iter() - .filter(|&candidate| milp.queries_reaching(candidate).len() == 2) - .collect(); - assert!(!shared.is_empty()); - require_one_of(milp, &shared); - }) - .unwrap(); - let latencies: Vec = checked - .plan - .usage - .latency_per_query - .iter() - .map(|latency| latency.total_ms()) - .collect(); - assert!(latencies[0] != latencies[1], "{latencies:?}"); - assert!(checked - .plan - .per_node - .values() - .any(|node| node.detail.contains("holds"))); -} - -// Raw retention: with one scan per window, the source keeps the longer -// window's raw data once. -#[test] -fn raw_retention_keeps_the_longest_window() { - let checked = solve_and_check(&two_windows(), &cost_inputs(), |milp| { - let excluded = pane_chains_and_ingestion(milp); - forbid(milp, &excluded); - }) - .unwrap(); - assert!(checked - .plan - .per_node - .values() - .any(|node| node.detail.contains("of raw data on disk"))); -} - -// A scan read through two offsets in one query evaluation: the scan run -// reads the union of the two windows, and the MILP charges exactly that -// union (`δ` over the window kinds read), as `plan_cost` does. -#[test] -fn a_scan_read_through_two_offsets_is_costed_at_their_union() { - let fixture = through_stages( - &[query( - "sum_over_time(m[5m]) + sum_over_time(m[5m] offset 1h)", - 60_000, - )], - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let excluded = pane_chains_and_ingestion(milp); - forbid(milp, &excluded); - }) - .unwrap(); - assert!( - checked - .solution - .size - .families - .contains_key("δ (scan window sets)"), - "{:?}", - checked.solution.size.families.keys().collect::>() - ); -} - -// A kept pane chain whose readers have different range-evaluation spans: -// it keeps the panes of the longest span, charged by retention levels. -#[test] -fn a_kept_pane_chain_keeps_the_longest_readers_span() { - let span = |span_ms| { - Some(RangeEvaluation { - step_ms: 60_000, - span_ms, - }) - }; - let fixture = through_stages_with( - &[ - Query { - range_evaluation: span(600_000), - ..query("sum_over_time(m[1h])", 60_000) - }, - Query { - range_evaluation: span(3_600_000), - ..query("sum_over_time(m[1h])", 60_000) - }, - ], - false, - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let chains = candidates_where(milp, |candidate, node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) && matches!( - candidate.materialization, - Materialization::MatAtIngestion(_) - ) - }); - require_one_of(milp, &chains); - }) - .unwrap(); - assert!( - checked - .solution - .size - .families - .contains_key("π (pane retention levels)"), - "{:?}", - checked.solution.size.families.keys().collect::>() - ); -} - -// A `NotMaterialized` input of a `MatAtQuery` candidate both queries read -// (`rate(m[5m])` under `sum` and `max`, one schedule group) runs once, in -// the candidate's context, and is costed as `plan_cost` costs it. -#[test] -fn a_shared_results_not_materialized_input_runs_once() { - let fixture = through_stages( - &[ - query("sum(rate(m[5m]))", 60_000), - query("max(rate(m[5m]))", 60_000), - ], - true, - ); - let mut shared = Vec::new(); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - shared = candidates_where(milp, |candidate, _| { - matches!(candidate.materialization, Materialization::MatAtQuery(_)) - }) - .into_iter() - .filter(|&m| milp.queries_reaching(m).len() == 2) - .filter(|&m| { - let context = milp.context_of(m).expect("a context"); - milp.choice[context] - .keys() - .any(|&c| milp.computes(context, c)) - }) - .collect(); - assert!(!shared.is_empty(), "a shared result over a computed input"); - require_one_of(milp, &shared); - }) - .unwrap(); - let m = *checked - .solution - .selected - .iter() - .find(|m| shared.contains(m)) - .expect("selected"); - assert_eq!(runs_of(&checked.dag, m), 1); - let node = checked - .dag - .candidate_of_node - .iter() - .position(|&c| c == m) - .expect("a node"); - for edge in checked.dag.edges.iter().filter(|e| e.consumer.0 == node) { - let input = checked.dag.candidate_of_node[edge.producer.0]; - let readers = checked - .dag - .edges - .iter() - .filter(|e| e.producer == edge.producer) - .count(); - assert_eq!(readers, 1, "m's input {input:?} is read by m alone"); - } -} - -// One ingestion scan read by sketches of two configurations (two quantile -// targets) runs once in its `I_r` and is costed once. -#[test] -fn one_ingestion_scan_shared_by_two_sketch_configurations_is_charged_once() { - let fixture = through_stages_with( - &[ - approximate("quantile_over_time(0.5, m[5m])", 60_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - true, - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let sketches = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::IngestionTime - && candidate.materialization != Materialization::NotMaterialized - && matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryAgg { .. }) - | Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }); - let sum: Expression = sketches - .iter() - .filter_map(|c| milp.selected.get(c).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 2.0)); - // Every query-time scan is excluded: both read ingestion state. - let scans = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::QueryTime && node.operator.kind_name() == "Scan" - }); - forbid(milp, &scans); - }) - .unwrap(); - let ingestion_scans: Vec = (0..checked.dag.candidate_of_node.len()) - .filter(|&node| { - let candidate = checked.dag.candidate_of_node[node]; - fixture.physical.candidates[candidate.0].timing == ExecutionTiming::IngestionTime - && fixture - .physical - .physical_node(candidate) - .operator - .kind_name() - == "Scan" - }) - .collect(); - assert_eq!(ingestion_scans.len(), 1, "{:?}", checked.dag); -} - -/// Solves `queries` (ingesting) with two ingestion-time sketch states -/// selected, every query-time scan forbidden and every ingestion-time state -/// whose refresh window is not one of `widths_ms` (phase 0) forbidden; -/// returns the checked solve (objective = `plan_cost`) and its -/// ingestion-time scan runs, each with the refresh windows of the states -/// reading it. -fn ingestion_scans_under_refresh_windows( - queries: &[Query<'_>], - widths_ms: &[u64], -) -> (Checked, Vec>) { - use asap_types::ir::materialized_view::RefreshWindow; - let fixture = through_stages_with(queries, true, true); - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let states = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::IngestionTime - && candidate.materialization != Materialization::NotMaterialized - }); - let window_of = |state: PhysicalCandidateId| { - milp.inputs - .ingestion_context_of_shareable - .get(&state) - .map(|&context| milp.inputs.contexts[context].kind) - }; - let (allowed, other): (Vec<_>, Vec<_>) = states.iter().partition(|&&state| { - widths_ms.iter().any(|&width_ms| { - window_of(state) - == Some(ContextKind::Ingestion(IngestionPipeline::Window( - RefreshWindow { - width_ms, - phase_ms: 0, - }, - ))) - }) - }); - forbid(milp, &other); - let sketches = candidates_where(milp, |_, node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }); - let sum: Expression = allowed - .iter() - .filter(|state| sketches.contains(state)) - .filter_map(|state| milp.selected.get(state).copied()) - .map(Expression::from) - .sum(); - milp.builder.row("test", constraint!(sum >= 2.0)); - let scans = candidates_where(milp, |candidate, node| { - candidate.timing == ExecutionTiming::QueryTime && node.operator.kind_name() == "Scan" - }); - forbid(milp, &scans); - }) - .unwrap(); - // Per ingestion-time scan node of the decoded selection: the pane widths - // of the chains above it. - let dag = &checked.dag; - let nodes = dag.candidate_of_node.len(); - let mut readers: Vec> = vec![Vec::new(); nodes]; - for edge in &dag.edges { - readers[edge.producer.0].push(edge.consumer.0); - } - let width_above = |scan: usize| { - let mut widths = BTreeSet::new(); - let mut stack = vec![scan]; - let mut seen = BTreeSet::new(); - while let Some(node) = stack.pop() { - if !seen.insert(node) { - continue; - } - let candidate = dag.candidate_of_node[node]; - match fixture.physical.physical_node(candidate).asap() { - Some(asap_types::ir::ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => { - widths.insert(*tumbling_window_width_ms); - } - _ => stack.extend(readers[node].iter().copied()), - } - } - widths - }; - let scans = (0..nodes) - .filter(|&node| { - let candidate = dag.candidate_of_node[node]; - fixture.physical.candidates[candidate.0].timing == ExecutionTiming::IngestionTime - && fixture - .physical - .physical_node(candidate) - .operator - .kind_name() - == "Scan" - }) - .map(width_above) - .collect(); - (checked, scans) -} - -// Two ingestion-time pane chains of one refresh window (10 s panes) over -// one source run in one ingestion pipeline: the scan below them runs and -// is charged once, and the objective equals `plan_cost`. -#[test] -fn one_ingestion_scan_under_chains_of_one_refresh_window_is_charged_once() { - let (_, scans) = ingestion_scans_under_refresh_windows( - &[ - approximate("quantile_over_time(0.5, m[5m])", 60_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - &[10_000], - ); - assert_eq!(scans, vec![BTreeSet::from([10_000])]); -} - -// Chains of 10 s and 60 s panes over one source run in two ingestion -// pipelines (#1151): the scan runs and is charged once in each, and the -// objective equals `plan_cost`, which charges it once per refresh window. -#[test] -fn ingestion_scans_under_chains_of_two_refresh_windows_are_charged_twice() { - let (_, scans) = ingestion_scans_under_refresh_windows( - &[ - approximate("quantile_over_time(0.5, m[10s])", 10_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - &[10_000, 60_000], - ); - let mut scans = scans; - scans.sort(); - assert_eq!( - scans, - vec![BTreeSet::from([10_000]), BTreeSet::from([60_000])] - ); -} - -// Every producer candidate of every input reads one view window on real -// Stage 2 output (ingesting, and not), so a refresh window computed from -// first producers is the realized plan's. -#[test] -fn producers_of_an_input_read_one_view_window_through_stages() { - let fixtures = [ - through_stages_with( - &[ - approximate("quantile_over_time(0.5, m[10s])", 10_000, 0.01), - approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.05), - ], - true, - true, - ), - through_stages_with( - &[ - query("sum by (job) (rate(m[1m]))", 30_000), - query("sum(increase(m[5m] offset 1m))", 60_000), - approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.05, - ), - ], - true, - true, - ), - through_stages( - &[ - query("avg_over_time(m[5m])", 60_000), - query("max_over_time(m[1m])", 60_000), - ], - true, - ), - ]; - for fixture in &fixtures { - assert_eq!( - crate::groups::producer_window_disagreements_of_all(&fixture.physical), - Vec::new() - ); - } -} - -// A stateless subtree read by two state nodes of one query evaluation -// (`avg` reads a sum and a count accumulator over one `q50/q50 + q90` -// subtree) is computed once: no candidate runs twice in the plan. -#[test] -fn a_stateless_subtree_two_state_nodes_read_runs_once() { - let fixture = through_stages( - &[query( - "avg by (job) (quantile_over_time(0.5, m[5m]) / quantile_over_time(0.5, m[5m]) \ - + quantile_over_time(0.9, m[5m]))", - 60_000, - )], - true, - ); - let checked = solve_and_check(&fixture, &cost_inputs(), |_| {}).unwrap(); - let mut runs: BTreeMap = BTreeMap::new(); - for &candidate in &checked.dag.candidate_of_node { - *runs.entry(candidate).or_default() += 1; - } - assert!(runs.values().all(|&n| n == 1), "{runs:?}"); -} - -// --------------------------------------------------------------------------- -// Structure -// --------------------------------------------------------------------------- - -// "Used only if read": a query context may not select a shareable -// candidate no used reader there reads, so a phantom read cannot satisfy -// `MinimumQueryReaders`. With every reader of a two-reader `MatAtQuery` -// result in the second query's context forbidden, selecting it is -// infeasible. -#[test] -fn a_phantom_reader_does_not_satisfy_minimum_query_readers() { - let fixture = shared_sum(); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - let mut milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options, - ) - .unwrap(); - let constrained: Vec = fixture - .physical - .reader_constraints - .iter() - .filter_map(|constraint| match constraint { - ReaderConstraint::MinimumQueryReaders { - candidate, minimum, .. - } if *minimum >= 2 => Some(*candidate), - _ => None, - }) - .filter(|&m| milp.selected.contains_key(&m) && milp.choice[1].contains_key(&m)) - .collect(); - let m = *constrained.first().expect("a two-reader shareable result"); - let selected = milp.selected[&m]; - milp.builder.row("test", constraint!(selected >= 1.0)); - let readers: Vec = milp.readers[1][&m] - .iter() - .filter_map(|reader| match reader { - Reader::Candidate(reader, _) => Some(*reader), - _ => None, - }) - .collect(); - assert!( - !milp.readers[1][&m] - .iter() - .any(|reader| matches!(reader, Reader::Output(_))), - "the result is not an output of the second query" - ); - for reader in readers { - let choice = milp.choice[1][&reader]; - milp.builder.row("test", constraint!(choice <= 0.0)); - } - assert_eq!( - solve_selection_milp(milp, 0).err(), - Some(SelectionMilpError::Infeasible) - ); -} - -// A `NotMaterialized` query-time candidate selected in two contexts is two -// nodes; a shareable candidate two queries read is one node. -#[test] -fn runs_decode_one_node_per_context_and_one_per_shareable_candidate() { - let fixture = through_stages( - &[query("sum(m)", 60_000), query("sum(m) * 2", 60_000)], - true, - ); - let unshared = solve_and_check(&fixture, &cost_inputs(), |milp| { - let shared = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::QueryTime - && candidate.materialization != Materialization::NotMaterialized - }); - forbid(milp, &shared); - let ingestion = candidates_where(milp, |candidate, _| { - candidate.timing == ExecutionTiming::IngestionTime - }); - forbid(milp, &ingestion); - }) - .unwrap(); - let mut runs: BTreeMap = BTreeMap::new(); - for &candidate in &unshared.dag.candidate_of_node { - *runs.entry(candidate).or_default() += 1; - } - assert!(runs.values().any(|&n| n == 2), "{runs:?}"); - let mut shared_result = Vec::new(); - let shared = solve_and_check(&fixture, &cost_inputs(), |milp| { - shared_result = candidates_where(milp, |candidate, _| { - candidate.materialization == Materialization::MatAtQuery(Medium::Memory) - }) - .into_iter() - .filter(|&m| milp.queries_reaching(m).len() == 2) - .collect(); - require_one_of(milp, &shared_result); - }) - .unwrap(); - let m = *shared - .solution - .selected - .iter() - .find(|m| shared_result.contains(m)) - .expect("selected"); - assert_eq!(runs_of(&shared.dag, m), 1); -} - -// A query over its latency bound: with the bound below the fastest plan's -// latency there is no plan; at the optimum's latency, the optimum stays. -#[test] -fn a_plan_over_a_latency_bound_is_infeasible() { - let fixture = shared_sum(); - let optimum = solve_and_check(&fixture, &cost_inputs(), |_| {}).unwrap(); - let latency = optimum.plan.usage.latency_per_query[1].total_ms(); - let mut bounded = shared_sum(); - bounded.entries[1].requirements.max_latency_ms = Some(latency * (1.0 + 1e-9)); - let at_bound = solve_and_check(&bounded, &cost_inputs(), |_| {}).unwrap(); - assert!(at_bound.plan.usage.latency_per_query[1].total_ms() <= latency * (1.0 + 1e-9)); - let mut tight = shared_sum(); - tight.entries[1].requirements.max_latency_ms = Some(1e-9); - assert!(matches!( - solve_and_check(&tight, &cost_inputs(), |_| {}).err(), - Some(SelectionMilpError::Infeasible) - )); -} - -// Cost models 2 and 3 are not encoded. -#[test] -fn cost_models_2_and_3_are_unsupported() { - use asap_cost::{ClusterLimits, ClusterObjective, CostModel}; - let fixture = shared_sum(); - for model in [ - CostModel::ProvisionedCapacity(ResourceUnitCost::illustrative()), - CostModel::ClusterCapacity(ClusterLimits { - memory_bytes: u64::MAX, - disk_bytes: u64::MAX, - cpu_cores: None, - objective: ClusterObjective::LowestLatency, - }), - ] { - let costs = CostInputs { - cost_model: model, - ..cost_inputs() - }; - let inputs = - prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - assert!(matches!( - build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options - ) - .err(), - Some(SelectionMilpError::UnsupportedCostModel(_)) - )); - } -} - -// Accuracy rows on `q0.5 / q0.9` over metric `m` declared non-negative, -// weights from the composition: a candidate unusable for the query (a -// DDSketch denominator sized for the whole target) is never depended on, -// and two at `parts = 2` are depended on together unless their pair is -// excluded. (Pass 1 already prunes KLL, whose rank error composes no -// guarantee through the quotient.) -#[test] -fn accuracy_rows_follow_the_composition_on_a_ratio() { - let fixture = through_stages_non_negative(&[approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.01, - )]); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let of = |wanted: &dyn Fn(&AccuracyWeight, usize) -> bool| -> Vec { - inputs.accuracy_weights[0] - .iter() - .filter(|(class, weight)| { - wanted(weight, inputs.candidate_data[class.0].accuracy_budget_parts) - }) - .map(|(class, _)| *class) - .collect() - }; - let unusable = of(&|weight, _| matches!(weight, AccuracyWeight::Unusable(_))); - let usable_at_two = of(&|weight, parts| *weight == AccuracyWeight::One && parts == 2); - assert!(!unusable.is_empty() && usable_at_two.len() >= 2); - let solve = |inputs: &MilpInputs, restrict: &dyn Fn(&mut SelectionMilp<'_>)| { - let mut milp = build_selection_milp( - &fixture.physical, - inputs, - &fixture.entries, - &costs, - SelectionMilpOptions::default(), - ) - .unwrap(); - restrict(&mut milp); - solve_selection_milp(milp, 0).map(|_| ()) - }; - // Each candidate's runs: `x[m]`, or `p` in a context computing it. - let runs_of = |milp: &SelectionMilp<'_>, candidates: &[PhysicalCandidateId]| -> Expression { - let mut sum = Expression::from(0.0); - for &candidate in candidates { - if let Some(&selected) = milp.selected.get(&candidate) { - sum += selected; - } - for context in 0..milp.choice.len() { - if milp.computes(context, candidate) { - if let Some(&choice) = milp.choice[context].get(&candidate) { - sum += choice; - } - } - } - } - sum - }; - assert_eq!( - solve(&inputs, &|milp| { - let sum = runs_of(milp, &unusable); - milp.builder.row("test", constraint!(sum >= 1.0)); - }) - .err(), - Some(SelectionMilpError::Infeasible) - ); - let two_of = |milp: &mut SelectionMilp<'_>| { - let sum = runs_of(milp, &usable_at_two); - milp.builder.row("test", constraint!(sum >= 2.0)); - }; - assert!(solve(&inputs, &two_of).is_ok()); - let mut excluded = inputs.clone(); - for (index, &first) in usable_at_two.iter().enumerate() { - for &second in &usable_at_two[index + 1..] { - excluded.accuracy_excluded_pairs[0].push((first, second)); - } - } - assert_eq!( - solve(&excluded, &two_of).err(), - Some(SelectionMilpError::Infeasible) - ); -} - -// The warm start is a feasible plan whose completion's objective equals -// what `plan_cost` says of it: fixing every binary to it, the solve -// returns the warm start's cost. -#[test] -fn the_warm_start_is_a_feasible_plan_costed_exactly() { - for fixture in [shared_sum(), two_windows()] { - let mut start_cost = None; - let checked = solve_and_check(&fixture, &cost_inputs(), |milp| { - let completion = crate::milp::completion::Completion::new(milp); - let start = - crate::milp::warm_start::warm_start(milp, &completion).expect("a warm start"); - start_cost = Some(start.cost_per_second); - let values: HashMap = start.values.into_iter().collect(); - let mut fixed = Vec::new(); - for choices in &milp.choice { - fixed.extend(choices.values().copied()); - } - fixed.extend(milp.selected.values().copied()); - for variable in fixed { - let value = values.get(&variable).copied().unwrap_or(0.0); - milp.builder.row("test", constraint!(variable == value)); - } - }) - .unwrap(); - let start_cost = start_cost.expect("costed"); - let tolerance = 1e-6 * start_cost.abs().max(1e-12); - assert!( - (checked.solution.cost_per_second - start_cost).abs() <= tolerance, - "warm start {start_cost:e} != plan_cost {:e}", - checked.solution.cost_per_second - ); - } -} - -// RangeHeavy templates 7 and 4: the widened scan puts a `Project` in the -// memo group of the scan it reads. The group's roots are split by -// self-read depth, so the plan is acyclic, valid, accepted by the realizer -// and costed exactly. -#[test] -fn a_widened_scan_reading_its_own_memo_group_plans_exactly() { - let fixture = through_stages( - &[ - approximate("sum by (job) (sum_over_time(m[1h]))", 60_000, 0.05), - approximate("sum_over_time(m[1h])", 60_000, 0.05), - ], - true, - ); - let reads_own_group = (0..fixture.physical.candidates.len()).any(|candidate| { - let group = fixture - .physical - .memo_group_of(PhysicalCandidateId(candidate)); - fixture.physical.candidates[candidate] - .producer_candidates_per_input - .iter() - .any(|producers| { - producers - .iter() - .any(|p| p.0 != candidate && fixture.physical.memo_group_of(*p) == group) - }) - }); - assert!( - reads_own_group, - "the fixture has a root reading its own group" - ); - for costs in [cost_inputs(), aws_fargate_cost_inputs()] { - solve_and_check(&fixture, &costs, |_| {}).unwrap(); - } -} - -// #1095 §5 (3a): queries over two metrics share no candidate, run of an `I_r` or -// source, so the model splits into one part per metric; solving the parts -// apart gives the plan and cost of one solve of the whole model. -#[test] -fn independent_query_sets_solve_apart_to_the_same_plan() { - let queries = [ - query("sum_over_time(m[5m])", 60_000), - query("sum_over_time(m[1h])", 60_000), - query("sum(n) + max(n)", 30_000), - ]; - let fixture = through_stages_over(&queries, &["m", "n"], true, false, false); - let costs = cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let options = SelectionMilpOptions::default(); - let milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - options, - ) - .unwrap(); - let parts = independent_parts(&milp); - let queries_per_part: Vec> = parts.iter().map(|part| part.queries.clone()).collect(); - assert_eq!(queries_per_part, vec![vec![0, 1], vec![2]]); - let columns: usize = parts.iter().map(|part| part.columns.len()).sum(); - let rows: usize = parts.iter().map(|part| part.rows.len()).sum(); - assert_eq!(columns, milp.builder.variables.len()); - assert_eq!(rows, milp.builder.rows.len()); - - let apart = solve_and_check_by(&fixture, &costs, |_| {}, true).unwrap(); - let whole = solve_and_check_by(&fixture, &costs, |_| {}, false).unwrap(); - assert_eq!(apart.solution.parts.len(), 2); - assert_eq!(whole.solution.parts.len(), 1); - let tolerance = 1e-9 * whole.solution.cost_per_second.abs(); - assert!( - (apart.solution.cost_per_second - whole.solution.cost_per_second).abs() <= tolerance, - "apart {:e} != whole {:e}", - apart.solution.cost_per_second, - whole.solution.cost_per_second - ); - assert_eq!(apart.dag.candidate_of_node, whole.dag.candidate_of_node); -} - -thread_local! { - /// Set by a test to build the big-M bounds as before #1095 step 1 - /// ([`super::tests_loose_big_m`]). - pub(crate) static LOOSE_BIG_M: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - -/// Per row of family `family` in `fixture`'s model (built with the loose -/// big-M bounds when `loose`), the largest coefficient on an `x` variable. -fn big_m_per_row(fixture: &Fixture, family: &str, loose: bool) -> Vec { - LOOSE_BIG_M.with(|cell| cell.set(loose)); - let costs = aws_fargate_cost_inputs(); - let inputs = prepare_milp_inputs(&fixture.physical, &fixture.entries, &fixture.data, &costs); - let milp = build_selection_milp( - &fixture.physical, - &inputs, - &fixture.entries, - &costs, - SelectionMilpOptions::default(), - ) - .unwrap(); - LOOSE_BIG_M.with(|cell| cell.set(false)); - let selected: std::collections::HashSet = milp.selected.values().copied().collect(); - milp.builder - .rows - .iter() - .zip(&milp.builder.row_families) - .filter(|(_, &f)| f == family) - .map(|(row, _)| { - row.expression() - .linear_coefficients() - .filter(|(variable, _)| selected.contains(variable)) - .map(|(_, coefficient)| coefficient.abs()) - .fold(0.0, f64::max) - }) - .collect() -} - -// #1095 step 1: `Ē[m]` in `V ≥ E − Ē (x[m] − z)` is the sum over the -// context's slots of its costliest run. With a rollup read every 20 s and -// every 30 s, a shared context has slots with several runs, so the bound is -// below the old sum over every run; the plan and its cost are unchanged. -#[test] -fn a_tighter_context_execution_bound_keeps_the_plan() { - let fixture = through_stages( - &[ - query("sum by (job) (sum_over_time(m[10m]))", 20_000), - query("sum(sum_over_time(m[10m]))", 30_000), - ], - true, - ); - let family = "V (context execution at a rate component)"; - let tight = big_m_per_row(&fixture, family, false); - let loose = big_m_per_row(&fixture, family, true); - assert_eq!(tight.len(), loose.len()); - assert!(tight - .iter() - .zip(&loose) - .all(|(t, l)| *t <= l * (1.0 + 1e-12))); - assert!(tight.iter().zip(&loose).any(|(t, l)| *t < l * (1.0 - 1e-9))); - - let tightened = solve_and_check(&fixture, &aws_fargate_cost_inputs(), |_| {}).unwrap(); - LOOSE_BIG_M.with(|cell| cell.set(true)); - let looser = solve_and_check(&fixture, &aws_fargate_cost_inputs(), |_| {}); - LOOSE_BIG_M.with(|cell| cell.set(false)); - let looser = looser.unwrap(); - let tolerance = 1e-9 * looser.solution.cost_per_second.abs(); - assert!( - (tightened.solution.cost_per_second - looser.solution.cost_per_second).abs() <= tolerance - ); - assert_eq!( - tightened.dag.candidate_of_node, - looser.dag.candidate_of_node - ); -} diff --git a/crates/plan-selection/src/milp/warm_start.rs b/crates/plan-selection/src/milp/warm_start.rs deleted file mode 100644 index b7de6ae37..000000000 --- a/crates/plan-selection/src/milp/warm_start.rs +++ /dev/null @@ -1,780 +0,0 @@ -//! A feasible starting plan for the group selection MILP (#1095 §2.0, -//! "Solving strategy" step 1), so HiGHS starts with an incumbent. -//! -//! **Each query's cheapest plan.** Queries are planned one after another -//! into one plan. A query's plan is derived from its output references -//! down: in each context, each slot an input needs takes the candidate -//! with the cheapest estimated subtree (its usage at the context's rate -//! plus its inputs' cheapest subtrees), unless the context already chose a -//! candidate of that slot, which every reader must then accept (one -//! candidate per slot per context); a dead end backtracks to the next -//! candidate. A shareable candidate's context is derived once, when it is -//! first selected; a candidate of an `I_r` is chosen in the `I_r` of the -//! reading context's refresh window. -//! -//! **Repairs per query.** If the query's plan depends on more approximate -//! values than one of them is sized for, on a pair excluded together, or -//! on a candidate unusable for it, the query is planned again with exact -//! candidates only; if its estimated latency is over its bound, it is -//! planned again by latency alone. -//! -//! **Greedy sharing.** A second plan prices what the queries planned -//! before already selected (a shareable candidate, a candidate of an `I_r`) at -//! zero, so later queries read it. Both plans, in workload order and -//! reversed, are costed exactly by the MILP's own rows and objective -//! ([`Completion`]); the cheapest feasible one is the start. -use std::collections::{BTreeMap, BTreeSet, HashMap}; - -use asap_cost::execution_demand::evaluation_rate; -use asap_types::stages::{output_references, PhysicalCandidateId, ReaderConstraint}; -use good_lp::Variable; - -use crate::groups::{AccuracyWeight, ContextKind, ResultSlot, RunKind}; - -use super::completion::Completion; -use super::cost_terms::NodeNumbers; -use super::model::{latency_bound_ms, SelectionMilp}; - -/// The most derivation steps one query's plan takes before giving up. -const MOST_STEPS_PER_QUERY: usize = 200_000; - -/// The starting plan. -pub(crate) struct StartingPlan { - /// Every variable's value. - pub(crate) values: Vec<(Variable, f64)>, - /// Its objective (cost per second without the request fees). - pub(crate) cost_per_second: f64, -} - -/// What a derivation minimizes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Objective { - Cost, - Latency, -} - -/// The candidates a query's derivation may use. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Approximate { - /// Approximate candidates usable for the query. - Usable, - /// Those sized for at least this many shares. - SizedFor(usize), - /// Exact candidates only. - None, -} - -/// A plan under construction: per context, the candidate of each slot, and -/// the selected shareable candidates. -#[derive(Debug, Clone, Default)] -struct Plan { - chosen: Vec>, - selected: BTreeSet, - /// Per multi-slot input `(context, reader, input)`, the slot it reads. - slot_reads: BTreeMap<(usize, PhysicalCandidateId, usize), ResultSlot>, -} - -/// One change to a [`Plan`], to undo. -enum Undo { - Chose(usize, ResultSlot), - Selected(PhysicalCandidateId), - SlotRead(usize, PhysicalCandidateId, usize), -} - -/// A feasible plan (module documentation), or `None` when none was found. -/// It tries a fixed number of query orders ([`MORE_ORDERS`]), so every run -/// starts from the same plan. -pub(crate) fn warm_start( - milp: &SelectionMilp<'_>, - completion: &Completion, -) -> Option { - let estimates = Estimates::new(milp); - let mut best: Option = None; - let count = milp.query_count(); - let mut orders: Vec> = vec![(0..count).collect(), (0..count).rev().collect()]; - // More orders: a fixed pseudo-random sequence of permutations, so runs - // are reproducible. - let mut state: u64 = 0x9e37_79b9_7f4a_7c15; - for _ in 0..MORE_ORDERS { - let mut order: Vec = (0..count).collect(); - for at in (1..count).rev() { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - order.swap(at, (state % (at as u64 + 1)) as usize); - } - orders.push(order); - } - // (share, amortized): alone; reading what earlier queries selected; and - // that with each shareable candidate's own cost divided among the - // queries that may reach it. - let strategies = [(false, false), (true, false), (true, true)]; - let mut tried = 0; - for order in &orders { - for &(share, amortized) in &strategies { - tried += 1; - let Some(plan) = - sequential_plan(milp, &estimates, order, share, amortized, &BTreeSet::new()) - else { - debug(|| format!("warm start: no plan (share {share}, amortized {amortized})")); - continue; - }; - let binaries = binaries_of(milp, &plan); - let completed = completion.complete(&binaries); - if let Err(reason) = &completed { - debug(|| format!("warm start: completion failed (share {share}): {reason}")); - } - if let Ok(completed) = completed { - debug(|| { - format!( - "warm start: plan (share {share}, amortized {amortized}) costs {:e}, {} shareable selected", - completed.cost_per_second, - plan.selected.len() - ) - }); - if best - .as_ref() - .is_none_or(|known| completed.cost_per_second < known.cost_per_second) - { - best = Some(StartingPlan { - values: completion.values(&completed), - cost_per_second: completed.cost_per_second, - }); - } - } - } - } - // Greedy sharing: prefer one more shareable candidate that several - // queries may read (most readers first), kept when the exact cost - // falls. - let mut sharable: Vec<(usize, PhysicalCandidateId)> = milp - .inputs - .context_of_shareable - .keys() - .map(|&m| (estimates.readers[m.0], m)) - .filter(|&(readers, _)| readers >= 2) - .collect(); - sharable.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); - let mut preferred: BTreeSet = BTreeSet::new(); - for (_, m) in sharable { - if best.is_none() { - break; - } - let mut trial = preferred.clone(); - trial.insert(m); - tried += 1; - let Some(plan) = sequential_plan(milp, &estimates, &orders[0], true, false, &trial) else { - continue; - }; - if let Ok(completed) = completion.complete(&binaries_of(milp, &plan)) { - if best - .as_ref() - .is_some_and(|known| completed.cost_per_second < known.cost_per_second) - { - preferred = trial; - best = Some(StartingPlan { - values: completion.values(&completed), - cost_per_second: completed.cost_per_second, - }); - } - } - } - debug(|| { - format!( - "warm start: {tried} plans tried, best {:?}", - best.as_ref().map(|best| best.cost_per_second) - ) - }); - best -} - -/// The pseudo-random orders tried after workload order and its reverse. -const MORE_ORDERS: usize = 30; - -/// The plan's binaries: `p[ctx, c]` and `x[m]` set to 1. -fn binaries_of(milp: &SelectionMilp<'_>, plan: &Plan) -> Vec<(Variable, f64)> { - let mut binaries = Vec::new(); - for (context, chosen) in plan.chosen.iter().enumerate() { - for candidate in chosen.values() { - if let Some(&variable) = milp.choice[context].get(candidate) { - binaries.push((variable, 1.0)); - } - } - } - for shareable in &plan.selected { - if let Some(&variable) = milp.selected.get(shareable) { - binaries.push((variable, 1.0)); - } - } - for (key, slot) in &plan.slot_reads { - if let Some(links) = milp.slot_links.get(key) { - for &(at, link) in links { - if at == *slot { - binaries.push((link, 1.0)); - } - } - } - } - binaries -} - -/// Every query planned in `order` into one plan; with `share`, what is -/// already selected costs nothing to read. -fn sequential_plan( - milp: &SelectionMilp<'_>, - estimates: &Estimates, - order: &[usize], - share: bool, - amortized: bool, - preferred: &BTreeSet, -) -> Option { - let mut plan = Plan { - chosen: vec![BTreeMap::new(); milp.choice.len()], - selected: BTreeSet::new(), - slot_reads: BTreeMap::new(), - }; - for &query in order { - let attempts = [ - (Objective::Cost, Approximate::Usable), - (Objective::Cost, Approximate::SizedFor(2)), - (Objective::Cost, Approximate::SizedFor(4)), - (Objective::Cost, Approximate::None), - (Objective::Latency, Approximate::Usable), - (Objective::Latency, Approximate::SizedFor(2)), - (Objective::Latency, Approximate::SizedFor(4)), - (Objective::Latency, Approximate::None), - ]; - // The first attempt meeting accuracy and the latency estimate; else - // the last meeting accuracy (the latency estimate is from each - // candidate's first costed usage, so it may be pessimistic: the - // completion checks the exact latency). - let mut planned = false; - let mut fallback = None; - for (attempt, (objective, approximate)) in attempts.into_iter().enumerate() { - let mut deriving = Derivation { - milp, - estimates, - query, - objective, - approximate, - share, - amortized, - preferred, - undo: Vec::new(), - steps: 0, - references: output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .map(|(reference, _)| reference) - .collect(), - }; - let derived = deriving.derive_query(&mut plan).is_ok(); - let accurate = derived && meets_accuracy(milp, &plan, query); - let fast = accurate && meets_latency(milp, &plan, query); - debug(|| { - format!( - "warm start: q{query} {objective:?}/{approximate:?}: derived {derived}, \ - accuracy {accurate}, latency {fast}, steps {}", - deriving.steps - ) - }); - if fast { - planned = true; - break; - } - if accurate { - fallback = Some(attempt); - } - let undo = std::mem::take(&mut deriving.undo); - undo_all(&mut plan, undo); - } - if let (false, Some(attempt)) = (planned, fallback) { - let (objective, approximate) = attempts[attempt]; - let mut deriving = Derivation { - milp, - estimates, - query, - objective, - approximate, - share, - amortized, - preferred, - undo: Vec::new(), - steps: 0, - references: output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - .filter(|(_, (q, _))| *q == query) - .map(|(reference, _)| reference) - .collect(), - }; - planned = deriving.derive_query(&mut plan).is_ok(); - } - if !planned { - return None; - } - } - Some(plan) -} - -fn undo_all(plan: &mut Plan, undo: Vec) { - for change in undo.into_iter().rev() { - match change { - Undo::Chose(context, slot) => { - plan.chosen[context].remove(&slot); - } - Undo::Selected(shareable) => { - plan.selected.remove(&shareable); - } - Undo::SlotRead(context, reader, input) => { - plan.slot_reads.remove(&(context, reader, input)); - } - } - } -} - -/// Per context and candidate, the estimated cost and latency of the -/// cheapest subtree, per query (the rate of a query context; shared -/// contexts at the slowest rate of the queries that may reach them). -struct Estimates { - /// Per context, its rate (executions per second). - rate: Vec, - /// Per candidate, its numbers (first costed context). - numbers: Vec>, - /// Per candidate, the queries that may depend on it (shareable ones). - readers: Vec, -} - -impl Estimates { - fn new(milp: &SelectionMilp<'_>) -> Self { - let horizon = milp.cost_inputs.non_repeating_query_horizon_seconds; - let query_rate: Vec = milp - .entries - .iter() - .map(|entry| { - evaluation_rate([(&entry.recurrence, entry.range_evaluation)], horizon) - .unwrap_or(1.0) - }) - .collect(); - let rate = milp - .inputs - .contexts - .iter() - .map(|context| match context.kind { - ContextKind::Query(query) => query_rate[query], - ContextKind::Ingestion(_) => 1.0, - ContextKind::Shared(m) if context.at_query_time => { - let readers = milp.queries_reaching(m); - readers - .iter() - .map(|&query| query_rate[query]) - .fold(f64::INFINITY, f64::min) - .min(1e9) - } - ContextKind::Shared(_) => 1.0, - }) - .collect(); - let numbers = milp - .inputs - .candidate_data - .iter() - .map(|data| data.usage.as_ref().ok().map(NodeNumbers::of)) - .collect(); - let mut readers = vec![0; milp.inputs.candidate_data.len()]; - for &m in milp.inputs.context_of_shareable.keys() { - readers[m.0] = milp.queries_reaching(m).len(); - } - Self { - rate, - numbers, - readers, - } - } -} - -/// One query's derivation into a plan. -struct Derivation<'m, 'a> { - milp: &'m SelectionMilp<'a>, - estimates: &'m Estimates, - query: usize, - objective: Objective, - approximate: Approximate, - share: bool, - /// Divide a shareable candidate's own cost among the queries that may - /// reach it. - amortized: bool, - /// Shareable candidates estimated free to read (greedy sharing). - preferred: &'m BTreeSet, - undo: Vec, - steps: usize, - /// The query's output references. - references: Vec, -} - -impl Derivation<'_, '_> { - /// Whether the query may use `candidate`. - fn allowed(&self, candidate: PhysicalCandidateId) -> bool { - let inputs = self.milp.inputs; - if inputs.candidate_data[candidate.0].approximate { - if self.approximate == Approximate::None - || self.milp.entries[self.query] - .requirements - .accuracy - .is_exact() - { - return false; - } - if let Some(AccuracyWeight::Unusable(_)) = - inputs.accuracy_weights[self.query].get(&candidate) - { - return false; - } - if let Approximate::SizedFor(shares) = self.approximate { - if inputs.candidate_data[candidate.0].accuracy_budget_parts < shares { - return false; - } - } - } - // A candidate needing two or more reading queries is not planned - // alone; one some of the query's outputs may not reach is not used. - let references = &self.references; - !self - .milp - .physical - .reader_constraints - .iter() - .any(|constraint| match constraint { - ReaderConstraint::MinimumQueryReaders { - candidate: c, - minimum, - .. - } => *c == candidate && *minimum > 1, - ReaderConstraint::AllowedOutputs { - candidate: c, - outputs, - } => *c == candidate && references.iter().any(|r| !outputs.contains(r)), - ReaderConstraint::IncompatibleOutputs { - candidate: c, - left, - right, - } => *c == candidate && references.contains(left) && references.contains(right), - ReaderConstraint::HighestSharedQueryMaterialization { .. } => false, - }) - } - - /// The estimated cost (or latency) of `candidate`'s own run read in - /// `context`, plus the cheapest subtrees of its inputs. - fn estimate( - &self, - plan: &Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - candidate: PhysicalCandidateId, - ) -> f64 { - if let Some(&known) = memo.get(&(context, candidate)) { - return known; - } - memo.insert((context, candidate), f64::INFINITY); - let milp = self.milp; - let inputs = milp.inputs; - let value = (|| { - if !self.allowed(candidate) { - return f64::INFINITY; - } - let Some(numbers) = self.estimates.numbers[candidate.0] else { - return f64::INFINITY; - }; - let kind = inputs.kind[candidate.0]; - // Where its own run is and where it reads its inputs. - let (run_context, already) = if milp.computes(context, candidate) { - (context, false) - } else { - match kind { - RunKind::Shareable => match milp.context_of(candidate) { - Some(own) => (own, plan.selected.contains(&candidate)), - None => return f64::INFINITY, - }, - RunKind::Ingestion => { - let Some(ingestion) = milp - .ingestion_context_read_by(context) - .filter(|&at| milp.choice[at].contains_key(&candidate)) - else { - return f64::INFINITY; - }; - let chosen = plan.chosen[ingestion].get(&inputs.slot[candidate.0]); - (ingestion, chosen == Some(&candidate)) - } - RunKind::PerContext => return f64::INFINITY, - } - }; - if (self.share && already) || self.preferred.contains(&candidate) { - return 0.0; - } - let mut own = match self.objective { - Objective::Cost => { - numbers.execution_cost * self.estimates.rate[run_context] - + numbers.standing_cost - } - Objective::Latency => numbers.latency_ms, - }; - if self.amortized && kind == RunKind::Shareable && self.objective == Objective::Cost { - own /= self.estimates.readers[candidate.0].max(1) as f64; - } - let mut total = own; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - let best = producers - .iter() - .filter(|producer| milp.choice[run_context].contains_key(producer)) - .filter(|producer| { - kind == RunKind::Shareable - || inputs.slot[producer.0] != inputs.slot[candidate.0] - }) - .map(|&producer| self.estimate(plan, memo, run_context, producer)) - .fold(f64::INFINITY, f64::min); - total += best; - } - total - })(); - memo.insert((context, candidate), value); - value - } - - fn derive_query(&mut self, plan: &mut Plan) -> Result<(), ()> { - let milp = self.milp; - let mut memo = HashMap::new(); - for (reference, (query, _)) in output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - { - if query != self.query { - continue; - } - let allowed: Vec = milp.inputs.output_candidates[reference] - .iter() - .copied() - .filter(|candidate| milp.choice[query].contains_key(candidate)) - .collect(); - self.read(plan, &mut memo, query, &allowed)?; - } - Ok(()) - } - - /// Makes `context` give one of `allowed` (one input's producers, or an - /// output's candidates). - fn read( - &mut self, - plan: &mut Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - allowed: &[PhysicalCandidateId], - ) -> Result { - self.steps += 1; - if self.steps > MOST_STEPS_PER_QUERY { - return Err(()); - } - let milp = self.milp; - let inputs = milp.inputs; - // A slot already chosen here with an allowed candidate. - for &candidate in allowed { - if plan.chosen[context].get(&inputs.slot[candidate.0]) == Some(&candidate) { - return Ok(candidate); - } - } - let mut options: Vec<(f64, PhysicalCandidateId)> = allowed - .iter() - .copied() - .filter(|candidate| milp.choice[context].contains_key(candidate)) - .filter(|candidate| !plan.chosen[context].contains_key(&inputs.slot[candidate.0])) - .map(|candidate| (self.estimate(plan, memo, context, candidate), candidate)) - .collect(); - options.retain(|(estimate, _)| estimate.is_finite()); - options.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))); - for (_, candidate) in options { - let mark = self.undo.len(); - if self.choose(plan, memo, context, candidate).is_ok() { - return Ok(candidate); - } - let undo = self.undo.split_off(mark); - undo_all(plan, undo); - } - Err(()) - } - - /// Chooses `candidate` for its slot in `context` and derives what it - /// needs. - fn choose( - &mut self, - plan: &mut Plan, - memo: &mut HashMap<(usize, PhysicalCandidateId), f64>, - context: usize, - candidate: PhysicalCandidateId, - ) -> Result<(), ()> { - let milp = self.milp; - let inputs = milp.inputs; - let slot = inputs.slot[candidate.0]; - plan.chosen[context].insert(slot, candidate); - self.undo.push(Undo::Chose(context, slot)); - if milp.computes(context, candidate) { - for (input, producers) in inputs.producer_candidates_per_input[candidate.0] - .iter() - .enumerate() - { - let producers: Vec = producers - .iter() - .copied() - .filter(|producer| inputs.slot[producer.0] != slot) - .collect(); - let read = self.read(plan, memo, context, &producers)?; - if milp.slot_links.contains_key(&(context, candidate, input)) { - plan.slot_reads - .insert((context, candidate, input), inputs.slot[read.0]); - self.undo.push(Undo::SlotRead(context, candidate, input)); - } - } - return Ok(()); - } - match inputs.kind[candidate.0] { - RunKind::Shareable => { - if plan.selected.contains(&candidate) { - return Ok(()); - } - plan.selected.insert(candidate); - self.undo.push(Undo::Selected(candidate)); - let own = milp.context_of(candidate).ok_or(())?; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - self.read(plan, memo, own, producers)?; - } - Ok(()) - } - RunKind::Ingestion => { - let ingestion = milp.ingestion_context_read_by(context).ok_or(())?; - if context == ingestion { - return Err(()); - } - self.read(plan, memo, ingestion, &[candidate]).map(|_| ()) - } - RunKind::PerContext => Err(()), - } - } -} - -/// The runs query `query` depends on in `plan`: `(context, candidate)` for -/// candidates computed in its context and in the contexts of the -/// shareable candidates it reaches, and each such shareable candidate -/// `(its context, itself)`. -fn runs_reached( - milp: &SelectionMilp<'_>, - plan: &Plan, - query: usize, -) -> BTreeSet<(usize, PhysicalCandidateId)> { - let inputs = milp.inputs; - let mut reached = BTreeSet::new(); - let mut stack: Vec<(usize, PhysicalCandidateId)> = Vec::new(); - for (reference, (q, _)) in output_references(&milp.physical.query_roots) - .into_iter() - .enumerate() - { - if q != query { - continue; - } - for &candidate in &inputs.output_candidates[reference] { - if plan.chosen[query].get(&inputs.slot[candidate.0]) == Some(&candidate) { - stack.push((query, candidate)); - } - } - } - while let Some((context, candidate)) = stack.pop() { - let (run_context, own) = if milp.computes(context, candidate) { - (context, false) - } else { - match inputs.kind[candidate.0] { - RunKind::Shareable => match milp.context_of(candidate) { - Some(own) => (own, true), - None => continue, - }, - RunKind::Ingestion => match milp.ingestion_context_read_by(context) { - Some(ingestion) => (ingestion, false), - None => continue, - }, - RunKind::PerContext => continue, - } - }; - if !reached.insert((run_context, candidate)) { - continue; - } - let slot = inputs.slot[candidate.0]; - for producers in &inputs.producer_candidates_per_input[candidate.0] { - for &producer in producers { - if !own && inputs.slot[producer.0] == slot { - continue; - } - if plan.chosen[run_context].get(&inputs.slot[producer.0]) == Some(&producer) { - stack.push((run_context, producer)); - break; - } - } - } - } - reached -} - -/// Whether query `query`'s approximate runs in `plan` meet the A1 rows. -fn meets_accuracy(milp: &SelectionMilp<'_>, plan: &Plan, query: usize) -> bool { - let inputs = milp.inputs; - let runs: Vec = runs_reached(milp, plan, query) - .into_iter() - .map(|(_, candidate)| candidate) - .filter(|candidate| inputs.candidate_data[candidate.0].approximate) - .filter(|candidate| inputs.accuracy_weights[query].contains_key(candidate)) - .collect(); - if runs.is_empty() { - return true; - } - if milp.entries[query].requirements.accuracy.is_exact() { - return false; - } - let count = runs.len(); - for &candidate in &runs { - if matches!( - inputs.accuracy_weights[query].get(&candidate), - Some(AccuracyWeight::Unusable(_)) - ) { - return false; - } - if inputs.candidate_data[candidate.0].accuracy_budget_parts < count { - return false; - } - } - !inputs.accuracy_excluded_pairs[query] - .iter() - .any(|(first, second)| runs.contains(first) && runs.contains(second)) -} - -/// Whether query `query`'s estimated latency in `plan` is within its bound. -fn meets_latency(milp: &SelectionMilp<'_>, plan: &Plan, query: usize) -> bool { - let Some(bound) = latency_bound_ms(&milp.entries[query]) else { - return true; - }; - let request_ms = milp.cost_inputs.measured_operation_costs.ns_per_evaluation * 1e-6; - let latency: f64 = runs_reached(milp, plan, query) - .into_iter() - .filter(|&(context, _)| milp.runs_at_query_time(context)) - .filter_map(|(_, candidate)| milp.inputs.candidate_data[candidate.0].usage.as_ref().ok()) - .map(|usage| usage.own_latency_ms) - .sum(); - debug(|| { - format!( - "warm start: q{query} estimated latency {} ms, bound {bound} ms", - latency + request_ms - ) - }); - latency + request_ms <= bound -} - -/// Prints `message` to stderr when `ASAP_STAGE3_DEBUG` is set. -pub(crate) fn debug(message: impl FnOnce() -> String) { - if std::env::var_os("ASAP_STAGE3_DEBUG").is_some() { - eprintln!("{}", message()); - } -} diff --git a/crates/plan-selection/src/models.rs b/crates/plan-selection/src/models.rs index bbc0d88f0..763476654 100644 --- a/crates/plan-selection/src/models.rs +++ b/crates/plan-selection/src/models.rs @@ -1,9 +1,6 @@ -//! The models Stage 3 checks and compares with: [`AccuracyCheckInputs`] -//! (validity) and [`CostInputs`] (the objective), and the engine's view of -//! both, [`SelectionModels`]. +//! The accuracy models Stage 3 checks with: [`AccuracyCheckInputs`]. use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; -use asap_cost::CostInputs; /// Inputs of the composed-accuracy check. `raw_data_retained` is a Stage 2 /// input only; to ignore accuracy guarantees the facade passes an @@ -18,56 +15,3 @@ pub struct AccuracyCheckInputs<'a> { /// guarantees). pub evidence: &'a dyn AccuracyEvidenceProvider, } - -/// Stage 3's accuracy model and evidence with its cost inputs, as the -/// engine reads them. -#[derive(Clone, Copy)] -pub(crate) struct SelectionModels<'a> { - pub accuracy: &'a dyn AccuracyModel, - pub evidence: &'a dyn AccuracyEvidenceProvider, - pub cost_inputs: &'a CostInputs, -} - -impl<'a> SelectionModels<'a> { - pub(crate) fn new( - accuracy_checks: &AccuracyCheckInputs<'a>, - cost_inputs: &'a CostInputs, - ) -> Self { - Self { - accuracy: accuracy_checks.model, - evidence: accuracy_checks.evidence, - cost_inputs, - } - } - - /// The built-in accuracy model and no evidence, under default cost - /// inputs. - #[cfg(test)] - pub(crate) fn builtin() -> SelectionModels<'static> { - static DEFAULT_COST_INPUTS: std::sync::LazyLock = - std::sync::LazyLock::new(CostInputs::default); - SelectionModels { - accuracy: &asap_accuracy::BuiltinAccuracyModel, - evidence: &asap_accuracy::NoAccuracyEvidence, - cost_inputs: &DEFAULT_COST_INPUTS, - } - } - - #[cfg(test)] - pub(crate) fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { - self.accuracy = accuracy; - self - } - - #[cfg(test)] - pub(crate) fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidenceProvider) -> Self { - self.evidence = evidence; - self - } - - #[cfg(test)] - pub(crate) fn with_cost_inputs(mut self, cost_inputs: &'a CostInputs) -> Self { - self.cost_inputs = cost_inputs; - self - } -} diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index e2617b580..d5eb6298e 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -7,8 +7,6 @@ use asap_cost::analytical::AnalyticalCostError; use asap_cost::{CostModel, PlanCost}; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::properties::AccuracyMetric; -#[cfg(any(test, feature = "reference-pricer"))] -use asap_types::ir::QueryRoot; use asap_types::stages::{ DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate, SelectedNodeId, }; @@ -183,32 +181,6 @@ impl std::fmt::Display for AcceptedAccuracy { } } -/// The whole-plan selector's result (`PlanSelector`): -/// the selected candidate among whole physical plans and the plan to run, -/// with what it kept of the other candidates. The reference pricer (feature -/// `reference-pricer`) selects among enumerated plans with it. Keeping -/// every candidate, `costs` and `accepted` list every valid candidate and -/// `rejected` every other one; otherwise only the selected candidate is -/// kept, and `rejected` holds the first rejection of each cause. -#[cfg(any(test, feature = "reference-pricer"))] -#[derive(Debug, Clone, PartialEq)] -pub struct WholePlanSelection { - pub selected: String, - /// The selected candidate's query roots with every weight requirement - /// Stage 3 confirmed made a proof: the plan to run. The Stage 2 - /// candidate keeps the open requirement. - pub plan: Vec, - pub costs: BTreeMap, - /// For each valid candidate, per query, the accuracy Stage 3 accepted. - pub accepted: BTreeMap>, - pub rejected: Vec, - /// The selected plan's tumbling-window retention, by node id in its physical DAG - /// ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)). - pub tumbling_window_retention_ms: BTreeMap, - /// The physical candidates checked, valid or not. - pub candidates_checked: u64, -} - /// Stage 3's result ([`select_plan`](crate::select_plan)): the selected DAG /// over the [`PhysicalASAPMemoCandidates`](asap_types::stages::PhysicalASAPMemoCandidates) /// it was given, its cost and accepted accuracy, and the search report. diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index ed5546432..f0468ae33 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -1,63 +1,21 @@ -//! Stage 3, plan selection: [`select_plan`] runs the group selection MILP -//! (#1095). Inputs it does not price yet (cost models 2 and 3, or a measured -//! operation cost table without a frozen calibration factor) are an -//! `UnsupportedCostModel` error (#888). -//! -//! One MILP solve and one realization, no loop: -//! 1. the MILP's inputs (candidate dominance, execution contexts, accuracy -//! weights) and the model, solved once with HiGHS from a warm start to -//! proven optimality (no time limit); -//! 2. the solution decoded into a [`DagOfSelectedCandidates`] -//! ([`crate::milp::decode`]) and validated against the catalog; -//! 3. the plan realized once through the injected realizer: a rejection is -//! a Stage 2 catalog bug ([`ErrorSelectingPlan::RealizerRejected`]); -//! 4. exact checks: `asap_cost::plan_cost` of the realized plan minus the -//! request fees equals the objective within a relative 10⁻⁶ (otherwise -//! an error, or, when the model encoded some term inexactly, -//! [`SearchReport::inexact_objective`]); every query's accuracy -//! (`satisfies`) and latency hold, or it is an error. -//! -//! With `ASAP_STAGE3_REPORT=`, one JSON line per run is appended -//! there: the model's size per family and per scope (the shareable -//! candidates' contexts apart), the timings, the warm start's cost, the -//! objective, `plan_cost` and the dual bound (measurement only). - -use std::collections::BTreeMap; -use std::time::Instant; - -use asap_cost::{kept_tumbling_window_chains, node_map, plan_cost, CostInputs}; -use asap_types::ir::{ASAPOp, Operator}; -use asap_types::stages::{ - AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidateError, PhysicalCandidateId, PruneReason, PrunedCandidate, RealizedPlan, - SelectedNodeId, -}; -use asap_types::workload::bounds::confirmed_weights; +//! Stage 3, plan selection: [`select_plan`] builds the selection MILP over +//! Stage 2's physical candidates and admissible edges +//! ([`crate::milp::model`]), solves it, decodes $\{x = 1\}$ and $\{y = 1\}$ +//! into a [`DagOfSelectedCandidates`](asap_types::stages::DagOfSelectedCandidates), +//! and realizes it once through the injected realizer. + +use asap_cost::CostInputs; +use asap_types::stages::{AssignmentRealizer, PhysicalASAPMemoCandidates}; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; -use crate::groups::{prepare_milp_inputs_with, MilpInputs}; -use crate::milp::decode::{decode, ChosenCandidates}; -use crate::milp::model::{ - build_selection_milp, solve_selection_milp, SelectionMilpError, SelectionMilpOptions, -}; -use crate::models::{AccuracyCheckInputs, SelectionModels}; -use crate::report::{ - ErrorSelectingPlan, InexactObjective, Rejection, RejectionCause, SearchReport, SearchStatus, - Selection, -}; - -/// The largest relative difference between the objective and `plan_cost` -/// that counts as equal. -const COST_EQUALITY_TOLERANCE: f64 = 1e-6; +use crate::models::AccuracyCheckInputs; +use crate::report::{ErrorSelectingPlan, Selection}; /// Stage 3, plan selection: select one plan among `physical`'s candidates -/// with the group selection MILP (module documentation). The selection is -/// realized through `realizer`, checked against every query's accuracy -/// target (`accuracy_checks`) and latency bound, and priced under -/// `cost_inputs`. `entries[q]` is query `q`'s workload entry (accuracy -/// target, latency bound, recurrence); `data` the data workload with its -/// bounds. The solve has no time limit: it runs to proven optimality. +/// (module documentation). The selection is realized through `realizer`, +/// checked against every query's accuracy target (`accuracy_checks`) and +/// priced under `cost_inputs`. `entries[q]` is query `q`'s workload entry; +/// `data` the data workload with its bounds. /// /// Err: invalid or unsupported cost inputs, an infeasible catalog, a /// solver failure, or the realizer rejected the selected plan @@ -73,301 +31,6 @@ pub fn select_plan( cost_inputs .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; - // The MILP's coefficients and `plan_cost` must use one calibration: a - // measured cost table needs a frozen factor. - if cost_inputs.frozen_calibration_factor.is_none() - && !cost_inputs.measured_operation_costs.operators.is_empty() - { - return Err(ErrorSelectingPlan::UnsupportedCostModel( - "a measured operation cost table without a frozen calibration factor".into(), - )); - } - let formula_scale = cost_inputs.frozen_calibration_factor.unwrap_or(1.0); - let models = SelectionModels::new(accuracy_checks, cost_inputs); - // The A1 rows decide accuracy only when the built-in rule does: a model - // or evidence that may accept an output otherwise is asked after the - // solve. - let enforce_accuracy = !(accuracy_checks.model.may_accept_regardless_of_guarantee() - || accuracy_checks.evidence.may_provide_evidence()); - - let construction_started = Instant::now(); - let inputs = prepare_milp_inputs_with(physical, entries, data, &models); - let pruned_candidates = pruned_before_the_milp(physical, &inputs); - let milp = build_selection_milp( - physical, - &inputs, - entries, - cost_inputs, - SelectionMilpOptions { - enforce_accuracy, - ..SelectionMilpOptions::default() - }, - ) - .map_err(selection_error)?; - let build_ms = construction_started.elapsed().as_millis() as u64; - let solution = solve_selection_milp(milp, build_ms).map_err(|error| { - with_uncostable_reasons(selection_error(error), &inputs, &pruned_candidates) - })?; - - let validation_started = Instant::now(); - let chosen = ChosenCandidates { - chosen: solution.chosen.clone(), - selected: solution.selected.clone(), - slot_reads: solution.slot_reads.clone(), - }; - let selected_dag = decode(physical, &inputs, &chosen).map_err(|reason| { - ErrorSelectingPlan::Solver(format!("the selection does not decode: {reason}")) - })?; - selected_dag.validate(physical).map_err(|reason| { - ErrorSelectingPlan::Solver(format!("the decoded selection is invalid: {reason}")) - })?; - let realized = realize(physical, &selected_dag, realizer)?; - let plan = &realized.plan; - let recurrence: Vec<_> = entries.iter().map(|e| e.recurrence.clone()).collect(); - let ranges: Vec<_> = entries.iter().map(|e| e.range_evaluation).collect(); - let (cost, _) = plan_cost( - &plan.dag, - &node_map(&plan.dag), - &kept_tumbling_window_chains(&plan.dag), - &recurrence, - &ranges, - data, - cost_inputs, - formula_scale, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: plan.id.clone(), - node, - error, - })?; - let plan_cost_per_second = cost.cost_per_second.ok_or_else(|| { - ErrorSelectingPlan::UnsupportedCostModel("a cost model without a cost per second".into()) - })? - solution.request_cost_per_second; - let objective_per_second = solution.cost_per_second; - let difference = (objective_per_second - plan_cost_per_second).abs(); - let tolerance = COST_EQUALITY_TOLERANCE * plan_cost_per_second.abs().max(f64::MIN_POSITIVE); - let within = difference <= tolerance; - report_run(&solution, objective_per_second, plan_cost_per_second); - let inexact_objective = if within { - None - } else if solution.size.inexact.is_empty() { - return Err(ErrorSelectingPlan::ObjectiveDiffersFromPlanCost { - objective_per_second, - plan_cost_per_second, - }); - } else { - Some(InexactObjective { - objective_per_second, - plan_cost_per_second, - inexact_terms: solution - .size - .inexact - .iter() - .map(|(reason, count)| (reason.to_string(), *count)) - .collect(), - }) - }; - let accepted_accuracy_per_query = - accuracy_check(plan, &accuracy_targets(entries), data, &models).map_err(|failure| { - ErrorSelectingPlan::SelectedPlanFailedCheck { - cause: failure.cause, - reason: failure.reason, - } - })?; - if let Some((cause, reason)) = latency_violation(&cost, entries) { - return Err(ErrorSelectingPlan::SelectedPlanFailedCheck { cause, reason }); - } - let nodes_with_proven_non_negative_weights = proven_non_negative_weights(&realized, data); - let validation_ms = validation_started.elapsed().as_millis() as u64; - Ok(Selection { - selected_dag, - nodes_with_proven_non_negative_weights, - cost, - accepted_accuracy_per_query, - pruned_candidates, - search_report: SearchReport { - status: SearchStatus::Optimal, - // The dual bound is on the objective, which leaves out the - // request fees every plan pays; the selection's cost has them. - cost_lower_bound_per_second: solution - .cost_lower_bound_per_second - .is_finite() - .then_some(solution.cost_lower_bound_per_second + solution.request_cost_per_second), - milp_variables: solution.size.variables, - milp_constraints: solution.size.rows, - model_construction_ms: solution.build_ms + solution.warm_start_ms, - solver_ms: solution.solve_ms, - validation_ms, - inexact_objective, - }, - }) -} - -/// The selection MILP's error as Stage 3's. -fn selection_error(error: SelectionMilpError) -> ErrorSelectingPlan { - let infeasible = |id: String, cause: RejectionCause, reason: String| { - ErrorSelectingPlan::NoValidCandidate(vec![Rejection { id, cause, reason }]) - }; - match error { - SelectionMilpError::UnsupportedCostModel(model) => { - ErrorSelectingPlan::UnsupportedCostModel(model.to_string()) - } - SelectionMilpError::EntryCount { roots, entries } => ErrorSelectingPlan::EntryCount { - candidate: "the physical memo".into(), - roots, - entries, - }, - SelectionMilpError::Infeasible => infeasible( - "workload".into(), - RejectionCause::Infeasible, - "no plan meets every query's accuracy target, latency bound and the reader \ - constraints together (the selection MILP is infeasible)" - .into(), - ), - SelectionMilpError::Cost(reason) | SelectionMilpError::Solver(reason) => { - ErrorSelectingPlan::Solver(reason) - } - } -} - -/// An infeasible selection with the reasons `asap-cost` could not cost -/// the candidates the model left out (e.g. a source missing the fact its -/// pricing needs), each once in candidate order: those candidates may be -/// why no plan is left. -fn with_uncostable_reasons( - error: ErrorSelectingPlan, - inputs: &MilpInputs, - pruned_candidates: &[PrunedCandidate], -) -> ErrorSelectingPlan { - let ErrorSelectingPlan::NoValidCandidate(mut rejections) = error else { - return error; - }; - let mut reasons: Vec<&str> = Vec::new(); - for pruned in pruned_candidates { - if pruned.reason != PruneReason::UsageNotCostable { - continue; - } - if let Err(reason) = &inputs.candidate_data[pruned.candidate.0].usage { - if !reasons.contains(&reason.as_str()) { - reasons.push(reason); - } - } - } - if let Some(rejection) = rejections.first_mut() { - if !reasons.is_empty() { - rejection.reason = format!( - "{}; candidates left out because they could not be costed: {}", - rejection.reason, - reasons.join("; ") - ); - } - } - ErrorSelectingPlan::NoValidCandidate(rejections) -} - -/// What Stage 3 removes before the MILP: dominated candidates, and those -/// `asap-cost` could not cost (the model leaves them out), in candidate -/// order. -fn pruned_before_the_milp( - physical: &PhysicalASAPMemoCandidates, - inputs: &MilpInputs, -) -> Vec> { - let mut reasons: BTreeMap = inputs - .pruned_candidates - .iter() - .map(|pruned| (pruned.class, PruneReason::DominatedBy(pruned.dominated_by))) - .collect(); - for (id, data) in inputs.candidate_data.iter().enumerate() { - if data.usage.is_err() { - reasons - .entry(PhysicalCandidateId(id)) - .or_insert(PruneReason::UsageNotCostable); - } - } - reasons - .into_iter() - .map(|(candidate, reason)| PrunedCandidate { - group: physical.memo_group_of(candidate), - candidate, - reason, - }) - .collect() -} - -/// `selected` realized through `realizer`; a rejection is a Stage 2 -/// catalog bug. -fn realize( - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - realizer: &dyn AssignmentRealizer, -) -> Result { - realizer - .realize(physical, selected) - .map_err(|error| match error { - PhysicalCandidateError::RejectedAssignment(reason) => { - ErrorSelectingPlan::RealizerRejected(reason) - } - error => ErrorSelectingPlan::RealizerRejected(error.to_string()), - }) -} - -/// The selected nodes whose summary (`SummaryAgg` or `SummaryTumblingWindowChain`) reads -/// update weights `data` proves non-negative although the candidate leaves -/// the requirement open. -fn proven_non_negative_weights( - realized: &RealizedPlan, - data: &DataWorkload, -) -> Vec { - realized - .operator_node_of_selected_node - .iter() - .enumerate() - .filter(|(_, node)| match &node.operator { - Operator::ASAP( - ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, - ) => confirmed_weights(&input.weight_domain, data) != input.weight_domain, - _ => false, - }) - .map(|(node, _)| SelectedNodeId(node)) - .collect() -} - -#[cfg(test)] -mod tests; - -/// Appends one JSON line describing this run to `ASAP_STAGE3_REPORT`, if -/// set (measurement only; failures to write are ignored). -fn report_run( - solution: &crate::milp::model::SelectionMilpSolution, - objective_per_second: f64, - plan_cost_per_second: f64, -) { - let Ok(path) = std::env::var("ASAP_STAGE3_REPORT") else { - return; - }; - let line = serde_json::json!({ - "variables": solution.size.variables, - "binaries": solution.size.binaries, - "rows": solution.size.rows, - "nonzeros": solution.size.nonzeros, - "scopes": solution.size.scopes.iter().map(|(scope, size)| (format!("{scope:?}"), size)).collect::>(), - "families": solution.size.families, - "inexact": solution.size.inexact, - "build_ms": solution.build_ms, - "warm_start_ms": solution.warm_start_ms, - "solve_ms": solution.solve_ms, - "warm_start_cost_per_second": solution.warm_start_cost_per_second, - "objective_per_second": objective_per_second, - "plan_cost_per_second": plan_cost_per_second, - "dual_bound_per_second": solution.cost_lower_bound_per_second, - "parts": solution.parts, - }); - use std::io::Write; - if let Ok(mut file) = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path) - { - let _ = writeln!(file, "{line}"); - } + let _ = (physical, entries, data, accuracy_checks, realizer); + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, its decode and realization") } diff --git a/crates/plan-selection/src/select/tests.rs b/crates/plan-selection/src/select/tests.rs deleted file mode 100644 index 68ce48681..000000000 --- a/crates/plan-selection/src/select/tests.rs +++ /dev/null @@ -1,173 +0,0 @@ -use super::*; -use crate::milp::model::tests::{approximate, cost_inputs, shared_sum, through_stages, Fixture}; -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel, NoAccuracyEvidence}; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; -use asap_types::stages::RealizedPlan; - -fn realizer(fixture: &Fixture) -> impl AssignmentRealizer + '_ { - asap_physical_asap_optimization::Stage2AssignmentRealizer { - original_roots: &fixture.roots, - entries: &fixture.entries, - data: &fixture.data, - query_time_results_retained: fixture.query_time_results_retained, - } -} - -fn select( - fixture: &Fixture, - accuracy: &dyn AccuracyModel, - realizer: &dyn AssignmentRealizer, -) -> Result { - select_plan( - &fixture.physical, - &fixture.entries, - &fixture.data, - &AccuracyCheckInputs { - model: accuracy, - evidence: &NoAccuracyEvidence, - }, - &cost_inputs(), - realizer, - ) -} - -/// `fixture` with metric `m` declared non-negative. -fn non_negative(mut fixture: Fixture) -> Fixture { - let source = fixture.data.sources.get_mut("m").expect("m"); - source.values_non_negative = true; - fixture -} - -// The group selection MILP's selection: an optimal, valid plan whose cost is -// `plan_cost`'s, the MILP's size and every query's accepted accuracy. The -// ratio of quantiles is a regression of #1127: the A1 rows admitted two -// KLL sketches at parts = 2, whose rank errors compose no guarantee through -// the quotient, so the search returned `SelectedPlanFailedCheck`. -#[test] -fn the_milp_search_selects_a_checked_plan() { - let ratio = || { - through_stages( - &[approximate( - "quantile_over_time(0.5, m[5m]) / quantile_over_time(0.9, m[5m])", - 60_000, - 0.01, - )], - true, - ) - }; - for fixture in [ - shared_sum(), - through_stages( - &[approximate("quantile_over_time(0.9, m[5m])", 60_000, 0.01)], - true, - ), - ratio(), - non_negative(ratio()), - ] { - let stage2 = realizer(&fixture); - let selection = select(&fixture, &BuiltinAccuracyModel, &stage2).unwrap(); - selection.selected_dag.validate(&fixture.physical).unwrap(); - let report = &selection.search_report; - assert_eq!(report.status, SearchStatus::Optimal); - assert_eq!(report.inexact_objective, None); - assert!(report.milp_variables > 0 && report.milp_constraints > 0); - let cost = selection.cost.cost_per_second.unwrap(); - assert!(report.relative_gap(cost).unwrap() <= 1e-6, "{report:?}"); - assert_eq!( - selection.accepted_accuracy_per_query.len(), - fixture.entries.len() - ); - for pruned in &selection.pruned_candidates { - assert!(matches!( - pruned.reason, - PruneReason::DominatedBy(_) | PruneReason::UsageNotCostable - )); - assert_eq!( - pruned.group, - fixture.physical.memo_group_of(pruned.candidate) - ); - } - } -} - -/// A realizer that rejects every selection. -struct Rejecting; - -impl AssignmentRealizer for Rejecting { - fn realize( - &self, - _physical: &PhysicalASAPMemoCandidates, - _selected: &DagOfSelectedCandidates, - ) -> Result { - Err(PhysicalCandidateError::RejectedAssignment( - "a combination the catalog allows".into(), - )) - } -} - -// A realizer rejection is a Stage 2 catalog bug: an error, no re-solve. -#[test] -fn a_realizer_rejection_is_realizer_rejected() { - let fixture = shared_sum(); - match select(&fixture, &BuiltinAccuracyModel, &Rejecting) { - Err(ErrorSelectingPlan::RealizerRejected(reason)) => { - assert!(reason.contains("the catalog allows"), "{reason}") - } - other => panic!("{other:?}"), - } -} - -/// An accuracy model that accepts no guarantee, exact included, and never -/// accepts an output otherwise: the A1 rows still run, so the MILP's plan -/// misses `satisfies`, as it would with a wrong weight table. -struct AcceptsNothing; - -impl AccuracyModel for AcceptsNothing { - fn satisfies(&self, _guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { - false - } - - fn may_accept_regardless_of_guarantee(&self) -> bool { - false - } -} - -// A selected plan that misses `satisfies` is an error, never returned. -#[test] -fn an_accuracy_miss_is_an_error() { - let fixture = shared_sum(); - let stage2 = realizer(&fixture); - match select(&fixture, &AcceptsNothing, &stage2) { - Err(ErrorSelectingPlan::SelectedPlanFailedCheck { cause, .. }) => { - assert!(cause.failed_a_check(), "{cause:?}") - } - other => panic!("{other:?}"), - } -} - -// Cost models 2 and 3 are not supported by the group selection MILP (#888). -#[test] -fn cost_model_2_is_unsupported() { - let fixture = shared_sum(); - let costs = CostInputs { - cost_model: asap_cost::CostModel::ProvisionedCapacity( - asap_cost::ResourceUnitCost::illustrative(), - ), - ..cost_inputs() - }; - let result = select_plan( - &fixture.physical, - &fixture.entries, - &fixture.data, - &AccuracyCheckInputs { - model: &BuiltinAccuracyModel, - evidence: &NoAccuracyEvidence, - }, - &costs, - &Rejecting, - ); - assert!(matches!( - result, - Err(ErrorSelectingPlan::UnsupportedCostModel(_)) - )); -} diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs deleted file mode 100644 index 5270385cd..000000000 --- a/crates/plan-selection/src/selector.rs +++ /dev/null @@ -1,471 +0,0 @@ -//! Choosing among whole checked plans: [`PlanSelector`] and -//! [`select_among_plans`]. The reference pricer (feature `reference-pricer`) -//! selects among enumerated plans with it. - -use std::collections::BTreeMap; - -use asap_cost::{ - worst_latency_ms, CalibrationFit, ClusterLimits, CostModel, PlanCost, PlanRanking, - ResourceUsage, COST_UNIT, -}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::workload::{DataWorkload, QueryRecurrence, QueryWorkloadEntry, RangeEvaluation}; - -use crate::checks::{accuracy_check, accuracy_targets, latency_violation}; -use crate::report::{ - AcceptedAccuracy, ErrorSelectingPlan, Rejection, RejectionCause, WholePlanSelection, -}; -use crate::SelectionModels; -use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{ - kept_tumbling_window_chains, node_map, plan_cost, query_reach, tumbling_window_retention_ms, -}; - -type Ranking = PlanRanking; -use asap_types::workload::bounds::with_confirmed_weights; - -/// A cost per second for a rejection reason: three decimals, or three -/// significant digits in scientific notation when it is nonzero and below -/// 0.001 (e.g. "2.361e-8" under [`ResourceUnitCost::aws_fargate_us_east_1`]'s -/// dollars), so small costs do not all read "0.000". -fn display_cost(cost_per_second: f64) -> String { - if cost_per_second != 0.0 && cost_per_second.abs() < 1e-3 { - format!("{cost_per_second:.3e}") - } else { - format!("{cost_per_second:.3}") - } -} - -/// Select among whole plans: reject candidates that the executor cannot -/// run or that miss a query's accuracy target or latency bound (or, under -/// cost model 3, the cluster's capacity), measure the rest's resource -/// usage per second and choose one under the deployment's cost model -/// (`models.cost_inputs`): the cheapest under cost models 1 and 2, the -/// best by the cluster objective under cost model 3 (the first on ties). `entries[i]` is the workload -/// entry of `candidate.roots[i]`: its accuracy target, its latency bound and -/// how often it is evaluated. Every candidate is kept in the selection -/// ([`PlanSelector`] with `keep_every_candidate`). -#[cfg(any(test, feature = "reference-pricer"))] -pub(crate) fn select_among_plans( - cands: &[PhysicalWorkloadPlan], - entries: &[QueryWorkloadEntry], - data: &DataWorkload, - models: SelectionModels<'_>, -) -> Result { - let mut selector = PlanSelector::new(entries, data, models, true)?; - if selector.needs_calibration() { - let mut fit = CalibrationFit::default(); - for candidate in cands { - match selector.calibration_fit(candidate) { - // Rejected below when offered. - Err(ErrorSelectingPlan::Cost { - error: AnalyticalCostError::Overflow, - .. - }) => {} - fit_or_error => fit.add(fit_or_error?), - } - } - selector.set_calibration(fit); - } - // An unpriceable candidate is rejected, as the search rejects it - // (`cost_not_representable`), rather than failing the selection. - for candidate in cands { - match selector.offer(candidate) { - Err( - error @ ErrorSelectingPlan::Cost { - error: AnalyticalCostError::Overflow, - .. - }, - ) => selector.reject( - &candidate.id, - RejectionCause::CostNotRepresentable, - error.to_string(), - ), - result => result?, - } - } - selector.finish().map(|(selection, _)| selection) -} - -/// Whole plans offered one at a time, in candidate order -/// ([`select_among_plans`] over all of them at once). Each candidate is -/// checked and priced as it is offered; only the best so far is kept, -/// unless `keep_every_candidate`. Either way it selects what -/// [`select_among_plans`] selects over the same candidates. -pub struct PlanSelector<'a> { - entries: &'a [QueryWorkloadEntry], - data: &'a DataWorkload, - models: SelectionModels<'a>, - targets: Vec, - recurrence: Vec, - range_evaluations: Vec>, - ranking: Ranking, - keep_every_candidate: bool, - candidates_checked: u64, - costs: BTreeMap, - accepted: BTreeMap>, - rejected: Vec, - /// The valid candidates kept, in candidate order: every one, the best - /// so far, or the Pareto front so far. - kept: Vec, - /// The factor that calibrates the formulas' CPU time to the measured - /// table's ([`Self::set_calibration`]); 1 without a measured table. - pub(crate) formula_scale: f64, -} - -impl<'a> PlanSelector<'a> { - /// A selector for the workload `entries` over `data` under `models`. - /// Bad settings are an error here, before any candidate. - pub fn new( - entries: &'a [QueryWorkloadEntry], - data: &'a DataWorkload, - models: SelectionModels<'a>, - keep_every_candidate: bool, - ) -> Result { - models - .cost_inputs - .validate() - .map_err(ErrorSelectingPlan::InvalidSettings)?; - Ok(Self { - entries, - data, - models, - targets: accuracy_targets(entries), - recurrence: entries.iter().map(|e| e.recurrence.clone()).collect(), - range_evaluations: entries.iter().map(|e| e.range_evaluation).collect(), - ranking: Ranking::of(&models.cost_inputs.cost_model), - keep_every_candidate, - candidates_checked: 0, - costs: BTreeMap::new(), - accepted: BTreeMap::new(), - rejected: Vec::new(), - kept: Vec::new(), - formula_scale: models.cost_inputs.frozen_calibration_factor.unwrap_or(1.0), - }) - } - - /// Whether the deployment's measured cost table prices some operator, - /// so the formulas' CPU time is calibrated to it: one factor for every - /// candidate, fitted over the nodes the table prices in every candidate - /// that passes the checks before pricing. The caller then gives every - /// candidate to [`Self::calibration_fit`], adds up the fits and passes - /// the sum to [`Self::set_calibration`] before the first - /// [`Self::offer`]. - pub fn needs_calibration(&self) -> bool { - self.models.cost_inputs.frozen_calibration_factor.is_none() - && !self - .models - .cost_inputs - .measured_operation_costs - .operators - .is_empty() - } - - /// `candidate`'s part of the calibration fit: its measured nodes' CPU - /// time, as measured and as the formulas price it, when it passes the - /// checks before pricing; nothing otherwise. - pub fn calibration_fit( - &self, - candidate: &PhysicalWorkloadPlan, - ) -> Result { - if candidate.roots.len() != self.entries.len() { - return Err(ErrorSelectingPlan::EntryCount { - candidate: candidate.id.clone(), - roots: candidate.roots.len(), - entries: self.entries.len(), - }); - } - if accuracy_check(candidate, &self.targets, self.data, &self.models).is_err() { - return Ok(CalibrationFit::default()); - } - let nodes = node_map(&candidate.dag); - let kept_chains = kept_tumbling_window_chains(&candidate.dag); - let (_, fit) = plan_cost( - &candidate.dag, - &nodes, - &kept_chains, - &self.recurrence, - &self.range_evaluations, - self.data, - self.models.cost_inputs, - 1.0, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; - Ok(fit) - } - - /// Calibrate the formulas with `fit`, the sum of every candidate's - /// [`Self::calibration_fit`]. - pub fn set_calibration(&mut self, fit: CalibrationFit) { - self.formula_scale = self - .models - .cost_inputs - .frozen_calibration_factor - .unwrap_or_else(|| fit.factor()); - } - - /// Check and price `candidate`, the next in candidate order, and keep - /// it if it can still be selected. - #[cfg(any(test, feature = "reference-pricer"))] - pub fn offer(&mut self, candidate: &PhysicalWorkloadPlan) -> Result<(), ErrorSelectingPlan> { - if candidate.roots.len() != self.entries.len() { - return Err(ErrorSelectingPlan::EntryCount { - candidate: candidate.id.clone(), - roots: candidate.roots.len(), - entries: self.entries.len(), - }); - } - self.candidates_checked += 1; - let accepted_accuracy = - match accuracy_check(candidate, &self.targets, self.data, &self.models) { - Ok(accepted) => accepted, - Err(failed) => { - self.reject(&candidate.id, failed.cause, failed.reason); - return Ok(()); - } - }; - let nodes = node_map(&candidate.dag); - let kept_chains = kept_tumbling_window_chains(&candidate.dag); - let settings = self.models.cost_inputs; - let (cost, _) = plan_cost( - &candidate.dag, - &nodes, - &kept_chains, - &self.recurrence, - &self.range_evaluations, - self.data, - settings, - self.formula_scale, - ) - .map_err(|(node, error)| ErrorSelectingPlan::Cost { - candidate: candidate.id.clone(), - node, - error, - })?; - let violation = - latency_violation(&cost, self.entries).or_else(|| match &settings.cost_model { - CostModel::ClusterCapacity(limits) => capacity_violation(&cost.usage, limits) - .map(|reason| (RejectionCause::OverClusterCapacity, reason)), - _ => None, - }); - if let Some((cause, reason)) = violation { - self.reject(&candidate.id, cause, reason); - return Ok(()); - } - if !self.keep_every_candidate { - // Ties keep the earlier candidate. - let selectable = self - .kept - .first() - .is_none_or(|best| self.ranking.better(&cost, &self.costs[&best.id])); - if !selectable { - return Ok(()); - } - } - if !self.keep_every_candidate { - for kept in self.kept.drain(..) { - self.costs.remove(&kept.id); - self.accepted.remove(&kept.id); - } - } - self.costs.insert(candidate.id.clone(), cost); - self.accepted - .insert(candidate.id.clone(), accepted_accuracy); - self.kept.push(candidate.clone()); - Ok(()) - } - - /// The selection over every candidate offered, and the selected - /// candidate as Stage 2 built it (the selection's plan is its roots - /// with the weight requirements Stage 3 confirmed made proofs). - pub fn finish( - mut self, - ) -> Result<(WholePlanSelection, PhysicalWorkloadPlan), ErrorSelectingPlan> { - let ranking = self.ranking; - let valid_costs: Vec<&PlanCost> = - self.kept.iter().map(|kept| &self.costs[&kept.id]).collect(); - let Some(best) = choose(ranking, &valid_costs) else { - return Err(ErrorSelectingPlan::NoValidCandidate(self.rejected)); - }; - let selected = self.kept[best].id.clone(); - for kept in &self.kept { - let id = &kept.id; - if *id != selected { - let (cause, reason) = lost_reason(ranking, id, &selected, &self.costs); - self.rejected.push(Rejection { - id: id.clone(), - cause, - reason, - }); - } - } - let built = self.kept.swap_remove(best); - let plan = with_confirmed_weights(&built.roots, self.data); - let tumbling_window_retention_ms = tumbling_window_retention_ms( - &built.dag, - &query_reach(&built.dag, &node_map(&built.dag)), - &self.range_evaluations, - ); - let selection = WholePlanSelection { - selected, - plan, - costs: self.costs, - accepted: self.accepted, - rejected: self.rejected, - tumbling_window_retention_ms, - candidates_checked: self.candidates_checked, - }; - Ok((selection, built)) - } - - /// Record that candidate `id` failed a check: every rejection when - /// keeping every candidate, otherwise the first of each cause. - fn reject(&mut self, id: &str, cause: RejectionCause, reason: String) { - if self.keep_every_candidate || !self.rejected.iter().any(|r| r.cause == cause) { - self.rejected.push(Rejection { - id: id.to_string(), - cause, - reason, - }); - } - } -} - -/// The index of the candidate `ranking` selects among `costs` (the first on -/// ties); `None` without candidates. -pub(crate) fn choose(ranking: Ranking, costs: &[&PlanCost]) -> Option { - (0..costs.len()).reduce( - |best, next| match ranking.better(costs[next], costs[best]) { - true => next, - false => best, - }, - ) -} - -/// Under cost model 3, the first limit of the cluster `usage` is over, as a -/// rejection reason. -fn capacity_violation(usage: &ResourceUsage, limits: &ClusterLimits) -> Option { - if usage.peak_memory_bytes > limits.memory_bytes { - return Some(format!( - "peak memory {} bytes is over the cluster's {} bytes", - usage.peak_memory_bytes, limits.memory_bytes - )); - } - if usage.disk_bytes > limits.disk_bytes { - return Some(format!( - "disk {} bytes is over the cluster's {} bytes", - usage.disk_bytes, limits.disk_bytes - )); - } - match limits.cpu_cores { - Some(cores) if usage.peak_cpu_cores > cores => Some(format!( - "{} CPU cores at peak is over the cluster's {cores}", - usage.peak_cpu_cores - )), - _ => None, - } -} - -/// Why valid candidate `id` was not `selected` under `ranking`, and the -/// reason with its numbers. -fn lost_reason( - ranking: Ranking, - id: &str, - selected: &str, - costs: &BTreeMap, -) -> (RejectionCause, String) { - let (cost, best) = (&costs[id], &costs[selected]); - if !ranking.better(best, cost) { - let tied = match ranking { - Ranking::Price => format!( - "{} {COST_UNIT}", - display_cost(cost.cost_per_second.unwrap_or_default()) - ), - _ => describe_usage(&cost.usage), - }; - return ( - RejectionCause::TiedLostOnEnumerationOrder, - format!( - "tied with the selected candidate {selected} at {tied}; lost on enumeration order" - ), - ); - } - match ranking { - Ranking::Price => ( - RejectionCause::Costlier, - format!( - "costlier: {} vs {} {COST_UNIT}", - display_cost(cost.cost_per_second.unwrap_or_default()), - display_cost(best.cost_per_second.unwrap_or_default()) - ), - ), - Ranking::Latency => ( - RejectionCause::Slower, - format!( - "slower: {} vs {}", - describe_usage(&cost.usage), - describe_usage(&best.usage) - ), - ), - Ranking::Memory => ( - RejectionCause::MoreMemory, - format!( - "more memory: {} vs {}", - describe_usage(&cost.usage), - describe_usage(&best.usage) - ), - ), - } -} - -/// E.g. `latency 126.0 ms, peak memory 9648000000 bytes, disk 480000000 bytes`. -fn describe_usage(usage: &ResourceUsage) -> String { - format!( - "latency {:.1} ms, peak memory {} bytes, disk {} bytes", - worst_latency_ms(usage), - usage.peak_memory_bytes, - usage.disk_bytes - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_cost::{QueryLatency, COST_SOURCE}; - - /// A candidate with the given latency, peak memory and disk under cost - /// model 3. - fn cluster_cost(latency_ms: f64, peak_memory_bytes: u64, disk_bytes: u64) -> PlanCost { - PlanCost { - usage: ResourceUsage { - peak_memory_bytes, - disk_bytes, - latency_per_query: vec![QueryLatency { - compaction_ms: 0.0, - query_ms: latency_ms, - }], - ..ResourceUsage::default() - }, - cost_per_second: None, - cost_unit: COST_UNIT, - source: COST_SOURCE, - per_node: BTreeMap::new(), - } - } - - /// Peak memory and disk are compared exactly: 100 bytes less at 124 GB - /// is less memory, not a tie. - #[test] - fn memory_differing_by_a_few_bytes_is_compared_exactly() { - let larger = cluster_cost(7.0, 124_000_000_100, 0); - let smaller = cluster_cost(7.0, 124_000_000_000, 0); - let costs = [&larger, &smaller]; - assert_eq!(choose(Ranking::Memory, &costs).unwrap(), 1); - assert_eq!(choose(Ranking::Latency, &costs).unwrap(), 1); - } -} diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 80589f63a..63802bcf6 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -31,17 +31,9 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" -[features] -# `test_support::query_time_reference`, for a runtime's correctness tests. -# It reads `asap-types`' test-only DAG helpers. -test-support = ["asap-types/test-support"] - [dev-dependencies] -# The reference pricer over fixed candidate plans. -asap-plan-selection = { path = "../plan-selection", features = ["reference-pricer"] } -# The tests compare installed plans with `test_support::query_time_reference`. -asap-planner = { path = ".", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization", features = ["test-support"] } +asap-plan-selection = { path = "../plan-selection" } +asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } # The deployment accuracy model tests implement `AccuracyModel`. diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index ea836d674..68e827cac 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -23,8 +23,7 @@ //! costing and display) is not the runtime's concern: the installable DAG //! has every chain expanded into the tumbling windows the executor runs. -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::rc::Rc; +use std::collections::{HashMap, HashSet}; use asap_cost::{NodePriceSource, OperatorKind}; use asap_logical_asap_optimization::DeploymentCapabilities; @@ -32,17 +31,15 @@ use asap_plan_selection::{SearchReport, Selection}; use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, }; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; + use asap_types::ir::physical_export::{ PhysicalASAPDAG, PhysicalASAPDAGEdge, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, - PhysicalASAPNodeId, ReferencePlanError, + PhysicalASAPNodeId, }; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::SchemaDerivationError; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, OperatorNode, - QueryRoot, -}; + +use asap_types::stages::RealizedPlan; use asap_types::workload::QueryWorkloadEntry; use serde::{Deserialize, Serialize}; @@ -50,7 +47,7 @@ use crate::{e2e_plan, PlanError, UserInput}; /// The version of the plan-file format [`InstallablePlan`] serializes to: /// the DAG, its nodes' fields and the query texts. -pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 11; +pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 13; /// The selected plan, ready to install and execute: what /// [`plan_installable`] returns and a runtime writes as its plan file. @@ -156,11 +153,6 @@ pub struct PredictedQueryUsage { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum InstallablePlanError { - /// Internal bug: the selected plan's timings do not hold, or it does - /// not export; or, for the query-time reference, bad input: the plan - /// cannot run at query time. - #[error("timing: {0}")] - Timing(#[from] ExecutionDataStateError), /// Internal bug: a tumbling-window chain of the selected plan does not expand. #[error("tumbling-window chain expansion: {0}")] TumblingWindowChainExpansion(#[from] SchemaDerivationError), @@ -216,15 +208,6 @@ impl From for InstallablePlanError { } } -impl From for InstallablePlanError { - fn from(error: ReferencePlanError) -> Self { - match error { - ReferencePlanError::InvalidDag(e) => Self::InvalidDag(e), - ReferencePlanError::Timing(e) => Self::Timing(e), - } - } -} - /// Plan `input` for a deployment runtime: the query-language frontends and /// Stages 1-3 as [`e2e_plan`] runs them, returning the single selected DAG /// ready to install, and beside it Stage 3's report of the selection @@ -250,27 +233,20 @@ pub async fn plan_installable( pass: input.pass, }; let output = e2e_plan(input).await?; - installable_plan_of( - &output.roots(), - output.selection.as_ref(), - &output.tumbling_window_retention_ms, - &entries, - ) - .map_err(PlanError::Installable) + let _ = (output, entries); + todo!("#1207 PR 6: install the plan the realizer built from the selected DAG") } -/// The installable plan of `roots`, the selected roots of a workload whose -/// entries are `entries` (in `QueryWorkload::entries()` order), with -/// Stage 3's report of its `selection` of them when there is one, and the -/// tumbling-window retention of the selected plan's kept chains -/// (`RealizedPlan::tumbling_window_retention_ms`, by node id in the DAG `roots` -/// export): what [`plan_installable`] returns after planning. A caller that -/// runs the planning stages itself (e.g. a developer tool) uses it to see -/// what a runtime would install. +/// The installable plan of `realized`, the plan the realizer +/// (`AssignmentRealizer::realize`) built from the selected DAG of a +/// workload whose entries are `entries` (in `QueryWorkload::entries()` +/// order), with Stage 3's report of its `selection` when there is one: what +/// [`plan_installable`] returns after planning. A caller that runs the +/// planning stages itself (e.g. a developer tool) uses it to see what a +/// runtime would install. pub fn installable_plan_of( - roots: &[QueryRoot], + realized: &RealizedPlan, selection: Option<&Selection>, - tumbling_window_retention_ms: &BTreeMap, entries: &[QueryWorkloadEntry], ) -> Result<(InstallablePlan, Option), InstallablePlanError> { // Each query's longest range-evaluation span, which the views it @@ -280,13 +256,7 @@ pub fn installable_plan_of( .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) .collect(); let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - // Every node keeps the timing Stage 3 gave it. - let timed = - apply_materialization_timings_to_roots(roots, &MaterializationAssignment::default())?; - let expanded = tumbling_window_retention_of_expanded_tumbling_windows( - &timed, - tumbling_window_retention_ms, - )?; + let expanded = tumbling_window_retention_of_expanded_tumbling_windows(realized)?; let report = selection.map(|selection| report_of(selection, &expanded.installed_node_ids)); let reader_span_ms_by_node = reader_span_ms_by_node(&expanded.dag, &range_spans_ms); let plan = InstallablePlan::of_dag( @@ -640,98 +610,20 @@ fn executor_refreshes( struct ExpandedWithRetention { dag: PhysicalASAPDAG, retention_ms_by_node: HashMap, - /// Each node id of the selected candidate's physical DAG (`timed` + /// Each node id of the selected candidate's physical DAG (its roots /// exported, Stage 3's ids) with the ids in `dag` that run it /// ([`PredictedNodeUsage::installed_node_ids`]). installed_node_ids: HashMap>, } -/// Expand the tumbling-window chains of `timed` and export it; each tumbling window takes the -/// retention Stage 2 gives its chain (`RealizedPlan::tumbling_window_retention_ms`, by -/// node id in the selected candidate's physical DAG, which is `timed` -/// exported). +/// The realized plan's DAG with its tumbling-window chains expanded; each +/// tumbling window takes the retention Stage 2 gives its chain +/// (`RealizedPlan::tumbling_window_retention_ms`). fn tumbling_window_retention_of_expanded_tumbling_windows( - timed: &[QueryRoot], - tumbling_window_retention_ms: &BTreeMap, + realized: &RealizedPlan, ) -> Result { - use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let selected = compile_physical_asap_workload_with_node_ids(timed)?; - // Reject unsupported raw-source shapes before expanding possibly thousands - // of tumbling windows. The complete expanded-plan validation remains authoritative. - selected.dag.check_raw_data_windows()?; - let mut retention_by_chain: Vec<(Rc, u64)> = Vec::new(); - for (&node_id, &retention_ms) in tumbling_window_retention_ms { - let chain = selected - .node_ids - .operator_node(node_id) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - .ok_or(InstallablePlanError::TumblingWindowRetentionOfUnknownChain { node_id })?; - retention_by_chain.push((Rc::clone(chain), retention_ms)); - } - let expansion = expand_tumbling_window_chains_in_workload(timed) - .map_err(InstallablePlanError::TumblingWindowChainExpansion)?; - let compiled = compile_physical_asap_workload_with_node_ids(&expansion.roots)?; - // Index physical identity once. Looking up every expanded tumbling window through - // the identity map's linear search makes this phase quadratic in tumbling windows. - let mut installed_ids_by_pointer = HashMap::with_capacity(compiled.dag.nodes.len()); - for node in &compiled.dag.nodes { - if let Some(operator) = compiled.node_ids.operator_node(node.id) { - installed_ids_by_pointer.insert(Rc::as_ptr(operator), node.id); - } - } - let installed_id = - |node: &Rc| installed_ids_by_pointer.get(&Rc::as_ptr(node)).copied(); - let mut retention_ms_by_node = HashMap::new(); - for expanded in &expansion.chains { - let Some((_, retention_ms)) = retention_by_chain - .iter() - .find(|(chain, _)| Rc::ptr_eq(chain, &expanded.chain)) - else { - continue; - }; - for tumbling_window in &expanded.tumbling_windows { - if let Some(node_id) = installed_id(tumbling_window) { - retention_ms_by_node.insert(node_id, *retention_ms); - } - } - } - // A chain runs as its tumbling windows; every other node as the node replacing it - // (itself when nothing below it changed). - let mut installed_node_ids = HashMap::new(); - for node in &selected.dag.nodes { - let Some(input) = selected.node_ids.operator_node(node.id) else { - continue; - }; - let ids: Vec = match expansion - .chains - .iter() - .find(|expanded| Rc::ptr_eq(&expanded.chain, input)) - { - Some(expanded) => { - let mut ids = Vec::with_capacity(expanded.tumbling_windows.len()); - for tumbling_window in &expanded.tumbling_windows { - if let Some(id) = installed_id(tumbling_window) { - ids.push(id); - } - } - ids - } - None => { - let replacement = expansion - .expanded_nodes - .iter() - .find(|(original, _)| Rc::ptr_eq(original, input)) - .map_or(input, |(_, replacement)| replacement); - installed_id(replacement).into_iter().collect() - } - }; - installed_node_ids.insert(node.id, ids); - } - Ok(ExpandedWithRetention { - dag: compiled.dag, - retention_ms_by_node, - installed_node_ids, - }) + let _ = realized; + todo!("#1207 PR 6: expand the realized DAG's tumbling-window chains") } /// Stage 3's record of its selected plan. @@ -792,8 +684,6 @@ fn report_of( mod tests { use super::*; use asap_types::ir::schema::{SketchAlgorithm, StatisticKind, SupportedSummary}; - use asap_types::ir::{NonASAPOp, Operator}; - use asap_types::physical::Materialization; thread_local! { /// The plan nodes handed to the executor's compiler by the views' @@ -859,200 +749,6 @@ mod tests { ); } - /// `TimeShift(10 min) <- TimeRange(5 min) <- Scan`, all at query time: - /// the shift above the range moves its window, which a window of a - /// fixed offset below the range cannot say, so it is refused; without - /// the shift the window is listed. - #[test] - fn a_time_shift_above_the_range_is_refused() { - use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::Source; - use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{ - apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency_ms".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(300), - kind: TimeRangeKind::Range, - child: scan, - })) - .unwrap(); - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: 600_000, - at: None, - }, - child: std::rc::Rc::clone(&range), - })) - .unwrap(); - let export = |root| { - let timed = apply_materialization_timings( - &root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() - }; - let install = |dag| { - InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) - }; - match install(export(shifted)) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::UnsupportedRawDataWindow { reason, .. }, - )) => { - assert_eq!(reason, "a subquery or a TimeShift above the range") - } - other => panic!("{other:?}"), - } - let plan = install(export(range)).unwrap(); - assert!(plan.materialized_views().unwrap().is_empty()); - assert_eq!( - plan.raw_data_retention().unwrap(), - [RawDataRetention::new( - Source::TimeSeries { - metric: "latency_ms".into() - }, - Some(300_000) - )] - ); - } - - // A summary stored at ingestion time over a window ending at the - // evaluation and one ending a minute before it cannot be built in one - // ingestion run, so the plan is refused, not built wrong. - #[test] - fn an_ingestion_build_of_windows_at_different_offsets_is_refused() { - use asap_types::ir::operator::operator_properties::TimeShift; - use asap_types::ir::operator::{Reduction, Source}; - use asap_types::ir::physical_export::compile_physical_asap_dag; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SummaryUpdate, - }; - use asap_types::ir::{ - apply_materialization_timings, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; - use asap_types::physical::Medium; - use std::time::Duration; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency_ms".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - // A one-minute window ending `offset_ms` before the evaluation. - let window = |offset_ms: i64| { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child: Rc::clone(&scan), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() - }; - // One sum over both windows' rows. - let both = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { - children: vec![window(0), window(60_000)], - })) - .unwrap(); - let merged = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: both, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&merged, Materialization::MatAtIngestion(Medium::Memory)); - let timed = - apply_materialization_timings(&merged, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { - end_offsets_ms, - .. - }, - )) => assert_eq!(end_offsets_ms, [0, 60_000]), - other => panic!("{other:?}"), - } - // A merge stored at ingestion time over two tumbling windows stored at - // ingestion time, one a minute older: an ingestion run would read - // both tumbling windows for the window ending at its end, so it is refused too. - let tumbling_window = |offset_ms: i64| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: window(offset_ms), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap() - }; - let tumbling_windows = [tumbling_window(0), tumbling_window(60_000)]; - let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.to_vec(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for node in tumbling_windows.iter().chain([&merge]) { - assignment.set(node, Materialization::MatAtIngestion(Medium::Memory)); - } - let timed = - apply_materialization_timings(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { - Err(InstallablePlanError::Uninstallable( - InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { - end_offsets_ms, - .. - }, - )) => assert_eq!(end_offsets_ms, [0, 60_000]), - other => panic!("{other:?}"), - } - } - /// Unrestricted summaries become the executor's families; a caller's /// own list and its retention flags are kept. #[test] diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index ffefd3f12..3aa06b879 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -32,8 +32,8 @@ pub use pass::{ PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, }; pub use run::{ - run_planning, selected_chosen_candidates, PlanSearchSettings, PlanningReport, PlanningRun, - SelectionError, Stage1ToStage3Pass, + run_planning, PlanSearchSettings, PlanningReport, PlanningRun, SelectionError, + Stage1ToStage3Pass, }; mod installable; @@ -43,32 +43,6 @@ pub use installable::{ INSTALLABLE_PLAN_WIRE_VERSION, }; -/// Helpers for a runtime's correctness tests (feature `test-support`). -#[cfg(any(test, feature = "test-support"))] -pub mod test_support { - use std::collections::HashMap; - - use crate::{InstallablePlan, InstallablePlanError}; - - /// For correctness tests only, never for installing: `plan` with - /// nothing materialized, the same operators, summaries and parameters - /// with every node at query time, and the same query texts. A runtime's - /// test compares its installed plan's answers with the executor running - /// this one over raw data samples. Validates `plan` first. - pub fn query_time_reference( - plan: &InstallablePlan, - ) -> Result { - plan.validate()?; - let dag = plan.dag.with_every_node_at_query_time()?; - InstallablePlan::of_dag( - dag, - plan.query_texts.clone(), - &HashMap::new(), - &HashMap::new(), - ) - } -} - // What a deployment runtime needs to build a `UserInput` and read an // `InstallablePlan`, so it depends on this crate and `asap-executor` alone. pub use asap_cost::{ diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 9d699cfc9..07e6596fd 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -16,11 +16,6 @@ use std::collections::BTreeMap; use std::rc::Rc; -use asap_types::ir::physical_export::compile_physical_asap_workload; -use asap_types::ir::properties::timing::{ - apply_materialization_timings_to_roots, MaterializationAssignment, -}; -use asap_types::ir::properties::ExecutionDataStateError; use asap_types::ir::OperatorNode; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; @@ -298,32 +293,6 @@ impl PlanOutput { nodes } - /// The workload as one physical ASAP DAG, with shared sub-DAGs exported - /// once. A node keeps the timing and materialization its pass gave it: - /// for [`Stage1ToStage3Pass`] that is Stage 3's selected plan as - /// selected (e.g. ingestion-time tumbling windows), the document a deployment - /// runtime executes. Only a node the pass left untimed takes the default - /// [`MaterializationAssignment`] (not materialized, computed when its - /// consumer runs). Operator queries become its `roots` and standalone scalar - /// queries its `scalar_roots`, keyed by position in the batch. That - /// position is the entry index, because `check_contract` requires dense - /// entry indices. - /// - /// An empty workload is an error - /// ([`PhysicalASAPDAGValidationError::NoRoots`](asap_types::ir::physical_export::PhysicalASAPDAGValidationError::NoRoots)). - /// A workload of only literal scalars yields a DAG with no nodes: just - /// its `scalar_roots`. - pub fn physical_asap_dag( - &self, - ) -> Result { - // A node several roots share by `Rc` is timed and exported once. - let timed = apply_materialization_timings_to_roots( - &self.roots(), - &MaterializationAssignment::default(), - )?; - compile_physical_asap_workload(&timed) - } - pub fn len(&self) -> usize { self.plans.len() + self.scalar_roots.len() } @@ -629,28 +598,6 @@ mod tests { assert!(check_contract(&output, "stub", 0).is_ok()); } - /// An empty workload has no DAG: the export's `NoRoots`, wrapped in - /// `InvalidPhysicalDAG`. A workload of only literal scalars exports a DAG - /// with no nodes, just its `scalar_roots`. - #[test] - fn physical_asap_dag_of_an_empty_and_a_literal_only_workload() { - use asap_types::ir::physical_export::PhysicalASAPDAGValidationError; - use asap_types::ir::ScalarExpr; - assert_eq!( - PlanOutput::new(Vec::new()).physical_asap_dag().err(), - Some(ExecutionDataStateError::InvalidPhysicalDAG( - PhysicalASAPDAGValidationError::NoRoots - )) - ); - let mut literal = PlanOutput::new(Vec::new()); - literal.scalar_roots = vec![(0, ScalarExpr::literal_f64(1.0))]; - let dag = literal.physical_asap_dag().unwrap(); - assert!(dag.nodes.is_empty() && dag.roots.is_empty()); - let [(0, ScalarExpr::Literal(_))] = dag.scalar_roots.as_slice() else { - panic!("one literal scalar root: {:?}", dag.scalar_roots) - }; - } - /// Registering a name twice fails instead of overwriting, so a comparison /// run cannot silently measure one pass twice. #[test] diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 5069ca1de..3f919230c 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -3,19 +3,15 @@ //! finally the selected plan built through Stage 2's realizer. No stage //! calls another, and none has a deadline: each runs to completion. -use asap_logical_asap_optimization::generate_dag_of_logical_memo_groups; -use asap_types::stages::PerComputationLogicalMemoCandidates; +use asap_logical_asap_optimization::memo_of_queries; + use std::collections::HashSet; use std::rc::Rc; use std::time::Instant; use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; -use asap_logical_asap_optimization::ChosenCandidatePerMemoGroup; use asap_logical_asap_optimization::DeploymentCapabilitiesError; -use asap_logical_asap_optimization::{ - generate_per_computation_logical_memo_candidates, PerComputationCandidateError, - PerComputationLogicalMemoCandidatesError, -}; +use asap_logical_asap_optimization::{pass1_memo, Pass1Error, PerComputationCandidateError}; use asap_physical_asap_optimization::{ generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, }; @@ -23,8 +19,7 @@ use asap_plan_selection::{select_plan, ErrorSelectingPlan, SearchReport, Selecti use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::stages::{ - AssignmentRealizer, CandidateForMemoGroupId, DagOfSelectedCandidates, - LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, + AssignmentRealizer, LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates, PhysicalCandidateError, RealizedPlan, }; use asap_types::workload::bounds::with_confirmed_weights_at; @@ -38,7 +33,7 @@ use crate::pass::{ #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct PlanSearchSettings { /// Keep Pass 1's output for inspection (`asap_planner`, the DAG - /// viewer): [`PlanningRun::per_computation_candidates`]. Default off: + /// viewer): [`PlanningRun::pass1_memo`]. Default off: /// production planning copies no memo. pub inspect: bool, } @@ -76,13 +71,8 @@ pub struct PlanningReport { /// Pass 2's output: memo groups and their candidates. pub memo_groups: usize, pub logical_candidates: usize, - /// Stage 2's output: operator nodes, physical candidates and pruned - /// candidates. - pub operator_nodes: usize, + /// Stage 2's output: physical candidates and pruned candidates. pub physical_candidates: usize, - /// Stage 2's producer links: over every physical candidate and input, - /// the candidates that may produce it. - pub physical_producer_links: usize, pub stage2_pruned_candidates: usize, pub pass2_ms: f64, pub stage2_ms: f64, @@ -97,7 +87,7 @@ pub struct PlanningRun { /// Stage 1, Pass 1's output: the memo groups with their per-computation /// candidates and the pruned candidates. /// `None` when Pass 1 failed or without [`PlanSearchSettings::inspect`]. - pub per_computation_candidates: Option, + pub pass1_memo: Option, /// Pass 2's output, when it ran. pub logical_asap_candidates: Option, /// Stage 2's output, when it ran: the catalog `selection`'s candidate @@ -128,7 +118,7 @@ pub fn run_planning( root.validate_structure()?; } // Stage 0: the memo of logical groups. - let initial = generate_dag_of_logical_memo_groups(&roots); + let initial = memo_of_queries(&roots); // Pass 1: the per-computation candidates the deployment can run and // some query can use. let accuracy_targets: Vec<_> = entries @@ -136,7 +126,7 @@ pub fn run_planning( .map(|entry| entry.requirements.accuracy.clone()) .collect(); let accuracy = models.accuracy_check_inputs(); - let pass1 = generate_per_computation_logical_memo_candidates( + let pass1 = pass1_memo( initial, &accuracy_targets, accuracy.model, @@ -149,14 +139,14 @@ pub fn run_planning( ); // A query Pass 1 cannot plan is an input error. let pass1 = match pass1 { - Err(PerComputationLogicalMemoCandidatesError::PerComputation(error)) => return Err(error), + Err(Pass1Error::PerComputation(error)) => return Err(error), pass1 => pass1, }; let mut report = PlanningReport { pass1_ms: elapsed_ms(started), ..PlanningReport::default() }; - let per_computation_candidates = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); + let pass1_memo = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); let realizer = Stage2AssignmentRealizer { original_roots: &roots, entries, @@ -183,7 +173,12 @@ pub fn run_planning( logical.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?, ); report.memo_groups = logical.memo.groups.len(); - report.logical_candidates = logical.memo.candidates.iter().map(Vec::len).sum(); + report.logical_candidates = logical + .memo + .groups + .iter() + .map(|group| group.candidates.len()) + .sum(); // Stage 2. let started = Instant::now(); let physical = generate_physical_asap_memo_candidates( @@ -197,14 +192,7 @@ pub fn run_planning( let physical = physical_asap_candidates.insert( physical.map_err(|error| SelectionError::PhysicalASAPCandidates(error.to_string()))?, ); - report.operator_nodes = physical.operator_nodes_of_candidates_for_memo_groups.len(); report.physical_candidates = physical.candidates.len(); - report.physical_producer_links = physical - .candidates - .iter() - .flat_map(|candidate| &candidate.producer_candidates_per_input) - .map(|producers| producers.len()) - .sum(); report.stage2_pruned_candidates = physical.pruned_candidates.len(); // Stage 3. let started = Instant::now(); @@ -238,7 +226,7 @@ pub fn run_planning( Ok(selection) })(); Ok(PlanningRun { - per_computation_candidates, + pass1_memo, logical_asap_candidates, physical_asap_candidates, selection, @@ -252,82 +240,6 @@ fn elapsed_ms(started: Instant) -> f64 { started.elapsed().as_secs_f64() * 1000.0 } -/// The selected plan's memo candidate for each group it uses: the memo -/// candidate whose root each selected node's candidate is. Factoring may give several -/// memo candidates of a group one root (identical terminal readers of -/// different inputs); the one reading the selected producers' groups is taken. -/// `memo` and `physical` are the run's -/// ([`PlanningRun::logical_asap_candidates`]'s memo and -/// [`PlanningRun::physical_asap_candidates`]); `selected` is the selection's -/// `selected_dag`. An error says why the plan is not one candidate per group. -pub fn selected_chosen_candidates( - memo: &Memo, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, -) -> Result { - use std::collections::HashMap; - let mut producer_groups = HashMap::<_, Vec>::new(); - for edge in &selected.edges { - producer_groups - .entry(edge.consumer) - .or_default() - .push(physical.memo_group_of(selected.candidate_of_node[edge.producer.0])); - } - let mut chosen_candidates = ChosenCandidatePerMemoGroup::new(); - for (node_id, &id) in selected.candidate_of_node.iter().enumerate() { - let node_id = asap_types::stages::SelectedNodeId(node_id); - let node = &physical.operator_nodes_of_candidates_for_memo_groups - [physical.candidates[id.0].operator_node.0]; - if !node.is_implementation_root { - continue; // a candidate's private node - } - let options = &node.candidates_for_memo_group; - let reads = |&&CandidateForMemoGroupId { group, index }: &&CandidateForMemoGroupId| { - producer_groups.get(&node_id).is_none_or(|read| { - read.iter().all(|&child| { - child == group - || memo.candidates[group.0 as usize][index] - .input_groups - .contains(&child) - }) - }) - }; - let &CandidateForMemoGroupId { group, index } = - options.iter().find(reads).ok_or_else(|| { - format!( - "selected candidate {} reads groups none of its memo candidates read", - id.0 - ) - })?; - if let Some(previous) = chosen_candidates.insert(group, index) { - if previous != index { - return Err(format!( - "the selected plan implements group {} with two candidates, {previous} and {index}", - group.0 - )); - } - } - } - // Every group the chosen candidates need, from the query roots, is chosen. - let mut pending: Vec<_> = asap_types::stages::root_groups(&memo.query_roots) - .into_iter() - .collect(); - let mut seen = HashSet::new(); - while let Some(group) = pending.pop() { - if !seen.insert(group) { - continue; - } - let index = chosen_candidates.get(&group).ok_or_else(|| { - format!( - "the selected plan uses group {} but no candidate of it is selected", - group.0 - ) - })?; - pending.extend(&memo.candidates[group.0 as usize][*index].input_groups); - } - Ok(chosen_candidates) -} - /// The planner pipeline behind the optimization-pass interface: the /// workload's roots in entry order, PromQL roots with each series' full /// identity as a column (the row representation per-series state needs at @@ -402,10 +314,9 @@ impl OptimizationPass for Stage1ToStage3Pass { } } } - output.tumbling_window_retention_ms = run - .selected_physical_plan - .map(|realized| realized.tumbling_window_retention_ms) - .unwrap_or_default(); + if let Some(realized) = run.selected_physical_plan { + output.tumbling_window_retention_ms = realized.tumbling_window_retention_ms; + } output.selection = Some(selection); Ok(output) } diff --git a/crates/planner/tests/deployment_accuracy_model.rs b/crates/planner/tests/deployment_accuracy_model.rs deleted file mode 100644 index 91cda2b04..000000000 --- a/crates/planner/tests/deployment_accuracy_model.rs +++ /dev/null @@ -1,374 +0,0 @@ -//! The deployment's accuracy model plugs into plan selection (#509 Stage 3) -//! only. Stage 1 composes guarantees with the built-in analytical rules, so -//! a different model leaves the candidates unchanged and changes only which -//! of them Stage 3 accepts. - -use std::collections::BTreeSet; -use std::rc::Rc; - -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel}; -use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; -use asap_plan_selection::reference_pricer::{select_among_plans, WholePlanSelection}; -use asap_plan_selection::{ErrorSelectingPlan, RejectionCause}; -use asap_planner::PlanningModels; - -/// The reference pricer's selection among every given candidate, and the -/// selected plan's operator roots. -struct PlanOutput { - plans: Vec, - selection: Option, -} - -struct QueryPlan { - root: Rc, -} -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::workload::{ - DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, RepeatedDemand, RepeatingEntry, RepetitionInterval, -}; - -/// A deployment that trusts no approximate estimate: only an exact -/// guarantee meets a target. -struct ExactOnlyAccuracyModel; - -impl AccuracyModel for ExactOnlyAccuracyModel { - fn satisfies(&self, guarantee: &AccuracyGuarantee, _target: &AccuracyTarget) -> bool { - guarantee.is_exact() - } -} - -/// `query` with a 1% rank error, repeated every ten minutes over metrics -/// sampled every 15 s. -fn rank_target_workload(query: &str) -> PlanningWorkload { - PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: None, - repeating_queries: Some(vec![RepeatingEntry { - query: Query(query.into()), - demand: RepeatedDemand::FixedInterval(RepetitionInterval(600_000)), - requirements: QueryRequirements { - accuracy: AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank), - max_latency_ms: None, - }, - predictability: Predictability::Unknown, - range_evaluation: None, - }]), - }, - data_workload: Some(DataWorkload { - arrival: DataArrival::Ingesting { - rows_per_second: 1.0, - }, - ..DataWorkload::sampled_every(promql_metric_names(query).unwrap(), DurationMs(15_000)) - }), - } -} - -/// Whether `root` reads a sketch state. -fn reads_sketch(root: &Rc) -> bool { - OperatorNode::reachable(root).into_iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) -} - -/// A small, fixed physical inventory, independent of deployment accuracy models. -/// Exercise each local implementation with the remaining computations exact. -/// These fixtures test evaluator hooks, not a timed search's archive prefix. -fn physical_candidates( - workload: &PlanningWorkload, -) -> Vec { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen; - use asap_types::physical::Materialization; - - let roots = lower_promql_query_workload(workload).expect("lowers"); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut choices = BTreeSet::from([vec![0; inventory.targets.len()]]); - for (index, target) in inventory.targets.iter().enumerate() { - for implementation in 1..target.choice_count() { - let mut choice = vec![0; inventory.targets.len()]; - choice[index] = implementation; - choices.insert(choice); - } - } - choices - .into_iter() - .enumerate() - .map(|(index, choice)| { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let mut candidate = optimize_physical_asap_chosen( - &format!("fixture-{index}"), - &roots, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - false, - &mut |_, _, _| Some(Materialization::NotMaterialized), - ) - .unwrap() - .expect("query-time fixture is executable"); - candidate.id = format!("fixture-{index}"); - candidate - }) - .collect() -} - -fn plan_every_candidate( - workload: &PlanningWorkload, - candidates: &[asap_types::physical::PhysicalWorkloadPlan], - models: PlanningModels<'_>, -) -> Result { - let selection = select_among_plans( - candidates, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - &models.accuracy_check_inputs(), - models.cost_inputs, - )?; - let plans = selection - .plan - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryPlan { - root: Rc::clone(node), - }, - QueryRoot::Scalar(_) => panic!("operator queries only"), - }) - .collect(); - Ok(PlanOutput { - plans, - selection: Some(selection), - }) -} - -/// Every candidate Stage 3 was given: the valid ones it priced and the -/// ones it rejected. -fn stage3_candidate_ids(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .costs - .keys() - .cloned() - .chain(selection.rejected.iter().map(|r| r.id.clone())) - .collect() -} - -/// The candidates for which Stage 3 accepted an approximate output for some -/// query. -fn approximate_accepted(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .accepted - .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_metric.is_some())) - .map(|(id, _)| id.clone()) - .collect() -} - -#[tokio::test] -async fn the_deployment_accuracy_model_decides_only_in_plan_selection() { - // The user's workload: one p99 query with a 1% rank-error target. - let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - - let candidates = physical_candidates(&workload); - - // The built-in models: Stage 3 checks guarantees with BuiltinAccuracyModel. - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - // The candidates whose approximate output the built-in model accepts: - // the sketch candidates, sized in Stage 1 for the 1% target. - let builtin_approximate = approximate_accepted(&builtin); - assert!(!builtin_approximate.is_empty()); - - // The same workload with a deployment model that accepts only exact - // outputs, plugged into Stage 3 through `PlanningModels::with_accuracy`. - let exact_only_model = ExactOnlyAccuracyModel; - let exact_only = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&exact_only_model), - ) - .expect("plans with the exact-only model"); - - // Stages 1 and 2 do not read the model: Stage 3 got the same - // candidates, with the same guarantees and summary parameters. - assert_eq!( - stage3_candidate_ids(&builtin), - stage3_candidate_ids(&exact_only) - ); - - // Stage 3 now rejects each of those candidates for missing its target, - // accepts no approximate output and selects an exact plan. - let exact_only_selection = exact_only.selection.as_ref().unwrap(); - for id in &builtin_approximate { - assert!(exact_only_selection - .rejected - .iter() - .any(|r| &r.id == id && r.cause == RejectionCause::AccuracyTargetMissed)); - } - assert!(approximate_accepted(&exact_only).is_empty()); - assert!(!reads_sketch(&exact_only.plans[0].root)); -} - -/// A deployment that has measured every output itself and accepts each one -/// for any target, whatever its guarantee says. -struct AcceptsEveryOutput; - -impl AccuracyModel for AcceptsEveryOutput { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } -} - -/// The candidates for which Stage 3 accepted some query's output because -/// the deployment's model accepted it regardless of its guarantee. -fn accepted_regardless_of_guarantee(output: &PlanOutput) -> BTreeSet { - let selection = output.selection.as_ref().expect("plan selection ran"); - selection - .accepted - .iter() - .filter(|(_, queries)| queries.iter().any(|q| q.accepted_regardless_of_guarantee)) - .map(|(id, _)| id.clone()) - .collect() -} - -/// The ids of the candidates `output`'s selection rejected with `cause`. -fn rejected_with(output: &PlanOutput, cause: RejectionCause) -> BTreeSet { - output - .selection - .as_ref() - .expect("plan selection ran") - .rejected - .iter() - .filter(|r| r.cause == cause) - .map(|r| r.id.clone()) - .collect() -} - -/// A deployment model that accepts an output directly also accepts one with -/// no guarantee: the built-in model rejects the summary candidates of a -/// difference of two quantile estimates as having none, and the accepting -/// model makes them valid, and Stage 3 selects one. -#[tokio::test] -async fn the_deployment_accuracy_model_accepts_outputs_without_a_guarantee() { - // The user's workload: a difference of two quantile estimates, 1% rank - // error. Stage 1 has no analytical rule for a difference of rank - // estimates, so it keeps the summary candidates with no guarantee. - let workload = rank_target_workload("quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))"); - - let candidates = physical_candidates(&workload); - - // The built-in model (default hook: accepts nothing directly) rejects - // every summary candidate as having no guarantee and selects exact. - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - // Candidates rejected for having no guarantee. - let unguaranteed = rejected_with(&builtin, RejectionCause::NoAccuracyGuarantee); - assert!(!unguaranteed.is_empty()); - assert!(accepted_regardless_of_guarantee(&builtin).is_empty()); - assert!(!reads_sketch(&builtin.plans[0].root)); - - // The deployment's model accepts every output directly. - let accepting_model = AcceptsEveryOutput; - let accepting = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .expect("plans with the accepting model"); - let accepting_selection = accepting.selection.as_ref().unwrap(); - assert_eq!( - stage3_candidate_ids(&builtin), - stage3_candidate_ids(&accepting) - ); - - // Every candidate the built-in model rejected for having no guarantee - // is now valid and priced, accepted regardless of its guarantee, and no - // candidate fails the accuracy check. - assert!(accepted_regardless_of_guarantee(&accepting).is_superset(&unguaranteed)); - for id in &unguaranteed { - assert!(accepting_selection.costs.contains_key(id), "{id}"); - } - assert!(accepting_selection.rejected.iter().all(|r| !matches!( - r.cause, - RejectionCause::NoAccuracyGuarantee | RejectionCause::AccuracyTargetMissed - ))); - // A singleton previously rejected sketch is now selectable; the cheapest - // plan in the complete fixture need not be that sketch. - let sketch = candidates - .iter() - .find(|c| unguaranteed.contains(&c.id)) - .unwrap(); - let singleton = plan_every_candidate( - &workload, - std::slice::from_ref(sketch), - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .unwrap(); - assert_eq!(singleton.selection.as_ref().unwrap().selected, sketch.id); - assert!(reads_sketch(&singleton.plans[0].root)); -} - -/// The hook has priority over a guarantee that misses: a candidate whose -/// guarantee misses its target under the built-in models (a relative bound -/// for a rank target) is accepted under the accepting model. -#[tokio::test] -async fn the_deployment_accuracy_model_accepts_outputs_whose_guarantee_misses() { - // The user's workload: one p99 query with a 1% rank-error target. - let workload = rank_target_workload("quantile_over_time(0.99, lat[5m])"); - let candidates = physical_candidates(&workload); - // Under the built-in models: the candidates whose guarantee misses the - // target (a relative bound for a rank target). - let builtin = plan_every_candidate(&workload, &candidates, PlanningModels::builtin()) - .expect("plans with the built-in models"); - let missed = rejected_with(&builtin, RejectionCause::AccuracyTargetMissed); - assert!(!missed.is_empty()); - // The same workload with the model that accepts every output directly. - let accepting_model = AcceptsEveryOutput; - let accepting = plan_every_candidate( - &workload, - &candidates, - PlanningModels::builtin().with_accuracy(&accepting_model), - ) - .expect("plans with the accepting model"); - // Each of them is now valid, priced and accepted regardless of guarantee. - let accepting_selection = accepting.selection.as_ref().unwrap(); - for id in &missed { - assert!(accepting_selection.costs.contains_key(id), "{id}"); - assert!( - accepting_selection.accepted[id][0].accepted_regardless_of_guarantee, - "{id}" - ); - } - // The hook also overrides guarantees that meet the target: every - // accepted output, including exact and KLL candidates, is accepted - // regardless of its guarantee. - assert_eq!(accepting_selection.accepted.len(), candidates.len()); - for (id, queries) in &accepting_selection.accepted { - assert!( - queries.iter().all(|q| q.accepted_regardless_of_guarantee), - "{id}" - ); - } -} diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index bb91f08fe..9d06c8c46 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -5,10 +5,7 @@ mod time_series; mod columnar_batches; -mod deployment_accuracy_model; mod e2e_plan; -mod per_sample_updater; mod runtime_install; -mod sql_per_group_topk; mod sql_table_scans; mod summary_sharing; diff --git a/crates/planner/tests/per_sample_updater.rs b/crates/planner/tests/per_sample_updater.rs deleted file mode 100644 index b8f83f1e5..000000000 --- a/crates/planner/tests/per_sample_updater.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! A runtime that ingests sample by sample refreshes a materialized tumbling window's states -//! with `StateColumnUpdater`, the code the plan's `SummaryAgg` runs on a -//! batch (issue #762): the same states as the executor's tumbling-window build. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use asap_executor::physical_planner::compile; -use asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload as Payload; -use asap_executor::planner::ir::ASAPOp; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::summary_kernels::exact::ExactAccumulator; -use asap_executor::StateColumnUpdater; -use asap_planner::{ - plan_installable, test_support::query_time_reference, DeploymentCapabilities, FrontendInput, - PlanningModels, PlanningWorkload, UserInput, -}; -use futures::StreamExt; - -use crate::time_series; -use time_series::{series_identity, series_labels, time_series_tables, RawDataSample}; - -/// The evaluation time: the end of an hour. -const EVALUATION_MS: i64 = 3_600_000; - -#[tokio::test] -async fn per_sample_updates_build_the_materialized_tumbling_window_the_plan_refreshes() { - // What the user writes: the 5-minute rate of a counter, every minute, - // exactly, over 1,000 series sampled every 15 s. - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "rate(http_requests_total[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 66.7}}, - "sources": {"http_requests_total": {"series_count": 1000, "sample_interval": 15000}}}}"#, - ) - .unwrap(); - // No raw data samples retained: the plan stores rate tumbling windows built at ingestion. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - // The installable plan: the selected DAG and its materialized views. - let plan = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap() - .0; - let dag = &plan.dag; - // The newest tumbling window view and its window at this evaluation. - let views = plan.materialized_views().unwrap(); - let output = views - .iter() - .max_by_key(|output| { - output.inputs.raw_data_windows()[0] - .bounds_ms(EVALUATION_MS) - .1 - }) - .unwrap(); - let (start_ms, end_ms) = output.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - // Five one-minute tumbling windows, the newest (59 min, 60 min]. - assert_eq!(views.len(), 5); - assert_eq!((start_ms, end_ms), (3_540_000, 3_600_000)); - // The view node's summary family, as the plan declares it: the - // `SummaryAgg` at or below the view node. - let family = { - let mut id = output.node_id; - loop { - let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = &node.payload { - break family.clone(); - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - } - }; - // What the runtime received in the tumbling window: 3 series, one counter sample - // every 15 s, one series with a reset. - assert_eq!( - family, - asap_executor::planner::ir::schema::FieldDataType::ExactAggregate( - asap_executor::planner::ir::schema::ExactKind::Rate, - asap_executor::planner::ir::schema::ExactParams::Rate, - ) - ); - let samples: Vec = ["a", "b", "c"] - .iter() - .flat_map(|instance| { - let mut value = 0.0; - (1..=(end_ms - start_ms) / 15_000).map(move |step| { - value = if *instance == "c" && step == 3 { - 2.0 - } else { - value + 10.0 - }; - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "http_requests_total".to_string()), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: start_ms + step * 15_000, - value, - } - }) - }) - .collect(); - - // The executor's tumbling-window build: the samples bound at the plan's `Scan`, - // its `TimeRange` keeping the time range the tumbling window covers. - let compiled = compile(dag, &[], &[output.node_id as u64]).unwrap(); - let build = compiled - .instantiate(Arc::new(time_series_tables(dag, &samples))) - .unwrap(); - let context = RunContext::new( - Scope::Ingestion { - window_start_ms: start_ms, - window_end_ms: end_ms, - }, - Limits::default(), - ) - .unwrap(); - let tumbling_window = build - .execute(compiled.roots(), context) - .unwrap() - .remove(0) - .next() - .await - .unwrap() - .unwrap(); - // Its states, by series labels. - let state_column = tumbling_window - .schema() - .fields - .iter() - .position(|field| !field.is_plain()) - .unwrap(); - let from_dag: BTreeMap, String> = tumbling_window - .to_rows() - .iter() - .map(|row| { - let asap_executor::values::Value::Summary { state, .. } = &row[state_column] else { - panic!("a state") - }; - let exact = state.as_any().downcast_ref::().unwrap(); - ( - series_labels(tumbling_window.schema(), row), - serde_json::to_string(exact).unwrap(), - ) - }) - .collect(); - - // The per-sample build: one updater for the tumbling window, a group per series. - let mut updater = StateColumnUpdater::new(&family).unwrap(); - let mut groups: BTreeMap, u32> = BTreeMap::new(); - for sample in &samples { - let group = match groups.get(&sample.labels) { - Some(&group) => group, - None => { - let group = updater.add_group().unwrap(); - groups.insert(sample.labels.clone(), group); - group - } - }; - updater - .update(group, sample.value, sample.timestamp_ms) - .unwrap(); - } - // Sealed at tumbling window close: one state per group. - let states = updater.seal(); - let per_sample: BTreeMap, String> = groups - .into_iter() - .map(|(mut labels, group)| { - let state = states.state(group as usize); - let exact = state.as_any().downcast_ref::().unwrap(); - // A rate's series drops its metric name, as the DAG's does. - if !from_dag.keys().any(|l| l.contains_key("__name__")) { - labels.remove("__name__"); - } - (labels, serde_json::to_string(exact).unwrap()) - }) - .collect(); - assert_eq!(from_dag.len(), 3); - assert_eq!(per_sample, from_dag); -} - -/// Run `ids` of `dag` with the view states `inputs` bound at their nodes -/// and the raw `samples` at its `Scan`s, in `scope`: each id's answer rows -/// as (series labels, value), sorted. -async fn answers( - dag: &asap_planner::PhysicalASAPDAG, - inputs: BTreeMap, - samples: &[RawDataSample], - ids: &[u64], - scope: Scope, -) -> Vec, f64)>> { - // The plan's views whose states are bound; each one's schema is its DAG - // node's. - let views: Vec<_> = dag - .materialized_views() - .unwrap() - .into_iter() - .filter(|view| inputs.contains_key(&(view.node_id as u64))) - .collect(); - assert_eq!(views.len(), inputs.len()); - // The executor's plan for `ids`, reading the bound views. - let compiled = compile(dag, &views, ids).unwrap(); - // Each input as a view's states in an in-memory store. - let mut store = time_series_tables(dag, samples); - for (id, batch) in inputs { - store = store.with_materialized_view(id as _, vec![batch]).unwrap(); - } - let instance = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut results = Vec::new(); - for mut output in instance.execute(compiled.roots(), context).unwrap() { - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch.unwrap(); - for row in &batch.to_rows() { - let labels = series_labels(batch.schema(), row); - let value = row - .iter() - .find_map(|v| match v { - asap_executor::values::Value::Float64(v) => Some(*v), - _ => None, - }) - .unwrap(); - rows.push((labels, value)); - } - } - rows.sort_by(|a, b| a.0.cmp(&b.0)); - results.push(rows); - } - results -} - -/// A runtime builds every materialized tumbling window sample by sample, installs each as a -/// batch with `Batch::try_from_columns` (no DAG run), and answers the query -/// from them: the same answers as the plan's query-time reference over the -/// raw data samples. -#[tokio::test] -async fn tumbling_windows_installed_from_per_sample_states_answer_the_query() { - use asap_executor::arrow::array::{DictionaryArray, TimestampMillisecondArray, UInt32Array}; - use asap_executor::arrow::datatypes::UInt32Type; - use asap_executor::values::{Batch, BatchColumn}; - use std::sync::Arc; - // What the user writes: the 5-minute rate of a counter, every minute, - // exactly, over 1,000 series sampled every 15 s. - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "rate(http_requests_total[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 66.7}}, - "sources": {"http_requests_total": {"series_count": 1000, "sample_interval": 15000}}}}"#, - ) - .unwrap(); - // No raw data samples retained: five one-minute rate tumbling windows built at ingestion. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan = plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .unwrap() - .0; - let dag = &plan.dag; - // What the runtime received: 3 series, a counter sample every 15 s over - // the hour (one series resets), the last at the evaluation time. - let samples: Vec = ["a", "b", "c"] - .iter() - .flat_map(|instance| { - let mut value = 0.0; - (1..=EVALUATION_MS / 15_000).map(move |step| { - value = if *instance == "c" && step == 230 { - 3.0 - } else { - value + 7.0 - }; - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "http_requests_total".to_string()), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: step * 15_000, - value, - } - }) - }) - .collect(); - // Every materialized tumbling window, built sample by sample and installed as a batch. - let mut tumbling_windows = BTreeMap::new(); - for output in &plan.materialized_views().unwrap() { - // The time range the tumbling window covers, `(start_ms, end_ms]`, at this evaluation. - let (start_ms, end_ms) = output.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - // The view node's family and output schema, from the plan. - let node = dag.nodes.iter().find(|n| n.id == output.node_id).unwrap(); - let mut id = output.node_id; - let family = loop { - let node = dag.nodes.iter().find(|n| n.id == id).unwrap(); - if let Payload::ASAP(ASAPOp::SummaryAgg { family, .. }) = &node.payload { - break family.clone(); - } - id = dag - .edges - .iter() - .find(|e| e.consumer == id) - .unwrap() - .producer; - }; - // The runtime's per-sample build: a group per series ... - let mut updater = StateColumnUpdater::new(&family).unwrap(); - let mut identities: Vec = Vec::new(); - let mut group_of_series: BTreeMap, u32> = BTreeMap::new(); - for sample in samples - .iter() - .filter(|s| start_ms < s.timestamp_ms && s.timestamp_ms <= end_ms) - { - let group = match group_of_series.get(&sample.labels) { - Some(&group) => group, - None => { - let group = updater.add_group().unwrap(); - group_of_series.insert(sample.labels.clone(), group); - identities.push(series_identity(&sample.labels)); - group - } - }; - updater - .update(group, sample.value, sample.timestamp_ms) - .unwrap(); - } - // ... sealed at tumbling window close: one rate record per series. - let states = Arc::new(updater.seal()); - // The tumbling window's columns, in the view node's schema: the tumbling window end as - // its time, each series' state, and its identity. - let schema = Arc::new(node.output_schema.clone()); - let groups = identities.len(); - let columns = schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - if Some(index) == schema.time_index { - BatchColumn::Plain(Arc::new(TimestampMillisecondArray::from(vec![ - end_ms; - groups - ]))) - } else if !field.is_plain() { - BatchColumn::States { - indices: UInt32Array::from_iter_values(0..groups as u32), - states: states.clone(), - } - } else { - BatchColumn::Plain(Arc::new( - identities - .iter() - .map(String::as_str) - .collect::>(), - )) - } - }) - .collect(); - // Installed without running the DAG. - tumbling_windows.insert( - output.node_id as u64, - Batch::try_from_columns(schema, columns).unwrap(), - ); - } - assert_eq!(tumbling_windows.len(), 5); - let query = Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }; - let roots: Vec = plan.dag.roots.iter().map(|&r| r as u64).collect(); - // The query's answers from the installed tumbling windows. - let from_tumbling_windows = - answers(dag, tumbling_windows, &samples, &roots, query.clone()).await; - // The plan's query-time reference over the raw data samples. - let reference = query_time_reference(&plan).unwrap(); - let reference_roots: Vec = reference.dag.roots.iter().map(|&r| r as u64).collect(); - let expected = answers( - &reference.dag, - BTreeMap::new(), - &samples, - &reference_roots, - query, - ) - .await; - // One rate per series, equal to the reference. - assert_eq!(from_tumbling_windows[0].len(), 3); - assert_eq!(from_tumbling_windows, expected); -} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 7b952acb2..b8087aab7 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -16,28 +16,16 @@ use asap_executor::sources::{ }; use asap_executor::values::{Batch, Value}; use asap_executor::Error as ExecutorError; -use asap_physical_asap_optimization::test_support::split_shared_by_timing; use asap_planner::{ - plan_installable, run_planning, CostInputs, DeploymentCapabilities, FrontendInput, - InstallablePlan, InstallablePlanError, MaterializedView, MaterializedViewRefresh, OperatorKind, - OptimizationInput, OptimizationPass, OptimizeError, PhysicalASAPDAG, PlanOutput, - PlanSearchSettings, PlanningModels, PlanningWorkload, RefreshWindow, SelectedPlanReport, - SelectionBox, Stage1ToStage3Pass, UserInput, + plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, + MaterializedView, OperatorKind, PhysicalASAPDAG, PlanningModels, PlanningWorkload, + SelectedPlanReport, SelectionBox, UserInput, }; use asap_types::ir::operator::Source; -use asap_types::ir::schema::schema_support::with_promql_series_identity; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; -use asap_types::ir::{ - apply_materialization_timings_to_roots, ASAPOp, MaterializationAssignment, OperatorNode, - QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; -use asap_types::stages::HowToComputeAggregateMeasure; -use asap_types::stages::PruneReason; + use futures::{executor::block_on, StreamExt}; -use std::collections::HashMap; use std::ops::Bound; -use std::rc::Rc; + use time_series::{series_labels, time_series_tables, RawDataSample}; /// Example 4b's evaluation time: the end of the 60-minute window. @@ -80,31 +68,6 @@ fn sample(metric: &str, instance: &str, timestamp_ms: i64, value: f64) -> RawDat } } -/// Raw `latency_ms` samples of series `a` and `b`, one every 10 s from 70 -/// minutes before `EVALUATION_MS` to a minute after it (for a second -/// evaluation), off the tumbling-window boundaries, and one -/// sample of another metric the plan never reads. A sample's value says -/// which 10-minute tumbling window before `EVALUATION_MS` it falls in: `1000 × tumbling_window + -/// base + 0..=100`, with base 100 for `a` and 300 for `b` (tumbling window 0 is the -/// newest), so an answer read from the wrong tumbling windows is visibly off. -fn remote_write_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, base) in [("a", 100.0), ("b", 300.0)] { - for step in 0..426 { - let timestamp_ms = EVALUATION_MS - 4_195_000 + step * 10_000; - let tumbling_window = (EVALUATION_MS - timestamp_ms) / 600_000; - samples.push(sample( - "latency_ms", - instance, - timestamp_ms, - (1000 * tumbling_window) as f64 + base + (step * 37 % 101) as f64, - )); - } - } - samples.push(sample("cpu_seconds", "a", 5_000, 1e9)); - samples -} - thread_local! { /// How many `read_raw_data` calls the runs on this thread made. static RAW_DATA_READS: Cell = const { Cell::new(0) }; @@ -206,23 +169,6 @@ fn answers(batches: &[Batch]) -> Vec { answers } -/// Each answer is a p99 read from tumbling window `tumbling_window` of its series -/// ([`remote_write_samples`]): within `1000 × tumbling_window + base + 0..=100`. -fn assert_answers_from_tumbling_window(answers: &[Answer], tumbling_window: i64) { - for (labels, value) in answers { - let base = if labels["instance"] == "a" { - 100.0 - } else { - 300.0 - }; - let low = (1000 * tumbling_window) as f64 + base; - assert!( - (low..=low + 100.0).contains(value), - "{labels:?}: {value} is not from tumbling window {tumbling_window}" - ); - } -} - /// The control plane: plan `queries` for a deployment that keeps no raw /// samples, with its cost model read from JSON, and write the plan file. async fn plan_json(queries: &[&str]) -> String { @@ -280,453 +226,11 @@ async fn plan_with_report_for( .unwrap() } -/// What one data-plane evaluation of a plan file returned. -struct DataPlaneRun { - /// Per query, in workload order, the installed plan's answers. - answers_per_query: Vec>, - /// Per query, the query-time reference's answers. - reference_answers_per_query: Vec>, - /// How many materialized views this evaluation refreshed (ingestion or - /// query runs); the others it read came from the store. - outputs_built: usize, - /// How many `read_raw_data` calls those refreshes made. - raw_data_reads_refreshing: usize, -} - -/// One summary state this test stores: its refreshed state, and how long -/// to keep it. -struct StoredState { - /// The refreshed state, bound at every materialized view with its key. - state: Batch, - /// When the data it summarizes starts, in Unix milliseconds. - start_ms: i64, - /// How far back from the newest evaluation it is kept: the largest - /// `retention_ms` among the views it served. - retention_ms: u64, -} - -/// This stand-in runtime's summary store, by [`state_key`]. -type SummaryStore = BTreeMap; - -/// The key of `view`'s state at evaluation `evaluation_ms`, as a summary -/// store keys instances: its definition id and its selection made -/// concrete (relative time made absolute). A view whose selection -/// names no time (its definition reads a window relative to the -/// evaluation, e.g. an instant selector) is keyed by the evaluation too. -fn state_key(view: &MaterializedView, evaluation_ms: i64) -> String { - use std::ops::Bound; - let absolute = |bound: &Bound| match bound { - Bound::Included(at) => Bound::Included(evaluation_ms + at), - Bound::Excluded(at) => Bound::Excluded(evaluation_ms + at), - Bound::Unbounded => Bound::Unbounded, - }; - let boxes = view.coverage().selection(); - let timed = boxes.iter().any(|b| b.relative_time().is_some()); - let selection: Vec = boxes - .iter() - .map(|selection_box| { - SelectionBox::new( - selection_box.columns().clone(), - selection_box - .relative_time() - .map(|(lower, upper)| (absolute(&lower), absolute(&upper))), - ) - }) - .collect(); - serde_json::json!({ - "definition_id": view.coverage().definition_id(), - "selection": selection, - "evaluation_ms": (!timed).then_some(evaluation_ms), - }) - .to_string() -} - /// The node that answers each query of `plan`, in workload order. fn query_roots(plan: &InstallablePlan) -> Vec { plan.dag.roots.iter().map(|&root| root as u64).collect() } -/// The data plane at evaluation time `evaluation_ms`: from `plan_json`, -/// `samples` and the states earlier evaluations left in `store`, refresh each -/// materialized view the store lacks once, from its raw-data windows and the -/// views it reads: the views refreshed at ingestion by refresh group, one -/// ingestion run per group and span, then the others; answer every query; drop the -/// states past their retention; and answer the queries again with the -/// query-time reference. -fn run_data_plane( - plan_json: &str, - samples: &[RawDataSample], - evaluation_ms: i64, - store: &mut SummaryStore, -) -> DataPlaneRun { - run_data_plane_on( - plan_json, - samples, - evaluation_ms, - store, - IngestionBuilds::PerEvaluation, - ) -} - -/// When this stand-in runtime refreshes a materialized view an ingestion -/// run refreshes from raw data samples. -#[derive(Clone, Copy)] -enum IngestionBuilds { - /// For each evaluation, over windows ending at that evaluation. - PerEvaluation, - /// Tumbling, as a runtime that maintains tumbling windows as samples arrive: each - /// once, over windows ending on the grid of its window's width anchored - /// at the epoch (tumbling window end ≡ 0 mod width, as the backend data plane - /// builds them); an evaluation reads the newest one at or before it. - Tumbling, -} - -/// [`run_data_plane`] with at-ingestion views refreshed as `ingestion` says. -fn run_data_plane_on( - plan_json: &str, - samples: &[RawDataSample], - evaluation_ms: i64, - store: &mut SummaryStore, - ingestion: IngestionBuilds, -) -> DataPlaneRun { - // The installed plan, read back from its file and checked. - let installed: InstallablePlan = serde_json::from_str(plan_json).unwrap(); - installed.validate().unwrap(); - // The DAG the executor runs. - let dag = &installed.dag; - let views = installed.materialized_views().unwrap(); - // Every input of the query run, keyed by the node it is bound to. - let mut inputs = BTreeMap::new(); - // How many materialized views this evaluation refreshes. - let mut outputs_built = 0; - // Each view's state key and the time span of its data at this - // evaluation, `(start_ms, end_ms]`, by node id. - let mut planned: BTreeMap = BTreeMap::new(); - for view in &views { - // The evaluation its windows end at: this one, or for a tumbling - // at-ingestion view the newest point of its grid at or before it. - let built_for_ms = match (ingestion, view.refresh()) { - (IngestionBuilds::Tumbling, MaterializedViewRefresh::AtIngestion) => view - .inputs - .raw_data_windows() - .first() - .map_or(evaluation_ms, |window| { - evaluation_ms - evaluation_ms.rem_euclid(window.width_ms()) - }), - _ => evaluation_ms, - }; - // The span of the data it summarizes: its windows' and the views' - // it reads spans together. - let span = view - .inputs - .raw_data_windows() - .iter() - .map(|window| window.bounds_ms(built_for_ms)) - .chain( - view.inputs - .materialized_views - .iter() - .map(|id| planned[&(*id as u64)].1), - ) - .reduce(|(a, b), (c, d)| (a.min(c), b.max(d))) - .expect("a materialized view reads something"); - planned.insert(view.node_id as u64, (state_key(view, built_for_ms), span)); - } - // The runs that refresh the views the store lacks: for each refresh - // group, one ingestion run per span its views' data covers, with every - // view of the group over that span as a root (a node they share runs - // once); then each view refreshed on first read in this evaluation's - // query run. - let mut refreshes: Vec<(Vec<&MaterializedView>, Scope)> = Vec::new(); - for group in installed.ingestion_refresh_groups().unwrap() { - let mut by_span: BTreeMap<(i64, i64), Vec<&MaterializedView>> = BTreeMap::new(); - for member in &group.views { - let view = views - .iter() - .find(|view| view.node_id == member.node_id) - .unwrap(); - by_span - .entry(planned[&(view.node_id as u64)].1) - .or_default() - .push(view); - } - for (span, members) in by_span { - let scope = Scope::Ingestion { - window_start_ms: span.0, - window_end_ms: span.1, - }; - refreshes.push((members, scope)); - } - } - for view in views - .iter() - .filter(|view| view.refresh() == MaterializedViewRefresh::OnFirstRead) - { - let scope = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - refreshes.push((vec![view], scope)); - } - let raw_data_reads_before = RAW_DATA_READS.with(Cell::get); - for (members, scope) in refreshes { - // The views of the run whose state the store lacks, one per state. - let mut roots: Vec<&MaterializedView> = Vec::new(); - for view in members { - let key = &planned[&(view.node_id as u64)].0; - if !store.contains_key(key) - && !roots - .iter() - .any(|root| &planned[&(root.node_id as u64)].0 == key) - { - roots.push(view); - } - } - if roots.is_empty() { - continue; - } - // The views the run reads that it does not refresh, from the - // store; its raw data is bound at its `Scan`s. - let mut sources = BTreeMap::new(); - for id in roots - .iter() - .flat_map(|view| &view.inputs.materialized_views) - { - if !roots.iter().any(|root| root.node_id == *id) { - let key = &planned[&(*id as u64)].0; - sources.insert(*id as u64, store[key].state.clone()); - } - } - let root_ids: Vec = roots.iter().map(|view| view.node_id as u64).collect(); - // Each root's batches, in one batch per root, as the store keeps it. - let outputs = run(dag, sources, samples, &root_ids, scope); - for (view, batches) in roots.into_iter().zip(outputs) { - let (key, span) = &planned[&(view.node_id as u64)]; - let schema = batches[0].schema().clone(); - let state_rows = batches.iter().flat_map(|b| b.to_rows().to_vec()).collect(); - store.insert( - key.clone(), - StoredState { - state: Batch::try_new(schema, state_rows).unwrap(), - start_ms: span.0, - retention_ms: view.retention_ms(), - }, - ); - outputs_built += 1; - } - } - let raw_data_reads_refreshing = RAW_DATA_READS.with(Cell::get) - raw_data_reads_before; - // Each view's state and the span of its data, by node id. - let mut built: BTreeMap = BTreeMap::new(); - for view in &views { - let (key, span) = &planned[&(view.node_id as u64)]; - // One stored state serves every view with its key, so it is kept - // for the largest retention any of them requires. - let stored = store.get_mut(key).unwrap(); - stored.retention_ms = stored.retention_ms.max(view.retention_ms()); - built.insert(view.node_id as u64, (stored.state.clone(), *span)); - } - // The query run reads the views it reaches from the queries' roots - // without passing another view; those below are read only through the - // views refreshed from them. - let mut pending = query_roots(&installed); - let mut seen = std::collections::BTreeSet::new(); - while let Some(id) = pending.pop() { - if !seen.insert(id) { - continue; - } - match built.get(&id) { - Some((state, _)) => { - inputs.insert(id, state.clone()); - } - None => pending.extend( - dag.edges - .iter() - .filter(|edge| edge.consumer as u64 == id) - .map(|edge| edge.producer as u64), - ), - } - } - // The query evaluation, which reads the stored states. - let query = Scope::Query { - evaluation_time_ms: evaluation_ms, - }; - // Each query's answers: (series labels, value) per series. - let answers_per_query = run( - dag, - inputs, - samples, - &query_roots(&installed), - query.clone(), - ) - .iter() - .map(|batches| answers(batches)) - .collect(); - // The store drops each state whose data starts more than its retention - // before this evaluation. - store.retain(|_, state| state.start_ms >= evaluation_ms - state.retention_ms as i64); - - // The correctness reference: nothing stored, every window read from - // the raw samples at query time. - let reference = asap_planner::test_support::query_time_reference(&installed).unwrap(); - // The reference's DAG. - let reference_dag = &reference.dag; - // The reference's answers, which the installed plan's must equal. - let reference_answers_per_query = run( - reference_dag, - BTreeMap::new(), - samples, - &query_roots(&reference), - query, - ) - .iter() - .map(|batches| answers(batches)) - .collect(); - DataPlaneRun { - answers_per_query, - reference_answers_per_query, - outputs_built, - raw_data_reads_refreshing, - } -} - -/// The sketch algorithm of the tumbling windows view `id` of `plan` builds. -fn tumbling_window_sketch(plan: &InstallablePlan, id: usize) -> SketchAlgorithm { - let node = plan.dag.nodes.iter().find(|node| node.id == id).unwrap(); - match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => *kind.algorithm(), - other => panic!("not a sketch tumbling window: {other:?}"), - } -} - -/// Issue #1151's example: a p90 at rank error 0.01 over a minute and a p99 -/// at relative error 0.01 over five minutes, both every 10 s, over 1,000 -/// ingesting non-negative `latency_ms` gauges sampled every 10 s, on a -/// deployment that keeps no raw data: a KLL chain and a DDSketch chain, -/// both of 10 s tumbling windows, kept at ingestion over one `Scan`. Their tumbling windows are one refresh group, so each -/// 10 s window is read from raw data once, in one ingestion run that builds -/// both tumbling windows ending there: 30 reads for the 30 tumbling window windows of the first -/// evaluation (36 when each tumbling window view had its own run). Each state equals -/// the one a run of its view alone builds, and both queries answer as the -/// query-time reference. -#[tokio::test] -async fn tumbling_windows_of_one_refresh_window_read_each_raw_window_once() { - let entry = |query: &str, metric: &str| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"accuracy_metric": {metric:?}, "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }; - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}, {}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entry("quantile_over_time(0.9, latency_ms[1m])", "rank"), - entry("quantile_over_time(0.99, latency_ms[5m])", "relative"), - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - installed.check_compiles().unwrap(); - // One refresh group: the six 10 s KLL tumbling windows and the thirty 10 s - // DDSketch tumbling windows. - let groups = installed.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 1, "{groups:#?}"); - assert_eq!( - groups[0].refresh_window, - Some(RefreshWindow { - width_ms: 10_000, - phase_ms: 0 - }) - ); - let sketches: Vec = groups[0] - .views - .iter() - .map(|view| tumbling_window_sketch(&installed, view.node_id)) - .collect(); - let count = |algorithm| sketches.iter().filter(|a| **a == algorithm).count(); - assert_eq!(count(SketchAlgorithm::Kll), 6, "{sketches:?}"); - assert_eq!(count(SketchAlgorithm::DDSketch), 30, "{sketches:?}"); - - let samples = remote_write_samples(); - let mut store = SummaryStore::new(); - let evaluated = run_data_plane(&plan_json, &samples, EVALUATION_MS, &mut store); - assert_eq!(evaluated.outputs_built, 36); - assert_eq!(evaluated.raw_data_reads_refreshing, 30); - assert_eq!( - evaluated.answers_per_query, - evaluated.reference_answers_per_query - ); - assert_eq!(evaluated.answers_per_query[0].len(), 2); - - // Each tumbling window's state equals the one an ingestion run of its view alone - // builds. - for view in &groups[0].views { - let span = view.inputs.raw_data_windows()[0].bounds_ms(EVALUATION_MS); - let scope = Scope::Ingestion { - window_start_ms: span.0, - window_end_ms: span.1, - }; - let alone = run( - &installed.dag, - BTreeMap::new(), - &samples, - &[view.node_id as u64], - scope, - ) - .remove(0); - // `Value` has no equality; its debug form shows each state. - let rows = format!( - "{:?}", - alone - .iter() - .flat_map(|b| b.to_rows().to_vec()) - .collect::>() - ); - let stored = &store[&state_key(view, EVALUATION_MS)].state; - assert_eq!( - format!("{:?}", stored.to_rows()), - rows, - "view {}", - view.node_id - ); - } -} - -/// #509 Example 4b, as a runtime runs it: one `plan_installable` call, -/// `plan.json`, then each tumbling window view refreshed in an ingestion run and merged -/// in the query run. One p99 per series, read from the hour's oldest tumbling window, -/// equal to the query-time reference; the other metric is not read. -#[tokio::test] -async fn a_runtime_installs_and_runs_the_plan_from_one_facade_call() { - let plan_json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = installed.materialized_views().unwrap(); - assert!(!views.is_empty()); - assert_eq!(run.outputs_built, views.len()); - // The p99 of the hour comes from its oldest tumbling window, tumbling window 5. - assert_answers_from_tumbling_window(&run.answers_per_query[0], 5); - assert_eq!(run.answers_per_query[0][0].0["instance"], "a"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - /// The plan file carries Stage 3's predicted usage of the selected plan, so /// a runtime reports it against what it measures without planning again: /// for Example 4b's tumbling windows maintained at ingestion time, its nodes' prices @@ -795,392 +299,6 @@ async fn the_plan_carries_stage3s_predicted_usage() { .any(|node| node.installed_node_ids == [root])); } -/// The query's label matcher selects series `a` only: the runtime feeds -/// every sample, and the window's selector applies the plan's `Scan`. -#[tokio::test] -async fn label_matchers_select_the_series_a_tumbling_window_reads() { - let plan_json = - plan_json(&[r#"quantile_over_time(0.99, latency_ms{instance="a"}[60m])"#]).await; - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 1, "{answered:?}"); - assert_eq!(answered[0].0["instance"], "a"); - assert_answers_from_tumbling_window(answered, 5); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// The p99 over the last hour and over the hour before the last 10 -/// minutes may share selected tumbling windows. The runtime refreshes each distinct -/// state key once, independent of the optimizer's tumbling-window widths. Both answers -/// equal the reference. -#[tokio::test] -async fn an_hour_and_its_offset_twin_both_equal_the_reference() { - let plan_json = plan_json(&[ - "quantile_over_time(0.99, latency_ms[60m])", - "quantile_over_time(0.99, latency_ms[60m] offset 10m)", - ]) - .await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let distinct_states: BTreeSet<_> = installed - .materialized_views() - .unwrap() - .iter() - .map(|view| state_key(view, EVALUATION_MS)) - .collect(); - assert!(!distinct_states.is_empty()); - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, distinct_states.len()); - assert_eq!(run.answers_per_query.len(), 2); - // The last hour's p99 comes from tumbling window 5, the offset hour's from tumbling window 6. - assert_answers_from_tumbling_window(&run.answers_per_query[0], 5); - assert_answers_from_tumbling_window(&run.answers_per_query[1], 6); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Two copies of `sum by (job) (sum_over_time(m[10m]))`, each every 20 s, -/// one at phase 0 and one at phase 5 s, over 6 ingesting series of `m` -/// sampled every second. Tumbling windows shared by both must end on both schedules: -/// 20 s tumbling windows end only on the first's, so the second would read -/// `(t − 605 s, t − 5 s]` instead of `(t − 600 s, t]`. -#[tokio::test] -async fn queries_at_two_phases_read_tumbling_windows_on_both_schedules() { - assert_phased_copies_answer_as_reference(&[0, 5_000]).await; -} - -/// The same two copies, both at phase 5 s. A runtime anchors tumbling windows at the -/// epoch and the plan carries no tumbling-window origin, so 20 s tumbling windows would end 5 s -/// before every evaluation: the tumbling windows must also end on the phase itself. -#[tokio::test] -async fn queries_at_one_nonzero_phase_read_tumbling_windows_anchored_at_the_epoch() { - assert_phased_copies_answer_as_reference(&[5_000, 5_000]).await; -} - -/// Copies of `sum by (job) (sum_over_time(m[10m]))`, each every 20 s, copy -/// `q` at phase `phases_ms[q]`, over 6 ingesting series of `m` sampled -/// every second. Run by a runtime that refreshes each ingestion tumbling window once, on -/// the grid of its width anchored at the epoch, each answers as the -/// query-time reference at its own evaluation. -async fn assert_phased_copies_answer_as_reference(phases_ms: &[i64]) { - let entries: Vec = phases_ms - .iter() - .map(|phase_ms| { - format!( - r#"{{"query": "sum by (job) (sum_over_time(m[10m]))", - "demand": {{"fixed_interval_at": {{"interval": 20000, "evaluation_phase": {phase_ms}}}}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - // A deployment that keeps no raw data samples: the windows are views. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - assert!( - views - .iter() - .any(|view| view.refresh() == MaterializedViewRefresh::AtIngestion), - "{views:#?}" - ); - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 11 minutes before the first evaluation to 10 s after it; each - // value distinct, so a window off by 5 s at either edge answers - // differently. - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..680i64).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 660_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // The tumbling windows stored across every evaluation. - let mut store = SummaryStore::new(); - // Copy `q` evaluates `phases_ms[q]` after `EVALUATION_MS` (on the - // 20 s grid); each reads the tumbling windows the store holds by then. - for (query, phase_ms) in phases_ms.iter().enumerate() { - let evaluation_ms = EVALUATION_MS + phase_ms; - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert_eq!(run.answers_per_query[query].len(), 2, "one sum per job"); - assert_eq!( - run.answers_per_query[query], run.reference_answers_per_query[query], - "query {query} at {evaluation_ms}" - ); - } -} - -/// A query whose window equals its interval, with no declared phase: one -/// tumbling window per evaluation, maintained at ingestion time, answers -/// every evaluation without raw data. `quantile_over_time(0.99, m[5m])` -/// every 5 minutes at rank error 0.01 (a KLL per series) had no plan. -#[tokio::test] -async fn a_window_equal_to_its_interval_is_kept_at_ingestion_time() { - assert_answers_as_reference_without_raw_data( - "quantile_over_time(0.99, m[5m])", - r#"{"fixed_interval": 300000}"#, - RANK_ERROR_1_PERCENT, - &[0, 300_000, 600_000], - ) - .await; - assert_answers_as_reference_without_raw_data( - "sum by (job) (sum_over_time(m[20s]))", - r#"{"fixed_interval": 20000}"#, - "{}", - &[0, 20_000, 40_000, 60_000], - ) - .await; -} - -/// The same 20 s window every 20 s at phase 5 s: a 20 s window on the -/// epoch grid would end 5 s before every evaluation, so the plan must not -/// read one; it still answers every evaluation as the reference. -#[tokio::test] -async fn a_window_equal_to_its_interval_at_an_offset_phase_answers_as_reference() { - assert_answers_as_reference_without_raw_data( - "sum by (job) (sum_over_time(m[20s]))", - r#"{"fixed_interval_at": {"interval": 20000, "evaluation_phase": 5000}}"#, - "{}", - &[5_000, 25_000, 45_000, 65_000], - ) - .await; -} - -/// Rank error 0.01 with failure probability 0.01, as `requirements`. -const RANK_ERROR_1_PERCENT: &str = r#"{"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "whole_result"}}"#; - -/// Plans `query` with `demand` and `requirements` over 6 ingesting series of `m` sampled every -/// second, for a deployment that keeps no raw data, and runs it at -/// `EVALUATION_MS` plus each of `evaluation_offsets_ms` on a runtime that -/// refreshes each ingestion state once, on the epoch grid of its width. The -/// plan keeps state at ingestion time and reads no raw data at query time; -/// every answer equals the query-time reference. -async fn assert_answers_as_reference_without_raw_data( - query: &str, - demand: &str, - requirements: &str, - evaluation_offsets_ms: &[i64], -) { - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [ - {{"query": {query:?}, "demand": {demand}, "requirements": {requirements}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"# - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - assert!( - views - .iter() - .any(|view| view.refresh() == MaterializedViewRefresh::AtIngestion), - "{views:#?}" - ); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw data at query time" - ); - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 6 minutes before `EVALUATION_MS` to past the last evaluation; - // each value distinct, so a window off by a second answers differently. - let last_ms = EVALUATION_MS + evaluation_offsets_ms.iter().max().unwrap(); - let steps = (last_ms - EVALUATION_MS + 360_000) / 1_000 + 10; - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..steps).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 360_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // The states stored across every evaluation. - let mut store = SummaryStore::new(); - for offset_ms in evaluation_offsets_ms { - let evaluation_ms = EVALUATION_MS + offset_ms; - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert!(!run.answers_per_query[0].is_empty()); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{query} at {evaluation_ms}" - ); - } -} - -/// The same exact query twice, one copy every 10 s and one every 15 s -/// (hash-consed into one memo group), on a deployment that keeps no raw -/// data. A tumbling-window chain is built at ingestion time once per -/// tumbling window, on the grid of its width anchored at the epoch, so a -/// chain either copy reads must have a width dividing that copy's interval: -/// a 15 s chain read every 10 s ends up to 10 s before the evaluation -/// (#1173). Every evaluation of both copies over one minute answers as the -/// query-time reference. -#[tokio::test] -async fn queries_at_two_intervals_read_chains_on_their_own_grids() { - for query in [ - "sum by (job) (rate(m[1m]))", - "sum by (job) (sum_over_time(m[1m]))", - ] { - assert_copies_at_intervals_answer_as_reference(query, &[10_000, 15_000]).await; - } -} - -/// Copies of `query`, copy `q` every `intervals_ms[q]`, over 6 ingesting -/// series of `m` sampled every second, on a deployment that keeps no raw -/// data. Run by a runtime that refreshes each ingestion tumbling window -/// once, on the grid of its width anchored at the epoch, every evaluation -/// of every copy over one minute answers as the query-time reference. -async fn assert_copies_at_intervals_answer_as_reference(query: &str, intervals_ms: &[i64]) { - let entries: Vec = intervals_ms - .iter() - .map(|interval_ms| { - format!( - r#"{{"query": {query:?}, - "demand": {{"fixed_interval": {interval_ms}}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 6.0}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 1000}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - // A deployment that keeps no raw data samples: the windows are views. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(workload, &capabilities).await; - // Samples of `m`, `job` a (instances 1-3) and b (4-6), every second - // from 2 minutes before `EVALUATION_MS` to past the last evaluation; - // each value distinct, so a window off by a second answers differently. - let samples: Vec = (1..=6i64) - .flat_map(|instance| { - (0..200i64).map(move |step| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 120_000 + step * 1_000, - value: (instance * 1000 + step * step % 997) as f64, - }) - }) - .collect(); - // Every evaluation of every copy over one minute, in time order. - let mut evaluations: Vec<(usize, i64)> = intervals_ms - .iter() - .enumerate() - .flat_map(|(query, &interval_ms)| { - (0..=60_000 / interval_ms).map(move |k| (query, EVALUATION_MS + k * interval_ms)) - }) - .collect(); - evaluations.sort_by_key(|&(query, evaluation_ms)| (evaluation_ms, query)); - // The states stored across every evaluation. - let mut store = SummaryStore::new(); - for (copy, evaluation_ms) in evaluations { - let run = run_data_plane_on( - &plan_json, - &samples, - evaluation_ms, - &mut store, - IngestionBuilds::Tumbling, - ); - assert_eq!(run.answers_per_query[copy].len(), 2, "one answer per job"); - assert_eq!( - run.answers_per_query[copy], run.reference_answers_per_query[copy], - "{query}, copy {copy} (every {} ms) at {evaluation_ms}", - intervals_ms[copy] - ); - } -} - -/// A plan file of another DAG wire version is refused, and so is the -/// reference of a tampered plan file, with an error rather than a panic. -#[tokio::test] -async fn a_plan_file_of_another_version_or_tampered_is_refused() { - let plan_json = plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await; - let mut other_version: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - other_version.wire_version += 1; - assert!(matches!( - other_version.validate(), - Err(InstallablePlanError::UnsupportedWireVersion(_)) - )); - let mut tampered: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - tampered.query_texts.clear(); - assert!(matches!( - asap_planner::test_support::query_time_reference(&tampered), - Err(InstallablePlanError::QueriesDoNotMatchRoots { .. }) - )); - let mut missing_node: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - missing_node.dag.nodes.remove(0); - assert!(matches!( - asap_planner::test_support::query_time_reference(&missing_node), - Err(InstallablePlanError::InvalidDag(_)) - )); -} - /// `sum(latency_ms)` reads an instant selector: per series, only the latest /// sample in its 1 s lookback counts. Series `a` has samples at T-900 ms (1) /// and T-100 ms (10), series `b` at T-200 ms (100): the sum is 10 + 100. @@ -1217,682 +335,21 @@ async fn an_instant_selector_reads_the_latest_sample_per_series() { assert_eq!(answered[0].1, 110.0); } -/// A deployment that keeps raw data samples and query-time results across -/// evaluations (#509 Example 4's B3), with a dashboard panel -/// (`max_over_time(latency_ms[5m])` every minute over 1M series sampled -/// every 1.5 s, answered as the last hour at 1-minute steps): Stage 3 -/// keeps five 1-minute tumbling windows refreshed at query time, and keeps raw data -/// for refreshing the oldest tumbling window at the oldest step (5 min + 1 h). The first -/// evaluation, with an empty store, refreshes all five in its query run; the -/// next, a minute later, reuses the four still in its window and refreshes -/// only the newest. Each -/// evaluation's answers (each series' newest-tumbling-window maximum) equal the -/// reference. -#[tokio::test] -async fn query_time_results_kept_across_evaluations_are_built_in_query_runs() { - let capabilities = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "max_over_time(latency_ms[5m])", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}, - "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 666666.67}}, - "sources": {"latency_ms": {"series_count": 1000000, "sample_interval": 1500}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = installed.materialized_views().unwrap(); - assert_eq!(views.len(), 5); - assert!(views - .iter() - .all(|v| v.refresh() == MaterializedViewRefresh::OnFirstRead - && v.inputs.raw_data_windows()[0].width_ms() == 60_000)); - // Raw data is kept for refreshing the oldest tumbling window at the oldest step: - // the five tumbling windows' 5 minutes plus the hour of steps, as long as the tumbling windows - // themselves are kept. - let tumbling_window_lookback_ms = views - .iter() - .flat_map(|v| v.inputs.raw_data_windows()) - .map(asap_types::test_support::lookback_ms) - .max(); - assert_eq!(tumbling_window_lookback_ms, Some(300_000)); - assert!(views.iter().all(|v| v.retention_ms() == 3_900_000)); - let retention = installed.raw_data_retention().unwrap(); - assert_eq!( - retention - .iter() - .map(|r| r.lookback_ms()) - .collect::>(), - vec![Some(3_900_000)], - "{retention:?}" - ); - // The tumbling window cache the data plane keeps across evaluations. - let mut store = SummaryStore::new(); - let first = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut store, - ); - assert_eq!(first.outputs_built, 5); - assert_answers_from_tumbling_window(&first.answers_per_query[0], 0); - assert_eq!(first.answers_per_query, first.reference_answers_per_query); - let next = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS + 60_000, - &mut store, - ); - assert_eq!( - next.outputs_built, 1, - "four tumbling windows reused from the cache" - ); - assert_eq!(next.answers_per_query, next.reference_answers_per_query); -} - -/// An exact per-series quantile under a group average (issue #770), on a -/// deployment that keeps raw data samples: Stage 3 selects the exact plan, which -/// reads each series' raw data samples in the window at query time, and the -/// executor runs it. Series `a` (1..=10) and `b` (10, 20, ..., 100) are in -/// `east`, `c` (constantly 5) in `west`; a sample of each before the window -/// is not read. Per series, p99 interpolates between its two largest -/// samples (rank 0.99 × 9 = 8.91): 9.91 for `a`, 99.1 for `b`, 5 for `c`. -#[tokio::test] -async fn an_exact_per_series_quantile_runs_from_raw_data_samples() { - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "avg by (region) (quantile_over_time(0.99, latency_ms[100s]))", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 3.0}}, - "sources": {"latency_ms": {"series_count": 3, "sample_interval": 10000}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.materialized_views().unwrap().is_empty(), - "the exact plan" - ); - assert_eq!(installed.raw_data_retention().unwrap().len(), 1); - // Ten samples per series in the window, one every 10 s, and one before it. - let mut samples = Vec::new(); - for (instance, region, value) in [ - ("a", "east", (|step| step as f64) as fn(i64) -> f64), - ("b", "east", |step| 10.0 * step as f64), - ("c", "west", |_| 5.0), - ] { - for step in 0..=10 { - let mut sample = sample( - "latency_ms", - instance, - EVALUATION_MS - 105_000 + 10_000 * step, - if step == 0 { 1e9 } else { value(step) }, - ); - sample - .labels - .insert("region".to_string(), region.to_string()); - samples.push(sample); - } - } - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 2, "{answered:?}"); - for ((labels, value), (region, expected)) in answered - .iter() - .zip([("east", (9.91 + 99.1) / 2.0), ("west", 5.0)]) - { - assert_eq!(labels["region"], region); - assert!((value - expected).abs() < 1e-9, "{region}: {value}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - // ── A ratio of two relative-error quantiles (issue #852) ──────────────────── -/// Repeating `queries`, each every 10 minutes at error 0.05 in each -/// summary's own metric (W-RangeHeavy leaves the metric unset) and δ 0.01, -/// over 1,000 ingesting non-negative `latency_ms` gauges sampled -/// every second. -fn relative_workload(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 600000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.01, - "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, - "sources": {{"latency_ms": {{"series_count": 1000, "sample_interval": 1000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// The DDSketch α of every sketch `plan` builds; panics on any other sketch. -fn ddsketch_alphas(plan: &InstallablePlan) -> Vec { - plan.dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - } - | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => match kind.params() { - SketchParams::DDSketch { alpha } => Some(*alpha), - other => panic!("not a DDSketch: {other:?}"), - }, - _ => None, - }) - .collect() -} +// ── Materialized views refreshed from materialized views (issue #781) ────────────────── -/// PromQL `quantile_over_time(q, …)` of `values`: linear interpolation -/// between the sorted values at rank `q·(n − 1)`. -fn promql_quantile(q: f64, values: &mut [f64]) -> f64 { - values.sort_by(f64::total_cmp); - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) +/// The time selection `(from_ms, to_ms]` relative to the evaluation. +fn relative_time(from_ms: i64, to_ms: i64) -> Vec { + vec![SelectionBox::new( + Default::default(), + Some((Bound::Excluded(from_ms), Bound::Included(to_ms))), + )] } -/// The p90/p50 ratio of an hour (W-RangeHeavy template 8) on a deployment -/// that keeps no raw data samples: both quantiles come from DDSketches -/// sized for a share of the target (α = 1 − 1.05^{−1/2} ≈ 0.0241, so -/// `(a + b)/(1 − b) ≤ 0.05`), maintained in tumbling windows; nothing is read from -/// raw data at query time. Run on data, each series' ratio is within 0.05 -/// of the exact ratio of its hour's samples. -#[tokio::test] -async fn a_ratio_of_two_quantiles_runs_from_ddsketches_within_its_relative_target() { - let query = - "quantile_over_time(0.9, latency_ms[60m]) / quantile_over_time(0.5, latency_ms[60m])"; - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for(relative_workload(&[query]), &capabilities).await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.raw_data_retention().unwrap().is_empty(), - "no raw scan" - ); - let alphas = ddsketch_alphas(&installed); - assert!(!alphas.is_empty(), "DDSketch states"); - let share = 1.0 - 1.05f64.powf(-0.5); - for alpha in &alphas { - assert!(*alpha <= share + 1e-12, "α {alpha} > {share}"); - } - let samples = remote_write_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), 2, "{answered:?}"); - for (labels, value) in answered { - let mut hour: Vec = samples - .iter() - .filter(|s| { - s.labels["__name__"] == "latency_ms" - && s.labels["instance"] == labels["instance"] - && s.timestamp_ms > EVALUATION_MS - 3_600_000 - && s.timestamp_ms <= EVALUATION_MS - }) - .map(|s| s.value) - .collect(); - let exact = promql_quantile(0.9, &mut hour) / promql_quantile(0.5, &mut hour); - let error = (value - exact).abs() / exact; - assert!(error <= 0.05, "{labels:?}: {value} vs {exact} ({error})"); - } -} - -/// The ratio's numerator is also asked for on its own (W-RangeHeavy -/// templates 5 and 8): the p90 estimate is shared, so its state is sized -/// for its strictest reader, the ratio's share; the plan still reads no raw -/// data samples. -#[tokio::test] -#[ignore = "slow without a planning budget: over 30 min in a debug build; #1197"] -async fn a_quantile_shared_with_a_ratio_is_sized_for_the_ratio() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan_json = plan_json_for( - relative_workload(&[ - "quantile_over_time(0.9, latency_ms[60m])", - "quantile_over_time(0.9, latency_ms[60m]) / quantile_over_time(0.5, latency_ms[60m])", - ]), - &capabilities, - ) - .await; - let installed: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert!( - installed.raw_data_retention().unwrap().is_empty(), - "no raw scan" - ); - let share = 1.0 - 1.05f64.powf(-0.5); - for alpha in ddsketch_alphas(&installed) { - assert!(alpha <= share + 1e-12, "α {alpha} > {share}"); - } - let run = run_data_plane( - &plan_json, - &remote_write_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -// ── Materialized views refreshed from materialized views (issue #781) ────────────────── - -/// Stage 1 to Stage 3, then every node `store` names stored as it says, -/// the others as Stage 2 chose, and the plan timed again: how a test makes -/// the plan store a shape Stage 3 does not select today. -struct StoredAsForced { - store: fn(&OperatorNode) -> Option, -} - -impl OptimizationPass for StoredAsForced { - fn name(&self) -> &'static str { - "stored-as-forced" - } - - fn optimize(&self, input: OptimizationInput<'_>) -> Result { - let mut output = Stage1ToStage3Pass.optimize(input)?; - // Each node untimed once, so shared inputs stay shared, and its - // materialization in `assignment`. - let mut untimed: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut assignment = MaterializationAssignment::default(); - let roots: Vec> = output - .plans - .iter() - .map(|plan| untime(&plan.root, self.store, &mut untimed, &mut assignment)) - .collect(); - // A node read both at ingestion and at query time is copied, one - // copy per timing, as Stage 2 does. - let (roots, assignment) = split_shared_by_timing(&roots, &assignment); - let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - for (plan, root) in output.plans.iter_mut().zip(timed) { - let QueryRoot::Operator(root) = root else { - unreachable!("operator roots in, operator roots out") - }; - plan.root = root; - } - Ok(output) - } -} - -/// `node` untimed, with `store`'s materialization (or its own) recorded in -/// `assignment` for every node of its sub-DAG. -fn untime( - node: &Rc, - store: fn(&OperatorNode) -> Option, - untimed: &mut HashMap<*const OperatorNode, Rc>, - assignment: &mut MaterializationAssignment, -) -> Rc { - if let Some(done) = untimed.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|child| untime(child, store, untimed, assignment)); - copy.timing = None; - copy.materialization = Materialization::NotMaterialized; - let copy = Rc::new(copy); - // A shared non-summary node Stage 2 built once per evaluation is not - // kept: a build forced to ingestion time above it cannot read a - // query-time result. - let own = match node.asap() { - Some( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::SummaryMerge { .. }, - ) => node.materialization, - _ => Materialization::NotMaterialized, - }; - let materialization = store(node).unwrap_or(own); - if materialization != Materialization::NotMaterialized { - assignment.set(©, materialization); - } - untimed.insert(Rc::as_ptr(node), Rc::clone(©)); - copy -} - -/// Every `SummaryAgg` built at ingestion time. -fn summaries_at_ingestion(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })) - .then_some(Materialization::MatAtIngestion(Medium::Memory)) -} - -/// Every `SummaryMerge` built at query time and kept across evaluations. -fn merges_kept(node: &OperatorNode) -> Option { - matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })).then_some( - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ) -} - -/// Exact PromQL `queries`, every minute, over 6 ingesting series of `m` -/// sampled every 10 s; `tail` is appended to each entry's JSON (e.g. its -/// range evaluation). -fn exact_workload(queries: &[&str], tail: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}{tail}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, - "sources": {{"m": {{"series_count": 6, "sample_interval": 10000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// The plan file of `workload` on `capabilities`, with what `store` names -/// stored. -async fn forced_plan_json( - workload: &PlanningWorkload, - capabilities: &DeploymentCapabilities, - store: fn(&OperatorNode) -> Option, -) -> String { - let pass = StoredAsForced { store }; - let mut input = UserInput::new( - workload, - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(capabilities), - ); - input.pass = Some(&pass); - serde_json::to_string(&plan_installable(input).await.unwrap().0).unwrap() -} - -/// Samples of `m` for six series, `job` a (instances 1, 2, 3) and b (4, 5, -/// 6), every 10 s over the last 10 minutes; each value distinct. -fn job_samples() -> Vec { - let mut samples = Vec::new(); - for instance in 1..=6 { - for step in 0..60 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "m".to_string()), - ( - "job".to_string(), - if instance <= 3 { "a" } else { "b" }.to_string(), - ), - ("instance".to_string(), instance.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 595_000 + step * 10_000, - value: (instance * 1000 + (step * 37) % 101) as f64, - }); - } - } - samples -} - -/// The time selection `(from_ms, to_ms]` relative to the evaluation. -fn relative_time(from_ms: i64, to_ms: i64) -> Vec { - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))), - )] -} - -/// A per-group maximum and a per-group sum (`max by (job)` and -/// `sum by (job, instance)`), both refreshed at ingestion from one stored -/// per-series sum: Stage 1 shares the per-series sums the two queries read -/// (over scans that project other labels). The per-series sum is listed -/// first, then each group output, which reads only it, with its time -/// selection under a definition of its own; refreshed in that order from -/// stored inputs, they answer as the query-time reference. (A coarse sum -/// in place of the maximum rolls up from the fine sum instead: -/// [`a_coarse_sum_rolls_up_from_the_stored_fine_sum`].) -#[tokio::test] -async fn group_outputs_from_one_stored_per_series_sum_are_listed_and_built() { - let workload = exact_workload( - &[ - "max by (job) (sum_over_time(m[1m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [per_series, sums @ ..] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert_eq!(sums.len(), 2, "{:#?}", views); - assert!(per_series.inputs.materialized_views.is_empty()); - assert_eq!(per_series.inputs.raw_data_windows().len(), 1); - for sum in sums { - assert_eq!(sum.inputs.materialized_views, [per_series.node_id]); - assert!(sum.inputs.raw_data_windows().is_empty()); - assert_ne!( - sum.coverage().definition_id(), - per_series.coverage().definition_id() - ); - } - assert_ne!( - sums[0].coverage().definition_id(), - sums[1].coverage().definition_id() - ); - for output in &views { - assert_eq!(output.refresh(), MaterializedViewRefresh::AtIngestion); - assert_eq!(output.coverage().selection(), relative_time(-60_000, 0)); - assert_eq!(output.retention_ms(), 60_000); - } - assert!(plan.raw_data_retention().unwrap().is_empty()); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 3); - assert_eq!(run.answers_per_query[0].len(), 2, "one maximum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one sum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A coarse and a fine sum (`sum by (job)` and `sum by (job, instance)`): -/// the fine grouping is priced at no more than the 6 series, as the -/// per-series sum is, so the coarse sum rolls up from it and the plan -/// stores two outputs, the second reading only the first. Built in that -/// order, they answer as the query-time reference. -#[tokio::test] -async fn a_coarse_sum_rolls_up_from_the_stored_fine_sum() { - let workload = exact_workload( - &[ - "sum by (job) (sum_over_time(m[1m]))", - "sum by (job, instance) (sum_over_time(m[1m]))", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [first, rolled_up] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert!(first.inputs.materialized_views.is_empty()); - assert_eq!(rolled_up.inputs.materialized_views, [first.node_id]); - assert!(rolled_up.inputs.raw_data_windows().is_empty()); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 2); - assert_eq!(run.answers_per_query[0].len(), 2, "one sum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one sum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A maximum across series (`max by (job)`) refreshed at ingestion from the -/// stored per-series maxima, which the per-series query reads too: listed -/// after them with their time selection, under a definition of its own. -#[tokio::test] -async fn a_summary_across_series_from_per_series_summaries_is_listed_and_built() { - let workload = exact_workload( - &[ - "max by (job) (max_over_time(m[1m]))", - "max_over_time(m[1m])", - ], - "", - ); - let plan_json = forced_plan_json( - &workload, - &DeploymentCapabilities::default(), - summaries_at_ingestion, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let [per_series, across] = views.as_slice() else { - panic!("{:#?}", views); - }; - assert!(per_series.inputs.materialized_views.is_empty()); - assert_eq!(across.inputs.materialized_views, [per_series.node_id]); - assert!(across.inputs.raw_data_windows().is_empty()); - for output in [per_series, across] { - assert_eq!(output.refresh(), MaterializedViewRefresh::AtIngestion); - assert_eq!(output.coverage().selection(), relative_time(-60_000, 0)); - } - assert_ne!( - across.coverage().definition_id(), - per_series.coverage().definition_id() - ); - let run = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.outputs_built, 2); - assert_eq!(run.answers_per_query[0].len(), 2, "one maximum per job"); - assert_eq!(run.answers_per_query[1].len(), 6, "one maximum per series"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// A dashboard's 5-minute max over the last hour at 1-minute steps (#509 -/// Example 4's B3) keeps five 1-minute tumbling windows; with its merge kept too, the -/// merge is listed after the tumbling windows it reads, with their definition and the -/// union of their times, refreshed in the query run and kept for its window -/// plus the hour of steps; its tumbling windows are kept at least that long. Built -/// from the stored tumbling windows it answers as the reference, and a minute later -/// only the newest tumbling window and the new merge are refreshed. -#[tokio::test] -async fn a_kept_sliding_window_merge_is_listed_and_built() { - let capabilities = DeploymentCapabilities { - query_time_results_retained: true, - ..DeploymentCapabilities::default() - }; - let mut workload = exact_workload( - &["max_over_time(m[5m])"], - r#", "range_evaluation": {"step_ms": 60000, "span_ms": 3600000}"#, - ); - // Declared as sampled every second, so that keeping the tumbling windows is - // cheaper than rebuilding the window at every step; the samples the - // run reads are every 10 s, and exact answers do not depend on it. - let facts = workload - .data_workload - .as_mut() - .and_then(|data| data.sources.get_mut("m")) - .unwrap(); - facts.sample_interval = Some(asap_types::workload::DurationMs(1_000)); - let plan_json = forced_plan_json(&workload, &capabilities, merges_kept).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let views = plan.materialized_views().unwrap(); - let (merge, tumbling_windows) = views.split_last().unwrap(); - assert_eq!(tumbling_windows.len(), 5); - assert_eq!( - merge.inputs.materialized_views, - tumbling_windows - .iter() - .map(|tumbling_window| tumbling_window.node_id) - .collect::>() - ); - assert!(merge.inputs.raw_data_windows().is_empty()); - assert_eq!(merge.refresh(), MaterializedViewRefresh::OnFirstRead); - assert_eq!(merge.coverage().selection(), relative_time(-300_000, 0)); - assert_eq!(merge.retention_ms(), 300_000 + 3_600_000); - for (i, tumbling_window) in tumbling_windows.iter().enumerate() { - let i = i as i64; - assert_eq!( - tumbling_window.coverage().definition_id(), - merge.coverage().definition_id() - ); - assert_eq!( - tumbling_window.coverage().selection(), - relative_time(-(i + 1) * 60_000, -i * 60_000) - ); - assert!(tumbling_window.retention_ms() >= merge.retention_ms()); - } - let mut store = SummaryStore::new(); - let first = run_data_plane(&plan_json, &job_samples(), EVALUATION_MS, &mut store); - assert_eq!(first.outputs_built, 6); - assert_eq!(first.answers_per_query, first.reference_answers_per_query); - let next = run_data_plane( - &plan_json, - &job_samples(), - EVALUATION_MS + 60_000, - &mut store, - ); - assert_eq!( - next.outputs_built, 2, - "the newest tumbling window and the new merge" - ); - assert_eq!(next.answers_per_query, next.reference_answers_per_query); -} - -/// Example 4b's selected tumbling windows cover the hour and share one definition id, -/// each with its own time selection; each reads its tumbling window of -/// raw data samples and nothing stored. +/// Example 4b's selected tumbling windows cover the hour and share one definition id, +/// each with its own time selection; each reads its tumbling window of +/// raw data samples and nothing stored. #[tokio::test] async fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { let plan: InstallablePlan = @@ -1930,124 +387,6 @@ async fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() } } -/// PromQL `topk` returns whole input series: `topk by (region) (2, -/// sum_over_time(data[1m]))` answers, per region, the two series with the -/// largest sums, each with its own full label set (region, service and -/// instance; `sum_over_time` drops the metric name), not only `region`. -/// Series `e1`..`e3` are in `east` with sums 6, 60 and 30; `w1`, `w2` in -/// `west` with sums 9 and 3. -#[tokio::test] -async fn topk_answers_carry_the_chosen_series_labels() { - let workload: PlanningWorkload = serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "topk by (region) (2, sum_over_time(data[1m]))", - "demand": {"fixed_interval": 60000}, - "requirements": {}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 0.5}}, - "sources": {"data": {"series_count": 5, "sample_interval": 10000}}}}"#, - ) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let samples = topk_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_full_series_answers(&run, 4); - let labels = |instance: &str, region: &str, service: &str| { - BTreeMap::from([ - ("instance".to_string(), instance.to_string()), - ("region".to_string(), region.to_string()), - ("service".to_string(), service.to_string()), - ]) - }; - let mut answered = run.answers_per_query[0].clone(); - answered.sort_by(|a, b| a.0.cmp(&b.0)); - let mut expected = vec![ - (labels("e2", "east", "web"), 60.0), - (labels("e3", "east", "db"), 30.0), - (labels("w1", "west", "api"), 9.0), - (labels("w2", "west", "web"), 3.0), - ]; - expected.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!(answered.len(), expected.len(), "{answered:?}"); - for ((labels, value), (want_labels, want)) in answered.iter().zip(&expected) { - assert_eq!(labels, want_labels); - assert!((value - want).abs() < 1e-9, "{labels:?}: {value}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Samples of `data` for `topk_answers_carry_the_chosen_series_labels`: -/// three per series inside the last minute, each a third of its sum. -fn topk_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, region, service, sum) in [ - ("e1", "east", "api", 6.0), - ("e2", "east", "web", 60.0), - ("e3", "east", "db", 30.0), - ("w1", "west", "api", 9.0), - ("w2", "west", "web", 3.0), - ] { - // In time order, as a counter's samples arrive. - for step in (1..=3).rev() { - let mut sample = sample("data", instance, EVALUATION_MS - step * 15_000, sum / 3.0); - sample.labels.insert("region".into(), region.into()); - sample.labels.insert("service".into(), service.into()); - samples.push(sample); - } - } - samples -} - -/// The first query's `expected` answers, each a whole input series (it -/// carries its `instance`), and the same as the query-time reference's. -fn assert_full_series_answers(run: &DataPlaneRun, expected: usize) { - let answered = &run.answers_per_query[0]; - assert_eq!(answered.len(), expected, "{answered:?}"); - for (labels, _) in answered { - assert!(labels.contains_key("instance"), "{labels:?}"); - assert!(labels.contains_key("service"), "{labels:?}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Every top-k and bottom-k form, grouped or not, over sums, counts and -/// rates, answers whole input series: `topk(2, ...)` the two of five, -/// `by (region)` per region, `bottomk` likewise. -#[tokio::test] -async fn every_topk_form_answers_whole_series() { - for (query, expected) in [ - ("topk(2, sum_over_time(data[1m]))", 2), - ("topk by (region) (1, count_over_time(data[1m]))", 2), - ("topk by (region) (1, rate(data[1m]))", 2), - ("bottomk by (region) (1, sum_over_time(data[1m]))", 2), - ("bottomk(3, sum_over_time(data[1m]))", 3), - ] { - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval": 60000}}, - "requirements": {{}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.5}}}}, - "sources": {{"data": {{"series_count": 5, "sample_interval": 10000}}}}}}}}"# - )) - .unwrap(); - let plan_json = plan_json_for(workload, &DeploymentCapabilities::default()).await; - let run = run_data_plane( - &plan_json, - &topk_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_full_series_answers(&run, expected); - } -} - /// The MVP benchmark's p99 query: the per-series p99 of a gauge over 100 s, /// averaged per (region, service, job), every second, within 0.05 at /// failure probability 0.3, over 1,000 `data` series sampled every 100 ms, @@ -2059,469 +398,12 @@ fn p99_avg_workload(values_non_negative: bool) -> PlanningWorkload { "demand": {{"fixed_interval": 1000}}, "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3, "accuracy_scope": "whole_result"}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, - "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# - )) - .unwrap() -} - -/// [`p99_avg_workload`] planned for a deployment that keeps no raw data samples. -async fn plan_p99_avg( - values_non_negative: bool, -) -> Result<(InstallablePlan, Option), String> { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_installable(UserInput::new( - &p99_avg_workload(values_non_negative), - FrontendInput::Promql, - PlanningModels::builtin().with_deployment_capabilities(&capabilities), - )) - .await - .map_err(|error| error.to_string()) -} - -/// Without raw data samples, the only valid plan of the p99 query keeps one -/// DDSketch per series in one-second tumbling windows built at ingestion and merges -/// the window's tumbling windows at query time. A quantile -/// of merged tumbling windows is still one of the samples, so a declared non-negative -/// gauge carries the sketch's relative bound through `avg`: that plan is -/// selected, its answers equal the query-time reference's and are within -/// 5% of the exact averages. -#[tokio::test] -async fn a_non_negative_gauges_merged_ddsketch_tumbling_windows_carry_the_relative_bound_through_avg( -) { - let (plan, selection) = plan_p99_avg(true).await.expect("a valid plan"); - let selection = selection.as_ref().expect("Stage 3's report"); - let views = plan.materialized_views().unwrap(); - assert!(selection.cost_per_second.is_some()); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - assert_eq!( - stored_definitions(&plan, "DDSketch").len(), - 1, - "one DDSketch" - ); - assert!( - plan.raw_data_retention().unwrap().is_empty(), - "no raw samples read" - ); - // The query's guarantee, with `data`'s declared sign and its 1,000 - // series bound as Stage 3 binds them: a relative error of at most 0.05 - // at failure probability at most 0.3 (DDSketch's bound is - // deterministic). - let root = plan.dag.roots[0]; - let guarantee = plan - .dag - .nodes - .iter() - .find(|node| node.id == root) - .and_then(|node| node.guarantee.clone()) - .expect("the query's guarantee"); - let sign = asap_accuracy::test_support::raw_data_samples_sign_statistic("data"); - let bound = guarantee.bind(&|statistic| match statistic { - "input_row_count" => Some(1_000.0), - _ => (statistic == sign).then_some(1.0), - }); - assert_eq!( - bound.spec.accuracy_metric, - asap_types::ir::properties::AccuracyMetric::Relative - ); - let error = bound.spec.error.evaluate().expect("a numeric bound"); - assert!(error <= 0.05, "{error}"); - let failure = bound - .spec - .failure_probability - .evaluate() - .expect("a numeric failure probability"); - assert!(failure <= 0.3, "{failure}"); - - // Eight non-negative series, one sample every 100 ms over the last - // 110 s: two per (region, service, job) group. - let evaluation_ms = EVALUATION_MS; - let mut samples = Vec::new(); - let mut per_group: BTreeMap<(String, String, String), Vec>> = BTreeMap::new(); - for series in 0..8 { - let region = format!("r{}", series % 2); - let service = format!("s{}", series / 4); - let job = "j".to_string(); - let mut window = Vec::new(); - for step in 0..1_100i64 { - let timestamp_ms = evaluation_ms - 110_000 + 50 + step * 100; - let value = 10.0 * (series + 1) as f64 + ((step * 37 + series * 11) % 997) as f64; - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "data".to_string()), - ("region".to_string(), region.clone()), - ("service".to_string(), service.clone()), - ("job".to_string(), job.clone()), - ("instance".to_string(), format!("i{series}")), - ]), - timestamp_ms, - value, - }); - if timestamp_ms > evaluation_ms - 100_000 { - window.push(value); - } - } - per_group - .entry((region, service, job)) - .or_default() - .push(window); - } - let run = run_data_plane( - &serde_json::to_string(&plan).unwrap(), - &samples, - evaluation_ms, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - let answers = &run.answers_per_query[0]; - assert_eq!(answers.len(), per_group.len()); - for (labels, value) in answers { - let key = ( - labels["region"].clone(), - labels["service"].clone(), - labels["job"].clone(), - ); - // The exact answer: the mean of the group's per-series p99s, each - // PromQL's interpolated quantile of the series' window. - let exact = per_group[&key] - .iter() - .map(|window| { - let mut sorted = window.clone(); - sorted.sort_by(f64::total_cmp); - let rank = 0.99 * (sorted.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) - }) - .sum::() - / per_group[&key].len() as f64; - assert!( - (value - exact).abs() <= 0.05 * exact, - "{labels:?}: {value} vs exact {exact}" - ); - } -} - -/// Without a non-negative declaration, relative sketch error cannot propagate -/// through the average, so this no-raw deployment has no valid plan: plan -/// selection selects none. Pass 1 prunes the quantile sketches for that -/// reason (`NoErrorBoundForQueryTarget`) and Stage 2 the exact candidates, -/// which scan raw data at query time. -#[tokio::test] -async fn an_undeclared_gauges_merged_ddsketch_tumbling_windows_get_no_relative_bound_through_avg() { - let error = plan_p99_avg(false).await.expect_err("no valid plan"); - assert_plan_selection_selected_none(&error); - // Pass 1 prunes both quantile sketches for that reason: DDSketch's - // relative bound does not pass through the average of an undeclared - // gauge, and no rule carries KLL's rank bound through it. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - let no_bound = pass1_pruned_for_no_error_bound(&p99_avg_workload(false), models); - assert!( - no_bound.contains(&"DDSketch".to_string()) && no_bound.contains(&"Kll".to_string()), - "{no_bound:?}" - ); - // Declared non-negative, DDSketch's bound passes through the average. - let no_bound = pass1_pruned_for_no_error_bound(&p99_avg_workload(true), models); - assert!(!no_bound.contains(&"DDSketch".to_string()), "{no_bound:?}"); -} - -/// The summaries Pass 1 pruned planning `workload` (PromQL) with `models` -/// because no query reading them can meet its target -/// ([`PruneReason::NoErrorBoundForQueryTarget`]), as the facade's planning -/// run reports them (`PlanningRun::per_computation_candidates`, its -/// `pruned_candidates`): each family's name (e.g. `"Kll"`). -fn pass1_pruned_for_no_error_bound( - workload: &PlanningWorkload, - models: PlanningModels<'_>, -) -> Vec { - let roots = asap_frontend_promql::lower_promql_query_workload(workload) - .unwrap() - .into_iter() - .map(|root| match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(with_promql_series_identity(&node).unwrap_or(node)) - } - scalar => scalar, - }) - .collect(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let data = workload.data_workload.clone().unwrap_or_default(); - let run = run_planning( - roots, - &entries, - &data, - models, - PlanSearchSettings { inspect: true }, - ) - .unwrap(); - run.per_computation_candidates - .expect("Pass 1's output") - .pruned_candidates - .iter() - .filter(|pruned| pruned.reason == PruneReason::NoErrorBoundForQueryTarget) - .filter_map(|pruned| match &pruned.candidate { - HowToComputeAggregateMeasure::Summary(family) => Some(format!("{family:?}")), - _ => None, - }) - .collect() -} - -/// `error` says plan selection found no valid plan: no candidate is valid. -/// Not a solver, realizer or input failure. -fn assert_plan_selection_selected_none(error: &str) { - assert!( - error.contains("plan selection: no valid candidate"), - "{error}" - ); -} - -/// `avg by (g) (quantile_over_time(0.99, x[100s]))` every 10 s at rank -/// error 0.01 and δ 0.01, over 100 ingesting `x` series sampled every -/// second: ASAPQuery's random mixed workload query. Stage 1 has no -/// analytical rule for an average of quantile estimates, so every sketch -/// candidate has no guarantee. -fn average_of_p99s_workload() -> PlanningWorkload { - serde_json::from_str( - r#"{"query_workload": {"language": "promql", "repeating_queries": [{ - "query": "avg by (g) (quantile_over_time(0.99, x[100s]))", - "demand": {"fixed_interval": 10000}, - "requirements": {"accuracy": {"accuracy_metric": "rank", "max_error": 0.01, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}, - "predictability": {"predictable": {"known_at": null}}}]}, - "data_workload": {"arrival": {"ingesting": {"rows_per_second": 100.0}}, - "sources": {"x": {"series_count": 100, "sample_interval": 1000}}}}"#, - ) - .unwrap() -} - -/// Raw `x` samples of series `a` and `b` in group `g="1"` and `c` in group -/// `g="2"`, one every second from 200 s before `EVALUATION_MS` to it, off -/// the second boundaries: series `i`'s value is `100 × i + 1..=101`, all -/// positive, which DDSketch needs at run time (nothing declares it here). -fn grouped_samples() -> Vec { - let mut samples = Vec::new(); - for (i, (instance, g)) in [("a", "1"), ("b", "1"), ("c", "2")].iter().enumerate() { - for step in 0..200 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "x".to_string()), - ("instance".to_string(), instance.to_string()), - ("g".to_string(), g.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 199_500 + step * 1_000, - value: (100 * i) as f64 + 1.0 + ((step * 37 + 11 * i as i64) % 101) as f64, - }); - } - } - samples -} - -/// Quantile queries over one metric and window with different `by` labels -/// (ASAPQuery's seed-862 quantile queries) share one per-series DDSketch: -/// planned jointly on a deployment that keeps no raw data samples, every stored -/// DDSketch has one definition, and each query answers as its query-time -/// reference. Series of `data` carry `region`, `service` and `job`; values -/// are positive. -#[tokio::test] -async fn quantile_queries_with_different_groupings_share_one_per_series_sketch() { - let plan_json = jointly_planned( - &[ - "avg by (region) (quantile_over_time(0.99, data[100s]))", - "avg by (region, service, job) (quantile_over_time(0.9, data[100s]))", - "avg by (service) (quantile_over_time(0.5, data[100s]))", - ], - r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#, - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let sketches = stored_definitions(&plan, "DDSketch"); - assert_eq!(sketches.len(), 1, "{sketches:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), 2, "one answer per region"); - assert_eq!(run.answers_per_query[1].len(), 6); - assert_eq!(run.answers_per_query[2].len(), 3, "one answer per service"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// The six series of `data` the sharing tests read: `region` east/west, -/// `service` api/web/db, `job` batch/online; values positive, one every -/// 10 s over the last two minutes, in time order. -fn grouped_series_samples() -> Vec { - let mut samples = Vec::new(); - for (instance, region, service, job) in [ - ("s1", "east", "api", "online"), - ("s2", "east", "web", "online"), - ("s3", "east", "db", "batch"), - ("s4", "west", "api", "online"), - ("s5", "west", "web", "batch"), - ("s6", "west", "db", "batch"), - ] { - let base: f64 = instance[1..].parse().unwrap(); - for step in (0..12).rev() { - let mut sample = sample( - "data", - instance, - EVALUATION_MS - 5_000 - step * 10_000, - base * 10.0 + ((step * 37) % 11) as f64, - ); - sample.labels.insert("region".into(), region.into()); - sample.labels.insert("service".into(), service.into()); - sample.labels.insert("job".into(), job.into()); - samples.push(sample); - } - } - samples -} - -/// The exact answer of `avg by (g) (quantile_over_time(0.99, x[100s]))` at -/// `EVALUATION_MS` over [`grouped_samples`]: PromQL's p99 (linear -/// interpolation at rank `0.99 × (n − 1)`) of each series' samples in -/// `(EVALUATION_MS − 100 s, EVALUATION_MS]`, averaged per group. -fn exact_average_of_p99s(samples: &[RawDataSample]) -> BTreeMap { - let mut per_series: BTreeMap<(String, String), Vec> = BTreeMap::new(); - for s in samples - .iter() - .filter(|s| EVALUATION_MS - 100_000 < s.timestamp_ms && s.timestamp_ms <= EVALUATION_MS) - { - per_series - .entry((s.labels["g"].clone(), s.labels["instance"].clone())) - .or_default() - .push(s.value); - } - let mut per_group: BTreeMap> = BTreeMap::new(); - for ((g, _), mut values) in per_series { - values.sort_by(f64::total_cmp); - let rank = 0.99 * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - let p99 = values[low] + (values[high] - values[low]) * (rank - low as f64); - per_group.entry(g).or_default().push(p99); - } - per_group - .into_iter() - .map(|(g, p99s)| (g, p99s.iter().sum::() / p99s.len() as f64)) - .collect() -} - -/// Issue #774: without raw data samples retained, every candidate of an average -/// of p99s fails a check (the exact ones scan raw data samples at query time, -/// the sketch ones have no guarantee), so planning fails. With -/// `PlanningModels::ignore_accuracy_guarantees`, the cheapest -/// sketch plan is selected, the report marks its output as accepted without -/// a guarantee, and it runs: equal to its query-time reference and within a -/// loose tolerance of the exact answer. -#[tokio::test] -async fn ignoring_accuracy_guarantees_selects_a_sketch_plan_that_runs() { - let workload = average_of_p99s_workload(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let plan = |ignore: bool| { - let workload = workload.clone(); - let capabilities = capabilities.clone(); - async move { - plan_installable(UserInput::new( - &workload, - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_ignore_accuracy_guarantees(ignore), - )) - .await - } - }; - // Without the setting, no candidate is valid. - let error = plan(false) - .await - .expect_err("no candidate is valid without the setting"); - assert_plan_selection_selected_none(&error.to_string()); - // Pass 1 prunes every quantile sketch for it: no rule carries a rank - // bound through the average, and a relative one does not meet a rank - // target. With the setting, Stage 3 may accept an output without its - // guarantee, so Pass 1 prunes none. - let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); - let no_bound = pass1_pruned_for_no_error_bound(&workload, models); - assert!( - no_bound.contains(&"DDSketch".to_string()) && no_bound.contains(&"Kll".to_string()), - "{no_bound:?}" - ); - assert_eq!( - pass1_pruned_for_no_error_bound(&workload, models.with_ignore_accuracy_guarantees(true)), - Vec::::new() - ); - // With it. - let (installed, report) = plan(true).await.expect("a sketch plan is selected"); - // A per-series KLL or DDSketch plan. - let families: Vec = installed - .dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => Some(format!("{family:?}")), - _ => None, - }) - .collect(); - // Its sketches (beside the average's exact sum and count) are KLL or - // DDSketch. - let sketches: Vec<&String> = families - .iter() - .filter(|f| f.starts_with("Sketch")) - .collect(); - assert!( - !sketches.is_empty() - && sketches - .iter() - .all(|f| f.contains("Kll") || f.contains("DDSketch")), - "{families:?}" - ); - // The report marks the query's output as accepted without a guarantee. - let report = report.as_ref().expect("Stage 3's report"); - assert_eq!(report.queries_accepted_without_guarantee, vec![0]); - // It runs, equal to its query-time reference ... - let plan_json = serde_json::to_string(&installed).unwrap(); - let samples = grouped_samples(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - // ... and within 5 (of values spanning 100 per series) of the exact - // average of p99s, per group. - let exact = exact_average_of_p99s(&samples); - let answered: BTreeMap = run.answers_per_query[0] - .iter() - .map(|(labels, value)| (labels["g"].clone(), *value)) - .collect(); - assert_eq!( - answered.keys().collect::>(), - exact.keys().collect::>() - ); - for (g, value) in &answered { - assert!( - (value - exact[g]).abs() <= 5.0, - "g={g}: {value} vs exact {}", - exact[g] - ); - } + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, + "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# + )) + .unwrap() } /// The MVP benchmark's p99 query over an undeclared gauge, which has no @@ -2553,184 +435,6 @@ async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { ); } -/// [`p99_avg_workload`]'s query with `outer` in place of `avg`, planned -/// for a deployment that keeps no raw data samples. -async fn plan_p99( - outer: &str, - values_non_negative: bool, - ignore_accuracy: bool, -) -> (InstallablePlan, Option) { - let mut json = serde_json::to_value(p99_avg_workload(values_non_negative)).unwrap(); - json["query_workload"]["repeating_queries"][0]["query"] = - format!("{outer} by (region, service, job) (quantile_over_time(0.99, data[100s]))").into(); - let workload: PlanningWorkload = serde_json::from_value(json).unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - // A DDSketch kernel fixture, not a comparison of sketch families. - supported_summaries: ignore_accuracy - .then(|| only_summary_family(SketchAlgorithm::DDSketch)), - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(ignore_accuracy); - plan_installable(UserInput::new(&workload, FrontendInput::Promql, models)) - .await - .expect("a valid plan") -} - -/// `data` samples of series `0..series`, `value(series, step)` every -/// `interval_ms` over the `steps` steps ending just before `EVALUATION_MS`, -/// two series per (region, service, job) group; and each group's series' -/// windows `(EVALUATION_MS - 100 s, EVALUATION_MS]`. -#[allow(clippy::type_complexity)] -fn data_samples( - series: usize, - steps: i64, - interval_ms: i64, - value: impl Fn(usize, i64) -> f64, -) -> ( - Vec, - BTreeMap<(String, String, String), Vec>>, -) { - let mut samples = Vec::new(); - let mut per_group: BTreeMap<(String, String, String), Vec>> = BTreeMap::new(); - for s in 0..series { - let (region, service, job) = ( - format!("r{}", s % 2), - format!("s{}", s / 4), - "j".to_string(), - ); - let mut window = Vec::new(); - for step in 0..steps { - let timestamp_ms = EVALUATION_MS - steps * interval_ms + 50 + step * interval_ms; - let v = value(s, step); - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "data".to_string()), - ("region".to_string(), region.clone()), - ("service".to_string(), service.clone()), - ("job".to_string(), job.clone()), - ("instance".to_string(), format!("i{s}")), - ]), - timestamp_ms, - value: v, - }); - if timestamp_ms > EVALUATION_MS - 100_000 { - window.push(v); - } - } - per_group - .entry((region, service, job)) - .or_default() - .push(window); - } - (samples, per_group) -} - -/// PromQL's interpolated `quantile_over_time(q, ...)` of `window`. -fn interpolated_quantile(q: f64, window: &[f64]) -> f64 { - let mut sorted = window.to_vec(); - sorted.sort_by(f64::total_cmp); - let rank = q * (sorted.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - sorted[low] + (sorted[high] - sorted[low]) * (rank - low as f64) -} - -/// Runs `plan` over `samples` and checks every group's answer against -/// `combine` of its series' exact p99s, within the relative error 0.05, and -/// against the query-time reference. -fn assert_p99s_within_alpha( - plan: &InstallablePlan, - samples: &[RawDataSample], - per_group: &BTreeMap<(String, String, String), Vec>>, - combine: impl Fn(&[f64]) -> f64, -) { - let run = run_data_plane( - &serde_json::to_string(plan).unwrap(), - samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - let answers = &run.answers_per_query[0]; - assert_eq!(answers.len(), per_group.len()); - for (labels, value) in answers { - let key = ( - labels["region"].clone(), - labels["service"].clone(), - labels["job"].clone(), - ); - let p99s: Vec = per_group[&key] - .iter() - .map(|window| interpolated_quantile(0.99, window)) - .collect(); - let exact = combine(&p99s); - assert!( - (value - exact).abs() <= 0.05 * exact.abs(), - "{labels:?}: {value} vs exact {exact}" - ); - } -} - -/// A non-negative gauge that reports exactly 0 is still summarized by the -/// selected DDSketch tumbling windows (zero has its own bucket): each series' window -/// holds the samples 0, 0, 3 and 5 scaled by the series, and every answer -/// equals the query-time reference's and is within the relative error 0.05 -/// of the exact average of PromQL's interpolated p99s. -#[tokio::test] -async fn a_ddsketch_over_a_gauge_reporting_zero_runs_within_alpha() { - let (plan, _) = plan_p99("avg", true, false).await; - let views = plan.materialized_views().unwrap(); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - let (samples, per_group) = data_samples(8, 4, 20_000, |s, step| { - (s + 1) as f64 * [0.0, 0.0, 3.0, 5.0][step as usize] - }); - assert!(per_group.values().flatten().all(|window| window.len() == 4)); - let average = |p99s: &[f64]| p99s.iter().sum::() / p99s.len() as f64; - assert_p99s_within_alpha(&plan, &samples, &per_group, average); - // Many samples, a third of them zero. - let (samples, per_group) = data_samples(8, 1_100, 100, |s, step| { - (s + 1) as f64 * [0.0, 3.0, 5.0][(step % 3) as usize] - }); - assert_p99s_within_alpha(&plan, &samples, &per_group, average); -} - -/// Signed samples (negative, zero and positive) are summarized by the -/// selected DDSketch tumbling windows, through their negative store and zero bucket: -/// Signed interpolation has no general relative guarantee, so this kernel -/// fixture explicitly bypasses accuracy admission. Every answer equals the -/// query-time reference's and is -/// within 0.05 of the exact value, including series whose samples are all -/// negative. -#[tokio::test] -async fn a_ddsketch_over_signed_samples_runs_within_alpha() { - let (plan, report) = plan_p99("max", false, true).await; - let views = plan.materialized_views().unwrap(); - assert!( - !stored_definitions(&plan, "DDSketch").is_empty(), - "exercise signed DDS state" - ); - assert_eq!( - report.as_ref().unwrap().queries_accepted_without_guarantee, - vec![0] - ); - assert!(!views.is_empty(), "tumbling windows built at ingestion"); - let max = |p99s: &[f64]| p99s.iter().copied().fold(f64::NEG_INFINITY, f64::max); - // Series 0..4 cross zero; series 4..8 are all negative. - let (samples, per_group) = data_samples(8, 1_100, 100, |s, step| { - let scale = (s + 1) as f64; - if s < 4 { - scale * [-7.0, -3.0, 0.0, 2.0, 4.0][(step % 5) as usize] - } else { - scale * [-9.0, -8.0, -5.0, -1.0][(step % 4) as usize] - } - }); - assert_p99s_within_alpha(&plan, &samples, &per_group, max); -} - /// The ASAPQuery workloads' counter top-k query `topk(10, sum by (job) /// ((data_total[5m])))` for `f` `increase` or `rate`, every minute /// within 0.05 at failure probability 0.3, over 24 `data_total` counter @@ -2750,111 +454,6 @@ fn counter_topk_workload(function: &str) -> PlanningWorkload { .unwrap() } -/// The per-second rate of series `instance` (1 or 2) of job `job` (1..=12) -/// in [`counter_raw_data_samples`]: distinct for every series and every job's sum. -fn counter_rate(job: i64, instance: i64) -> f64 { - job as f64 + 0.1 * instance as f64 -} - -/// `data_total` samples of 12 jobs with 2 series each, every 10 s from -/// 595 s before `EVALUATION_MS`: each a counter growing linearly at -/// [`counter_rate`], so its 5-minute `increase` is 300 × its rate (the -/// samples 5 s inside each window end extrapolate to them exactly). -fn counter_raw_data_samples() -> Vec { - let mut samples = Vec::new(); - for job in 1..=12 { - for instance in 1..=2 { - for step in 0..60 { - let mut s = sample( - "data_total", - &format!("{job}-{instance}"), - EVALUATION_MS - 595_000 + step * 10_000, - counter_rate(job, instance) * (5 + step * 10) as f64, - ); - s.labels.insert("job".into(), format!("j{job:02}")); - samples.push(s); - } - } - } - samples -} - -/// Issue: with guarantees ignored and no raw data samples, the counter top-k -/// queries selected exact per-series tumbling windows followed by a query-time -/// CountSketchWithHeap top-k the executor cannot compile ("invalid keyed -/// evaluation shape or capacity"). The selected plan must be one the -/// executor compiles; it runs, equal to its query-time reference and to the -/// exact top 10 jobs (jobs 3..=12, each the sum of its two series). -#[tokio::test] -async fn ignoring_guarantees_counter_topk_plans_compile_and_answer_exactly() { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - let settings = CostInputs::default(); - for (function, seconds) in [("increase", 300.0), ("rate", 1.0)] { - let plan = plan_installable(UserInput::new( - &counter_topk_workload(function), - FrontendInput::Promql, - PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(true), - )) - .await - .unwrap_or_else(|error| panic!("{function}: {error}")) - .0; - // An exact plan: no heap over the per-job sums, which have one row - // per job. - assert!( - !plan.dag.nodes.iter().any(|node| matches!( - &node.payload, - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { - family: asap_executor::planner::ir::schema::FieldDataType::Sketch(..), - .. - } - ) - )), - "{function}" - ); - // The plan compiles as a runtime compiles it. - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane( - &plan_json, - &counter_raw_data_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{function}" - ); - let expected: Vec<(String, f64)> = (3..=12) - .map(|job| { - ( - format!("j{job:02}"), - (counter_rate(job, 1) + counter_rate(job, 2)) * seconds, - ) - }) - .collect(); - let mut answered: Vec<(String, f64)> = run.answers_per_query[0] - .iter() - .map(|(labels, value)| (labels["job"].clone(), *value)) - .collect(); - answered.sort_by(|a, b| a.0.cmp(&b.0)); - assert_eq!(answered.len(), expected.len(), "{function}: {answered:?}"); - for ((job, value), (want_job, want)) in answered.iter().zip(&expected) { - assert_eq!(job, want_job, "{function}"); - assert!( - (value - want).abs() <= 1e-9 * want, - "{function} {job}: {value} vs {want}" - ); - } - } -} - /// `InstallablePlan::check_compiles` reports what the Planner's executor /// refuses: the counter top-k plan with its answer column's type changed /// from Float64 to Utf8 does not compile, with the executor's reason. @@ -2895,897 +494,6 @@ async fn check_compiles_reports_the_executors_error() { ); } -/// `queries` every 10 s at `accuracy` (JSON) over six ingesting series of -/// `data`, a non-negative gauge, planned jointly on a deployment that keeps -/// no raw data samples: the plan file. -async fn jointly_planned(queries: &[&str], accuracy: &str) -> String { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {accuracy}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - let workload: PlanningWorkload = serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 0.6}}}}, - "sources": {{"data": {{"series_count": 6, "sample_interval": 10000, - "metric_type": "gauge", "values_non_negative": true}}}}}}}}"#, - entries.join(",") - )) - .unwrap(); - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_json_for(workload, &capabilities).await -} - -/// The definition ids of `plan`'s materialized views whose node's payload -/// mentions `family` (e.g. `"DDSketch"`, `"Sum"`). -fn stored_definitions(plan: &InstallablePlan, family: &str) -> BTreeSet { - plan.materialized_views() - .unwrap() - .iter() - .filter(|view| { - let node = plan - .dag - .nodes - .iter() - .find(|n| n.id == view.node_id) - .unwrap(); - serde_json::to_string(&node.payload) - .unwrap() - .contains(family) - }) - .map(|view| view.coverage().definition_id().to_owned()) - .collect() -} - -/// A fine and a coarse sum over one metric and window (ASAPQuery's -/// sum-fine and sum-coarse) store one Sum: both group sums are computed -/// from one stored per-series Sum (Stage 1 shares the per-series sums the -/// two queries' scans read), so every stored Sum has one definition, and -/// both answer as the query-time reference. -#[tokio::test] -async fn fine_and_coarse_sums_store_one_sum() { - let plan_json = jointly_planned( - &[ - "sum by (region, service, job) (sum_over_time(data[100s]))", - "sum by (region) (sum_over_time(data[100s]))", - ], - "{}", - ) - .await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - let sums = stored_definitions(&plan, "\"Sum\""); - assert_eq!(sums.len(), 1, "{sums:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), 6); - assert_eq!(run.answers_per_query[1].len(), 2, "one sum per region"); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// Per-series answers planned beside a grouping of the same per-series -/// values (over a scan with other labels) answer as their query-time -/// reference, and the two share one stored per-series state: a top-k of -/// per-series sums keeps each chosen series' labels, and per-series -/// DDSketch quantiles keep every series' labels. -#[tokio::test] -async fn per_series_answers_beside_a_grouping_share_one_state_and_answer_as_the_reference() { - for (queries, family, accuracy, answers) in [ - ( - [ - "topk(2, sum_over_time(data[100s]))", - "sum by (region) (sum_over_time(data[100s]))", - ], - "\"Sum\"", - "{}", - 2, - ), - ( - [ - "quantile_over_time(0.9, data[100s])", - "avg by (region) (quantile_over_time(0.99, data[100s]))", - ], - "DDSketch", - r#"{"accuracy": {"accuracy_metric": "relative", "max_error": 0.05, - "failure_probability": 0.01, "accuracy_scope": "per_value"}}"#, - 6, - ), - ] { - let plan_json = jointly_planned(&queries, accuracy).await; - let plan: InstallablePlan = serde_json::from_str(&plan_json).unwrap(); - assert_eq!(stored_definitions(&plan, family).len(), 1, "{queries:?}"); - let run = run_data_plane( - &plan_json, - &grouped_series_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query[0].len(), answers, "{queries:?}"); - assert_eq!(run.answers_per_query[1].len(), 2, "{queries:?}: per region"); - assert_eq!( - run.answers_per_query, run.reference_answers_per_query, - "{queries:?}" - ); - } -} - -/// A heap top-k query `query` (the ASAPQuery workloads' top-k-sum or -/// counter top-k shape) every 10 s, within 0.05 at failure probability 0.3, -/// over 10,000 series of `metric` sampled every 100 ms, with `facts` (its -/// type and sign, as JSON fields). -fn heap_topk_workload(query: &str, metric: &str, facts: &str) -> PlanningWorkload { - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ - "query": {query:?}, - "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {{"max_error": 0.05, "failure_probability": 0.3}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 100000.0}}}}, - "sources": {{"{metric}": {{"series_count": 10000, "sample_interval": 100, - {facts}}}}}}}}}"# - )) - .unwrap() -} - -/// The executor's support for `family` alone: a deployment whose only -/// sketch family is `family` (exact plans stay allowed). -fn only_summary_family(family: SketchAlgorithm) -> Vec { - asap_executor::supported_summary_families() - .into_iter() - .filter(|supported| supported.family == family) - .collect() -} - -/// A deployment's accuracy model that accepts, whatever its guarantee, only -/// a query output that reads a heap of this algorithm, and no other output: -/// it forces the cheapest plan with that heap, as a deployment that requires -/// the heap would. -struct HeapPlansOnly(SketchAlgorithm); - -impl asap_accuracy::AccuracyModel for HeapPlansOnly { - fn satisfies( - &self, - _guarantee: &asap_types::ir::properties::AccuracyGuarantee, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - false - } - - fn accepts_output_regardless_of_guarantee( - &self, - output: &QueryRoot, - _target: &asap_types::ir::properties::AccuracyTarget, - ) -> bool { - let QueryRoot::Operator(root) = output else { - return false; - }; - OperatorNode::reachable(root).iter().any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. } - | ASAPOp::SummaryTumblingWindowChain { family: FieldDataType::Sketch(kind, _), .. }) - if *kind.algorithm() == self.0) - }) - } -} - -/// `workload` planned with accuracy guarantees ignored (a top-k estimate -/// has none) on a deployment that keeps no raw samples; with `heap`, the -/// deployment's accuracy model accepts only outputs that read a heap of that -/// algorithm ([`HeapPlansOnly`]). -async fn plan_ignoring_guarantees( - workload: &PlanningWorkload, - heap: Option, -) -> Result { - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - // Ignoring guarantees replaces the accuracy model, so a forced heap goes - // through its own accuracy model instead. - let settings = CostInputs::default(); - let heap_only = heap.map(HeapPlansOnly); - let models = PlanningModels::builtin() - .with_deployment_capabilities(&capabilities) - .with_cost_inputs(&settings) - .with_ignore_accuracy_guarantees(heap.is_none()); - let models = match &heap_only { - Some(model) => models.with_accuracy(model), - None => models, - }; - plan_installable(UserInput::new(workload, FrontendInput::Promql, models)) - .await - .map(|(plan, _)| plan) - .map_err(|e| e.to_string()) -} - -/// The heap sketch `plan` builds: its algorithm and width. -fn heap_of(plan: &InstallablePlan) -> Option<(SketchAlgorithm, u32)> { - plan.dag.nodes.iter().find_map(|node| match &node.payload { - asap_executor::planner::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => match family { - FieldDataType::Sketch(kind, _) => match kind.params() { - SketchParams::CmsWithHeap { width, .. } - | SketchParams::CountSketchWithHeap { width, .. } => { - Some((*kind.algorithm(), *width)) - } - _ => None, - }, - _ => None, - }, - _ => None, - }) -} - -/// The heap of `algorithm`'s point-error bound on an item of a partition -/// whose items' true values are `values`: `ε·L1` for Count-Min (ε = e/w, -/// an overestimate), `ε·L2` for Count Sketch (ε = √(3/w)), which is at most -/// `ε·L1`. -fn heap_bound(algorithm: &SketchAlgorithm, width: u32, values: &[f64]) -> f64 { - let width = f64::from(width); - match algorithm { - SketchAlgorithm::CmsWithHeap => { - std::f64::consts::E / width * values.iter().map(|v| v.abs()).sum::() - } - SketchAlgorithm::CountSketchWithHeap => { - (3.0 / width).sqrt() * values.iter().map(|v| v * v).sum::().sqrt() - } - other => panic!("not a heap: {other:?}"), - } -} - -/// Checks a heap top-k plan's `answers` against the `truth` (each item's -/// exact value, by partition): per partition, `k` answers, the true top `k` -/// recalled, each item named by its labels (`item_label`, and `partition` -/// when partitioned), and each estimate within the heap's bound. -fn assert_heap_topk_answers( - answers: &[Answer], - truth: &BTreeMap>, - (partition, item_label, k): (Option<&str>, &str, usize), - (algorithm, width): (&SketchAlgorithm, u32), -) { - let mut answered: BTreeMap> = BTreeMap::new(); - for (labels, value) in answers { - // A range function's result drops the metric name, as the exact - // top-k's does. - assert!(!labels.contains_key("__name__"), "{labels:?}"); - let group = partition.map_or(String::new(), |p| labels[p].clone()); - let item = labels - .get(item_label) - .unwrap_or_else(|| panic!("{item_label} kept: {labels:?}")); - answered - .entry(group) - .or_default() - .push((item.clone(), *value)); - } - assert_eq!( - answered.keys().collect::>(), - truth.keys().collect::>() - ); - for (group, items) in &answered { - let exact = &truth[group]; - let bound = heap_bound( - algorithm, - width, - &exact.values().copied().collect::>(), - ); - let mut ranked: Vec<(&String, &f64)> = exact.iter().collect(); - ranked.sort_by(|a, b| b.1.total_cmp(a.1)); - let top: BTreeSet<&String> = ranked.iter().take(k).map(|(item, _)| *item).collect(); - assert_eq!(items.len(), k, "{group}: {items:?}"); - let recalled = items.iter().filter(|(item, _)| top.contains(item)).count(); - assert_eq!(recalled, k, "{group}: recall of {top:?} in {items:?}"); - for (item, estimate) in items { - let error = estimate - exact[item]; - assert!( - error.abs() <= bound + 1e-9 * exact[item].abs(), - "{group}/{item}: {estimate} vs exact {}, bound {bound}", - exact[item] - ); - if *algorithm == SketchAlgorithm::CmsWithHeap { - assert!( - error >= -1e-9 * exact[item].abs(), - "Count-Min overestimates" - ); - } - } - } -} - -/// `data` samples of 2 regions with 40 series each, six inside the last -/// minute per series: series `r` (rank 1..=40) of a region sums to -/// `1000 / r^1.1` (west at half that), a skewed ranking. -fn skewed_series_samples() -> (Vec, BTreeMap>) { - let mut samples = Vec::new(); - let mut truth: BTreeMap> = BTreeMap::new(); - for (region, scale) in [("east", 1.0), ("west", 0.5)] { - for rank in 1..=40 { - let instance = format!("{region}-{rank:02}"); - let sum = scale * 1000.0 / f64::from(rank).powf(1.1); - for step in 0..6 { - let mut s = sample( - "data", - &instance, - EVALUATION_MS - 55_000 + step * 10_000, - sum / 6.0, - ); - s.labels.insert("region".into(), region.into()); - samples.push(s); - } - truth - .entry(region.to_string()) - .or_default() - .insert(instance, sum); - } - } - (samples, truth) -} - -/// `workload`'s plan with only `algorithm`'s heap plans accepted -/// ([`HeapPlansOnly`] forces the heap; guarantees are not ignored): it builds that heap, compiles as a runtime compiles it, and -/// both it and its query-time reference answer `samples` with the true top -/// `k` per partition, each named by its labels and within the heap's bound -/// of `truth` ([`assert_heap_topk_answers`]). -async fn assert_forced_heap_answers( - workload: &PlanningWorkload, - algorithm: SketchAlgorithm, - (samples, truth): (&[RawDataSample], &BTreeMap>), - labels: (Option<&str>, &str, usize), -) { - let plan = plan_ignoring_guarantees(workload, Some(algorithm)) - .await - .unwrap_or_else(|error| panic!("{algorithm:?}: {error}")); - let (built, width) = heap_of(&plan).expect("a heap plan"); - assert_eq!(built, algorithm); - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane(&plan_json, samples, EVALUATION_MS, &mut SummaryStore::new()); - for answers in [ - &run.answers_per_query[0], - &run.reference_answers_per_query[0], - ] { - assert_heap_topk_answers(answers, truth, labels, (&algorithm, width)); - } -} - -/// The ASAPQuery workloads' top-k-sum shape over a gauge, -/// `topk by (region) (3, sum_over_time(data[1m]))`: the fused -/// whole-expression Count Sketch heap (one per region over the raw -/// samples, ranking series identities), forced with guarantees ignored, -/// compiles, and answers each region's true top 3 series, with their full -/// labels, within its bound of their exact sums. (A Count-Min heap needs -/// the weights proven non-negative.) -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_topk_sum_plan_compiles_and_answers_within_its_bound() { - let workload = heap_topk_workload( - "topk by (region) (3, sum_over_time(data[1m]))", - "data", - r#""metric_type": "gauge""#, - ); - let (samples, truth) = skewed_series_samples(); - assert_forced_heap_answers( - &workload, - SketchAlgorithm::CountSketchWithHeap, - (&samples, &truth), - (Some("region"), "instance", 3), - ) - .await; -} - -/// `data` samples of 2 regions with 40 series each inside the last minute: -/// series `r` (rank 1..=40) has `round(200 / r^1.1)` samples (at least 1), -/// a skewed count ranking. -fn skewed_count_samples() -> (Vec, BTreeMap>) { - let mut samples = Vec::new(); - let mut truth: BTreeMap> = BTreeMap::new(); - for region in ["east", "west"] { - for rank in 1..=40 { - let instance = format!("{region}-{rank:02}"); - let count = (200.0 / f64::from(rank).powf(1.1)).round().max(1.0) as i64; - for step in 0..count { - // Spread over (EVALUATION_MS - 59 s, EVALUATION_MS - 1 s]. - let at = EVALUATION_MS - 1_000 - step * 58_000 / count; - let mut s = sample("data", &instance, at, 1.0 + step as f64); - s.labels.insert("region".into(), region.into()); - samples.push(s); - } - truth - .entry(region.to_string()) - .or_default() - .insert(instance, count as f64); - } - } - samples.sort_by_key(|s| s.timestamp_ms); - (samples, truth) -} - -/// `topk by (region) (3, count_over_time(data[1m]))`: each forced fused -/// heap (each sample counts 1) compiles and answers each region's true top -/// 3 series, with their labels, within its bound of their counts. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_topk_count_plan_compiles_and_answers_within_its_bound() { - let workload = heap_topk_workload( - "topk by (region) (3, count_over_time(data[1m]))", - "data", - r#""metric_type": "gauge""#, - ); - let (samples, truth) = skewed_count_samples(); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - assert_forced_heap_answers( - &workload, - algorithm, - (&samples, &truth), - (Some("region"), "instance", 3), - ) - .await; - } -} - -/// The per-second rate of series `instance` (1 or 2) of job `job` (1..=12) -/// in [`skewed_counter_raw_data_samples`]: `50 / job^1.2`, plus a little per -/// instance, a skewed ranking of the jobs' sums. -fn skewed_counter_rate(job: i64, instance: i64) -> f64 { - 50.0 / (job as f64).powf(1.2) + 0.01 * instance as f64 -} - -/// `data_total` samples of 12 jobs with 2 series each, every 10 s from 595 s -/// before `EVALUATION_MS`, each a counter growing linearly at -/// [`skewed_counter_rate`], as [`counter_raw_data_samples`]. -fn skewed_counter_raw_data_samples() -> Vec { - let mut samples = Vec::new(); - for job in 1..=12 { - for instance in 1..=2 { - for step in 0..60 { - let mut s = sample( - "data_total", - &format!("{job}-{instance}"), - EVALUATION_MS - 595_000 + step * 10_000, - skewed_counter_rate(job, instance) * (5 + step * 10) as f64, - ); - s.labels.insert("job".into(), format!("j{job:02}")); - samples.push(s); - } - } - } - samples -} - -/// The ASAPQuery workloads' counter top-k, `topk(3, sum by (job) -/// ((data_total[5m])))` for `f` `increase` or `rate`: each forced fused -/// heap (one over the per-series values, ranking jobs) compiles and answers -/// the true top 3 jobs, with their `job` labels, within its bound of their -/// exact sums. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_counter_topk_plan_compiles_and_answers_within_its_bound() { - let samples = skewed_counter_raw_data_samples(); - for (function, seconds) in [("increase", 300.0), ("rate", 1.0)] { - let workload = heap_topk_workload( - &format!("topk(3, sum by (job) ({function}(data_total[5m])))"), - "data_total", - r#""metric_type": "counter""#, - ); - // Each job's exact sum over its two series. - let truth = BTreeMap::from([( - String::new(), - (1..=12) - .map(|job| { - let sum = skewed_counter_rate(job, 1) + skewed_counter_rate(job, 2); - (format!("j{job:02}"), sum * seconds) - }) - .collect::>(), - )]); - for algorithm in [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap, - ] { - assert_forced_heap_answers(&workload, algorithm, (&samples, &truth), (None, "job", 3)) - .await; - } - } -} - -/// `data` samples of 40 regions with 3 series each, inside the last -/// minute: region `r` (rank 1..=40) sums to `1000 / r^1.1` over -/// its series (a skewed ranking of the regions) and holds `3 · -/// round(200 / r^1.1)` samples (a skewed count ranking), returned with each -/// region's exact sum and exact count. -fn skewed_region_raw_data_samples() -> ( - Vec, - BTreeMap, - BTreeMap, -) { - let mut samples = Vec::new(); - let mut sums = BTreeMap::new(); - let mut counts = BTreeMap::new(); - for rank in 1..=40i64 { - let region = format!("r{rank:02}"); - let sum = 1000.0 / (rank as f64).powf(1.1); - let per_series_steps = (200.0 / (rank as f64).powf(1.1)).round().max(1.0) as i64; - for series in 0..3 { - let instance = format!("{region}-{series}"); - for step in 0..per_series_steps { - // Spread over (EVALUATION_MS - 59 s, EVALUATION_MS - 1 s]. - let at = EVALUATION_MS - 1_000 - step * 58_000 / per_series_steps; - let mut s = sample("data", &instance, at, sum / 3.0 / per_series_steps as f64); - s.labels.insert("region".into(), region.clone()); - samples.push(s); - } - } - sums.insert(region.clone(), sum); - counts.insert(region, (3 * per_series_steps) as f64); - } - samples.sort_by_key(|s| s.timestamp_ms); - (samples, sums, counts) -} - -/// The ASAPQuery workloads' grouped top-k-sum shape, -/// `topk(3, sum by (region) ((data[1m])))` for `f` `sum_over_time` or -/// `count_over_time`: each forced fused heap (one over the raw data samples, -/// keyed by `region`, weighted by the sample value or by one) compiles and -/// answers the true top 3 regions within its bound of their exact sums or -/// counts, each named by its `region` label alone, as the exact top-k's. -/// A Count-Min heap over sample values needs the gauge declared -/// non-negative; a Count Sketch heap takes any gauge. -#[tokio::test] -#[ignore = "fails on main too: Stage 3 rejects the forced heap plan (SelectedPlanFailedCheck, AccuracyTargetMissed); not a wall-clock issue; #1015"] -async fn a_forced_heap_grouped_topk_sum_plan_compiles_and_answers_within_its_bound() { - let (samples, sums, counts) = skewed_region_raw_data_samples(); - let gauge = r#""metric_type": "gauge""#; - let non_negative = r#""metric_type": "gauge", "values_non_negative": true"#; - for (function, truth, facts, algorithm) in [ - ( - "sum_over_time", - &sums, - non_negative, - SketchAlgorithm::CmsWithHeap, - ), - ( - "sum_over_time", - &sums, - gauge, - SketchAlgorithm::CountSketchWithHeap, - ), - ( - "count_over_time", - &counts, - gauge, - SketchAlgorithm::CmsWithHeap, - ), - ( - "count_over_time", - &counts, - gauge, - SketchAlgorithm::CountSketchWithHeap, - ), - ] { - let query = format!("topk(3, sum by (region) ({function}(data[1m])))"); - let workload = heap_topk_workload(&query, "data", facts); - let truth = BTreeMap::from([(String::new(), truth.clone())]); - let plan = plan_ignoring_guarantees(&workload, Some(algorithm)) - .await - .unwrap_or_else(|error| panic!("{query} {algorithm:?}: {error}")); - let (built, width) = heap_of(&plan).expect("a heap plan"); - assert_eq!(built, algorithm, "{query}"); - plan.check_compiles().unwrap(); - let plan_json = serde_json::to_string(&plan).unwrap(); - let run = run_data_plane( - &plan_json, - &samples, - EVALUATION_MS, - &mut SummaryStore::new(), - ); - for answers in [ - &run.answers_per_query[0], - &run.reference_answers_per_query[0], - ] { - assert_heap_topk_answers(answers, &truth, (None, "region", 3), (&algorithm, width)); - for (labels, _) in answers.iter() { - assert_eq!( - labels.keys().collect::>(), - ["region"], - "{query} {algorithm:?}: the group label alone" - ); - } - } - // The installed plan (stored states) answers as its query-time - // reference does, within the heap's bound. - let bound = heap_bound( - &algorithm, - width, - &truth[""].values().copied().collect::>(), - ); - let reference: BTreeMap<_, _> = run.reference_answers_per_query[0] - .iter() - .map(|(labels, value)| (labels.clone(), *value)) - .collect(); - for (labels, value) in &run.answers_per_query[0] { - let expected = reference - .get(labels) - .unwrap_or_else(|| panic!("{query} {algorithm:?}: {labels:?} in the reference")); - assert!( - (value - expected).abs() <= bound, - "{query} {algorithm:?}: {value} vs {expected}" - ); - } - } -} - // ── Classic-histogram `histogram_quantile` (issue #890) ───────────────────── -/// The classic bucket metric of the histogram tests. -const BUCKET_METRIC: &str = "http_request_duration_seconds_bucket"; - -/// Exact PromQL `queries`, every minute, over ingesting classic-bucket -/// series of [`BUCKET_METRIC`] sampled every 10 s. -fn exact_bucket_workload(queries: &[&str]) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 60000}}, - "requirements": {{"accuracy": {{"max_error": 0.0, "failure_probability": 0.0}}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 2.4}}}}, - "sources": {{"{BUCKET_METRIC}": {{"series_count": 24, "sample_interval": 10000}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -/// Cumulative bucket counters of [`BUCKET_METRIC`], one sample every 10 s -/// over the 5 minutes before `EVALUATION_MS` (first 295 s before it, last -/// 5 s before it), each rising at a constant per-second rate from 1000, so -/// `rate(…[5m])` of each is its slope. Per `(service, job)`, the slopes of -/// buckets `le` 0.1, 0.5, 1 and `+Inf`, each on instances `1` and `2`: -/// - `orders`/`api`: 0.5, 1.5, 2, 2 (summed over instances: 1, 3, 4, 4); -/// - `batch`/`worker`: 0.5, 0.5, 0.5, 2.5 (most observations above 1, in -/// `+Inf` alone; summed: 1, 1, 1, 5); -/// - `idle`/`idle`: 0 everywhere (no observations). -fn bucket_samples() -> Vec { - let mut samples = Vec::new(); - for (service, job, slopes) in [ - ("orders", "api", [0.5, 1.5, 2.0, 2.0]), - ("batch", "worker", [0.5, 0.5, 0.5, 2.5]), - ("idle", "idle", [0.0; 4]), - ] { - for (le, slope) in ["0.1", "0.5", "1", "+Inf"].into_iter().zip(slopes) { - for instance in ["1", "2"] { - for step in 0..30 { - samples.push(RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), BUCKET_METRIC.to_string()), - ("service".to_string(), service.to_string()), - ("job".to_string(), job.to_string()), - ("instance".to_string(), instance.to_string()), - ("le".to_string(), le.to_string()), - ]), - timestamp_ms: EVALUATION_MS - 295_000 + step * 10_000, - value: 1000.0 + slope * 10.0 * step as f64, - }); - } - } - } - } - samples -} - -/// The plan file of exact `queries` over [`bucket_samples`]' metric, on a -/// deployment that keeps raw data samples. -async fn bucket_plan_json(queries: &[&str]) -> String { - plan_json_for( - exact_bucket_workload(queries), - &DeploymentCapabilities::default(), - ) - .await -} - -/// `histogram_quantile` over classic buckets as the issue's dashboard -/// writes it: p50, p95 and p99 of the summed bucket rates of one service, -/// in one workload. Stage 3 selects an exact plan the executor compiles, -/// and it answers with Prometheus' cumulative-bucket interpolation: over -/// summed rates 1, 3, 4, 4 at `le` 0.1, 0.5, 1, `+Inf`, rank 0.5 · 4 = 2 -/// lies in the 0.5 bucket, 0.1 + 0.4 · (2 − 1) / (3 − 1) = 0.3; rank 3.8 -/// and 3.96 in the 1 bucket, 0.5 + 0.5 · 0.8 = 0.9 and 0.5 + 0.5 · 0.96 = -/// 0.98. The result has no labels: `sum by (le)` keeps only `le`, which -/// the quantile drops. -#[tokio::test] -async fn classic_histogram_quantiles_run_with_prometheus_interpolation() { - let selector = format!(r#"rate({BUCKET_METRIC}{{service="orders"}}[5m])"#); - let queries: Vec = [0.5, 0.95, 0.99] - .iter() - .map(|q| format!("histogram_quantile({q}, sum by (le) ({selector}))")) - .collect(); - let queries: Vec<&str> = queries.iter().map(String::as_str).collect(); - let plan_json = bucket_plan_json(&queries).await; - let run = run_data_plane( - &plan_json, - &bucket_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - assert_eq!(run.answers_per_query.len(), 3); - for (answers, expected) in run.answers_per_query.iter().zip([0.3, 0.9, 0.98]) { - assert_eq!(answers.len(), 1, "{answers:?}"); - let (labels, value) = &answers[0]; - assert!(labels.is_empty(), "le is dropped: {labels:?}"); - assert!((value - expected).abs() < 1e-9, "{value} vs {expected}"); - } - assert_eq!(run.answers_per_query, run.reference_answers_per_query); -} - -/// One p99 per `job` (`sum by (job, le)`): the result keeps `job` and -/// drops `le`. `worker`'s rank 0.99 · 5 = 4.95 lies past every finite -/// bucket (most observations are in `+Inf` alone), so Prometheus answers -/// the largest finite bound, 1; `idle` has no observations, so NaN. The -/// p99 alert (`> 0.5`) keeps `api` (0.98) and `worker` (1) and drops -/// `idle` (NaN compares false). A selector that matches no series gives an -/// empty result. Over one series' buckets, without `sum`, the result keeps -/// that series' labels but `le` (and the name `rate` dropped). -#[tokio::test] -async fn a_classic_histogram_quantile_keeps_labels_and_handles_inf_and_empty_input() { - let by_job = format!("histogram_quantile(0.99, sum by (job, le) (rate({BUCKET_METRIC}[5m])))"); - let alert = format!("{by_job} > 0.5"); - let empty = format!( - r#"histogram_quantile(0.5, sum by (le) (rate({BUCKET_METRIC}{{service="missing"}}[5m])))"# - ); - // Without `sum`, each series is its own histogram: `instance` 1's - // buckets alone (rates 0.5, 1.5, 2, 2), keeping its labels but `le`. - let per_series = format!( - r#"histogram_quantile(0.99, rate({BUCKET_METRIC}{{service="orders",instance="1"}}[5m]))"# - ); - let plan_json = bucket_plan_json(&[&by_job, &alert, &empty, &per_series]).await; - let run = run_data_plane( - &plan_json, - &bucket_samples(), - EVALUATION_MS, - &mut SummaryStore::new(), - ); - let job = |answers: &[Answer]| -> Vec<(String, f64)> { - answers - .iter() - .map(|(labels, value)| { - assert_eq!(labels.keys().collect::>(), ["job"], "{labels:?}"); - (labels["job"].clone(), *value) - }) - .collect() - }; - for answers in [&run.answers_per_query, &run.reference_answers_per_query] { - let per_job = job(&answers[0]); - assert_eq!(per_job.len(), 3, "{per_job:?}"); - assert_eq!(per_job[0].0, "api"); - assert!((per_job[0].1 - 0.98).abs() < 1e-9, "{per_job:?}"); - assert_eq!(per_job[1].0, "idle"); - assert!(per_job[1].1.is_nan(), "{per_job:?}"); - assert_eq!(per_job[2], ("worker".to_string(), 1.0)); - let alerting = job(&answers[1]); - assert_eq!(alerting.len(), 2, "{alerting:?}"); - assert_eq!(alerting[0].0, "api"); - assert_eq!(alerting[1], ("worker".to_string(), 1.0)); - assert!(answers[2].is_empty(), "{:?}", answers[2]); - let [(labels, value)] = answers[3].as_slice() else { - panic!("{:?}", answers[3]); - }; - assert_eq!( - labels.keys().collect::>(), - ["instance", "job", "service"], - "{labels:?}" - ); - assert!((value - 0.98).abs() < 1e-9, "{value}"); - } -} - // ── PromQL absent() (issue #895) ───────────────────────────────────────────── - -/// The Prometheus stale marker, which Remote Write carries when a series ends. -const STALE_MARKER: f64 = f64::from_bits(0x7ff0_0000_0000_0002); - -/// The answers of the exact plan of PromQL `query` over `samples` of `m` -/// (6 series, every 10 s), with the query-time reference's. -async fn exact_answers(query: &str, samples: &[RawDataSample]) -> Vec { - let plan_json = plan_json_for( - exact_workload(&[query], ""), - &DeploymentCapabilities::default(), - ) - .await; - let run = run_data_plane(&plan_json, samples, EVALUATION_MS, &mut SummaryStore::new()); - assert_eq!(run.answers_per_query, run.reference_answers_per_query); - run.answers_per_query[0].clone() -} - -/// A sample of `m` at `ago_ms` before `EVALUATION_MS` with `labels`. -fn m_sample(labels: &[(&str, &str)], ago_ms: i64, value: f64) -> RawDataSample { - let mut sample = sample("m", "1", EVALUATION_MS - ago_ms, value); - for (name, label) in labels { - sample.labels.insert(name.to_string(), label.to_string()); - } - sample -} - -/// `absent(v)` answers as Prometheus' `funcAbsent`: one sample of value 1 -/// when `v` selects no series at the evaluation time, labelled with the -/// selector's equality matchers (`job`, not the negated `instance` and -/// `env`); no sample when some series matches, whatever its value -/// (0 included). A series whose newest sample is the stale marker has -/// ended, so `absent` answers 1 again. -#[tokio::test] -async fn absent_answers_as_prometheus_through_the_installed_plan() { - let query = r#"absent(m{job="a",instance!="3",env!="prod"})"#; - let one = vec![(BTreeMap::from([("job".to_string(), "a".to_string())]), 1.0)]; - // No sample at all. - assert_eq!(exact_answers(query, &[]).await, one); - // Only series the matchers reject: job b, env prod, instance 3. - let rejected = [ - m_sample(&[("job", "b")], 1_000, 1.0), - m_sample(&[("job", "a"), ("env", "prod")], 1_000, 1.0), - { - let mut other = m_sample(&[("job", "a")], 1_000, 1.0); - other.labels.insert("instance".into(), "3".into()); - other - }, - ]; - assert_eq!(exact_answers(query, &rejected).await, one); - // A matching series, valued 0 or not: no sample. - for value in [0.0, 5.0] { - let samples = [m_sample(&[("job", "a")], 1_000, value)]; - assert_eq!(exact_answers(query, &samples).await, vec![], "{value}"); - } - // The last matching series ended (its newest sample is stale). - let ended = [ - m_sample(&[("job", "a")], 5_000, 5.0), - m_sample(&[("job", "a")], 1_000, STALE_MARKER), - ]; - assert_eq!(exact_answers(query, &ended).await, one); -} - -/// `absent(m)` (no matcher) synthesizes a label-less sample; the missing-job -/// alert shape (several equality matchers) keeps every matcher's label. -#[tokio::test] -async fn absent_labels_come_from_the_equality_matchers_only() { - assert_eq!( - exact_answers("absent(m)", &[]).await, - vec![(BTreeMap::new(), 1.0)] - ); - let alert = r#"absent(m{kubernetes_namespace="mw-content-history-reconcile-enrich",release="production",job_name="mw_content_history_reconcile_enrich"})"#; - let labels: BTreeMap = [ - ( - "kubernetes_namespace", - "mw-content-history-reconcile-enrich", - ), - ("release", "production"), - ("job_name", "mw_content_history_reconcile_enrich"), - ] - .into_iter() - .map(|(name, value)| (name.to_string(), value.to_string())) - .collect(); - assert_eq!(exact_answers(alert, &[]).await, vec![(labels.clone(), 1.0)]); - let running = [m_sample( - &labels - .iter() - .map(|(name, value)| (name.as_str(), value.as_str())) - .collect::>(), - 1_000, - 3600.0, - )]; - assert_eq!(exact_answers(alert, &running).await, vec![]); -} diff --git a/crates/planner/tests/sql_per_group_topk.rs b/crates/planner/tests/sql_per_group_topk.rs deleted file mode 100644 index d4ea3a199..000000000 --- a/crates/planner/tests/sql_per_group_topk.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! SQL's per-group top-k idiom, `ROW_NUMBER() OVER (PARTITION BY city -//! ORDER BY COUNT(*) DESC) … WHERE rn <= k`, planned with -//! `plan_installable` and run with the Planner's executor: the exact plan -//! compiles and answers each city's top hosts with their counts, the same -//! rows PromQL `topk by (city)` answers over the same data, and the plan -//! Stage 3 selects under an approximate target installs (each materialized -//! view it keeps refreshes). - -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; - -use asap_executor::physical_planner::compile; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::InMemoryDataStore; -use asap_executor::values::{Batch, Value}; -use asap_planner::{ - plan_installable, test_support::query_time_reference, FrontendInput, InstallablePlan, - InstallablePlanError, PlanningModels, PlanningWorkload, SqlCatalog, UserInput, -}; -use asap_types::ir::operator::Source as DataSource; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::NonASAPOp; -use futures::{executor::block_on, StreamExt}; - -use crate::time_series; -use time_series::{time_series_tables, RawDataSample}; - -/// The evaluation time of every run. -const EVALUATION_MS: i64 = 1_800_000_000_000; - -/// The issue's per-city top 5 hosts by row count over the last 5 minutes. -const TOP_5_HOSTS_PER_CITY: &str = "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, \ - ROW_NUMBER() OVER (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY city, host) WHERE rn <= 5"; - -/// The issue's global top 5 hosts by row count over the last 5 minutes. -const TOP_5_HOSTS: &str = "SELECT host, COUNT(*) AS c FROM metrics \ - WHERE ts > now() - INTERVAL '5' MINUTE GROUP BY host ORDER BY c DESC LIMIT 5"; - -/// The same per-city top 5 in PromQL, each table row a sample of the -/// series `metrics{city, host}`. -const PROMQL_TOP_5_HOSTS_PER_CITY: &str = - "topk by (city) (5, sum by (city, host) (count_over_time(metrics[5m])))"; - -/// `metrics(ts, city, host, value)`, as the issue declares it (all but -/// `ts` nullable). -fn metrics_schema() -> Schema { - Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("host", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ) -} - -/// Repeating `queries` in `language` (its workload JSON), every 10 s at -/// `accuracy`, over `metrics` ingesting 666.67 rows per second from 100 -/// series: the issue's workload. -fn workload(language: &str, queries: &[&str], accuracy: &str) -> PlanningWorkload { - let entries: Vec = queries - .iter() - .map(|query| { - format!( - r#"{{"query": {query:?}, "demand": {{"fixed_interval": 10000}}, - "requirements": {{"accuracy": {accuracy}}}, - "predictability": {{"predictable": {{"known_at": null}}}}}}"# - ) - }) - .collect(); - serde_json::from_str(&format!( - r#"{{"query_workload": {{"language": {language}, "repeating_queries": [{}]}}, - "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 666.67}}}}, - "sources": {{"metrics": {{"series_count": 100}}}}}}}}"#, - entries.join(",") - )) - .unwrap() -} - -const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; -const APPROXIMATE: &str = r#"{"max_error": 0.01, "failure_probability": 0.001}"#; - -async fn plan_sql(queries: &[&str], accuracy: &str) -> InstallablePlan { - let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); - plan_installable(UserInput::new( - &workload(r#"{"sql": "datafusion_sql"}"#, queries, accuracy), - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await - .expect("plan_installable plans the SQL workload") - .0 -} - -async fn plan_promql(query: &str) -> InstallablePlan { - plan_installable(UserInput::new( - &workload(r#""promql""#, &[query], EXACT), - FrontendInput::Promql, - PlanningModels::builtin(), - )) - .await - .expect("plan_installable plans the PromQL workload") - .0 -} - -/// Rows per `(city, host)` in the last 5 minutes. `nyc` has seven hosts -/// with distinct counts at the 5th place (`h5` 5, `h6` and `h7` 3), so the -/// top 5 does not depend on how ties are broken; `sf` has two hosts, fewer -/// than 5. `h7` also has 100 rows from before the window, which no answer -/// counts. -const RECENT_ROWS: [(&str, &str, i64); 9] = [ - ("nyc", "h1", 9), - ("nyc", "h2", 8), - ("nyc", "h3", 7), - ("nyc", "h4", 6), - ("nyc", "h5", 5), - ("nyc", "h6", 3), - ("nyc", "h7", 3), - ("sf", "s1", 4), - ("sf", "s2", 2), -]; - -/// The table rows: per `(city, host)`, its [`RECENT_ROWS`] count one second -/// apart ending a second before the evaluation, and `h7`'s 100 older rows -/// 10 to 11.65 minutes before it. -fn rows() -> Vec<(i64, &'static str, &'static str)> { - let mut rows = Vec::new(); - for (city, host, count) in RECENT_ROWS { - for i in 0..count { - rows.push((EVALUATION_MS - 1000 * (i + 1), city, host)); - } - } - for i in 0..100 { - rows.push((EVALUATION_MS - 600_000 - 1000 * i, "nyc", "h7")); - } - rows -} - -/// The exact answer: each city's top 5 hosts by recent row count. -fn expected_top_5_per_city() -> BTreeSet<(String, String, i64)> { - let mut per_city: BTreeMap<&str, Vec<(&str, i64)>> = BTreeMap::new(); - for (city, host, count) in RECENT_ROWS { - per_city.entry(city).or_default().push((host, count)); - } - per_city - .into_iter() - .flat_map(|(city, mut hosts)| { - hosts.sort_by_key(|&(_, count)| std::cmp::Reverse(count)); - hosts - .into_iter() - .take(5) - .map(move |(host, count)| (city.to_string(), host.to_string(), count)) - }) - .collect() -} - -/// Each answer row's `city` (when it has one), `host` and the ranked count -/// (the column named `ranked`), read by name. -fn ranked_rows(batches: &[Batch], ranked: &str) -> BTreeSet<(String, String, i64)> { - batches - .iter() - .flat_map(|batch| { - let names: Vec<&str> = batch - .schema() - .fields - .iter() - .map(|f| f.name.as_str()) - .collect(); - let column = |name: &str| names.iter().position(|n| *n == name); - let (city, host, count) = ( - column("city"), - column("host").unwrap_or_else(|| panic!("a host column in {names:?}")), - column(ranked).unwrap_or_else(|| panic!("a {ranked} column in {names:?}")), - ); - batch - .to_rows() - .into_iter() - .map(move |row| { - let text = |index: usize| match &row[index] { - Value::Utf8(text) => text.to_string(), - other => panic!("a label: {other:?}"), - }; - let count = match row[count] { - Value::Int64(n) => n, - Value::Float64(x) if x.fract() == 0.0 => x as i64, - ref other => panic!("a count: {other:?}"), - }; - (city.map(text).unwrap_or_default(), text(host), count) - }) - .collect::>() - }) - .collect() -} - -/// Query `query` of `plan` with every node at query time, at -/// [`EVALUATION_MS`], with `metrics` bound to [`rows`]: its batches. -fn run_sql(plan: &InstallablePlan, query: usize) -> Vec { - let reference = plan - .dag - .with_every_node_at_query_time() - .expect("the plan runs at query time"); - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .expect("the plan scans the table"); - let rows = rows() - .into_iter() - .map(|(ts, city, host)| { - vec![ - Value::Timestamp(ts), - Value::Utf8(city.into()), - Value::Utf8(host.into()), - Value::Float64(1.0), - ] - }) - .collect(); - let batch = Batch::try_new(schema.clone(), rows).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data( - DataSource::Table { - table_ref: "metrics".into(), - }, - schema, - vec![batch], - ) - .unwrap(); - let roots = [reference.roots[query] as u64]; - let bound = compile(&reference, &[], &roots) - .and_then(|compiled| compiled.instantiate(Arc::new(store))) - .expect("the plan binds to the table"); - let batches = collect(bound.execute(&roots, query_scope()).unwrap().remove(0)); - batches -} - -/// The PromQL plan's query-time reference at [`EVALUATION_MS`], each table -/// row of [`rows`] a sample of `metrics{city, host}`: its batches. -fn run_promql(plan: &InstallablePlan) -> Vec { - let samples: Vec = rows() - .into_iter() - .map(|(timestamp_ms, city, host)| RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), "metrics".to_string()), - ("city".to_string(), city.to_string()), - ("host".to_string(), host.to_string()), - ]), - timestamp_ms, - value: 1.0, - }) - .collect(); - let reference = query_time_reference(plan).unwrap(); - let dag = &reference.dag; - let roots = [reference.dag.roots[0] as u64]; - let compiled = compile(dag, &[], &roots).expect("the executor compiles the plan"); - // The samples bound at the plan's `Scan` of `metrics`. - let instance = compiled - .instantiate(Arc::new(time_series_tables(dag, &samples))) - .unwrap(); - let batches = collect( - instance - .execute(compiled.roots(), query_scope()) - .unwrap() - .remove(0), - ); - batches -} - -fn query_scope() -> RunContext { - RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap() -} - -/// Every batch of one query's `output`. -fn collect(mut output: impl futures::Stream> + Unpin) -> Vec -where - B: std::ops::Deref, - E: std::fmt::Debug, -{ - block_on(async { - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.expect("the query runs")).clone()); - } - batches - }) -} - -/// The exact per-city top 5 compiles and answers each city's top hosts -/// with their counts, `sf` with its two. -#[tokio::test] -async fn an_exact_sql_per_city_top_5_runs() { - let plan = plan_sql(&[TOP_5_HOSTS_PER_CITY], EXACT).await; - plan.check_compiles().unwrap(); - let answered = ranked_rows(&run_sql(&plan, 0), "topk_5"); - assert_eq!(answered, expected_top_5_per_city()); -} - -/// SQL's `ROW_NUMBER` per-city top 5 and PromQL's `topk by (city)` answer -/// the same rows over the same data. -#[tokio::test] -async fn sql_and_promql_per_city_top_5_answer_alike() { - let sql = ranked_rows( - &run_sql(&plan_sql(&[TOP_5_HOSTS_PER_CITY], EXACT).await, 0), - "topk_5", - ); - let promql = ranked_rows( - &run_promql(&plan_promql(PROMQL_TOP_5_HOSTS_PER_CITY).await), - "topk_5", - ); - assert_eq!(sql, promql); - assert_eq!(sql, expected_top_5_per_city()); -} - -/// The global top 5 answers its hosts, not only their counts. -#[tokio::test] -async fn an_exact_sql_top_5_answers_its_hosts() { - let plan = plan_sql(&[TOP_5_HOSTS], EXACT).await; - let answered: BTreeSet<(String, i64)> = ranked_rows(&run_sql(&plan, 0), "topk_5") - .into_iter() - .map(|(_, host, count)| (host, count)) - .collect(); - let expected = [("h1", 9), ("h2", 8), ("h3", 7), ("h4", 6), ("h5", 5)] - .map(|(host, count)| (host.to_string(), count)); - assert_eq!(answered, BTreeSet::from(expected)); -} - -/// The issue's two queries under ε 0.01, δ 0.001: the plan Stage 3 selects -/// installs, so the executor refreshes every materialized view it keeps and -/// compiles its query run. -#[tokio::test] -async fn the_selected_approximate_plan_installs_and_compiles() { - let plan = plan_sql(&[TOP_5_HOSTS, TOP_5_HOSTS_PER_CITY], APPROXIMATE).await; - plan.check_compiles().unwrap(); -} - -/// With `rn` selected, the per-city top 5 keeps its window function, which -/// the executor does not run, so no plan of the workload compiles. Stage 3 -/// does not compile candidates; installation compiles the selected plan -/// (each view's refresh, then the query run) and refuses it with the -/// executor's reason rather than returning a plan a runtime cannot run. -#[tokio::test] -async fn a_plan_the_executor_cannot_run_is_refused_at_installation() { - let with_rn = - TOP_5_HOSTS_PER_CITY.replace("SELECT city, host, c FROM", "SELECT city, host, c, rn FROM"); - let catalog = SqlCatalog::new().with_table("metrics", metrics_schema()); - let planned = plan_installable(UserInput::new( - &workload( - r#"{"sql": "datafusion_sql"}"#, - &[TOP_5_HOSTS, &with_rn], - APPROXIMATE, - ), - FrontendInput::Sql { catalog: &catalog }, - PlanningModels::builtin(), - )) - .await; - match planned { - Err(asap_planner::PlanError::Installable( - error @ (InstallablePlanError::MaterializedViewNotExecutable { .. } - | InstallablePlanError::QueryRunNotExecutable { .. }), - )) => { - let diagnostic = error.to_string(); - assert!( - diagnostic.contains("value operation has no native implementation"), - "{diagnostic}" - ); - } - other => panic!("no plan is installed: {:?}", other.map(|_| ())), - } -} - -/// A per-city top 5 the heavy-hitter `TopK` does not cover runs exactly as -/// the per-partition `Limit{Sort}`, with the query's own columns: ordered by -/// the count then the host (a tie-breaking second key), and with a SELECT -/// list that drops the partition column. -#[tokio::test] -async fn generic_per_city_top_5_forms_run_exactly() { - let tie_broken = TOP_5_HOSTS_PER_CITY.replace("COUNT(*) DESC)", "COUNT(*) DESC, host)"); - let plan = plan_sql(&[&tie_broken], EXACT).await; - assert_eq!( - ranked_rows(&run_sql(&plan, 0), "c"), - expected_top_5_per_city() - ); - let hosts_only = - TOP_5_HOSTS_PER_CITY.replace("SELECT city, host, c FROM", "SELECT host, c FROM"); - let plan = plan_sql(&[&hosts_only], EXACT).await; - let expected: BTreeSet<_> = expected_top_5_per_city() - .into_iter() - .map(|(_, host, count)| (String::new(), host, count)) - .collect(); - assert_eq!(ranked_rows(&run_sql(&plan, 0), "c"), expected); -} diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 69dd65cf4..6dda13659 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -12,18 +12,16 @@ use std::sync::Arc; use asap_executor::physical_planner::compile; use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::sources::{ - DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, -}; +use asap_executor::sources::InMemoryDataStore; use asap_executor::values::{Batch, Value}; use asap_planner::{ plan_installable, DeploymentCapabilities, FrontendInput, InstallablePlan, MaterializedViewRefresh, PlanningModels, PlanningWorkload, SqlCatalog, UserInput, }; -use asap_types::ir::materialized_view::MaterializedView; + use asap_types::ir::operator::Source; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; +use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::NonASAPOp; use futures::{executor::block_on, StreamExt}; @@ -78,20 +76,6 @@ const APPROXIMATE: &str = r#"{"accuracy_metric": "relative", "max_error": 0.05, /// No approximation. const EXACT: &str = r#"{"max_error": 0.0, "failure_probability": 0.0}"#; -async fn plan(query: &str, accuracy: &str) -> InstallablePlan { - plan_all(&[query], accuracy).await -} - -async fn plan_all(queries: &[&str], accuracy: &str) -> InstallablePlan { - plan_for( - queries, - accuracy, - MINUTE_MS as u64, - &DeploymentCapabilities::default(), - ) - .await -} - /// [`plan_all`] for a deployment that keeps no raw data: every window is /// read by state maintained at ingestion time. async fn plan_without_raw_data(queries: &[&str], accuracy: &str) -> InstallablePlan { @@ -147,74 +131,6 @@ fn rows() -> Vec> { .collect() } -/// Runs query 0 of `plan`, every node at query time (its reference plan, -/// which reads the table's rows wherever the plan keeps state), at -/// [`EVALUATION_MS`] with `http` bound to [`rows`]: its output field types -/// and its rows. A `WHERE ts >= now() - INTERVAL` window is a `TimeRange` -/// the executor runs as a filter on `ts` against the evaluation time. -fn run(plan: &InstallablePlan) -> (Vec, Vec>) { - run_query(plan, 0) -} - -/// [`run`] for query `query` of `plan`. -fn run_query(plan: &InstallablePlan, query: usize) -> (Vec, Vec>) { - let reference = plan - .dag - .with_every_node_at_query_time() - .expect("the plan runs at query time"); - // The table's rows in the schema the plan's scan reads (its columns - // qualified by the table). - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .expect("the plan scans the table"); - let batch = Batch::try_new(schema.clone(), rows()).unwrap(); - let store = InMemoryDataStore::default() - .with_raw_data( - Source::Table { - table_ref: "http".into(), - }, - schema, - vec![batch], - ) - .unwrap(); - let roots = [reference.roots[query] as u64]; - let bound = compile(&reference, &[], &roots) - .and_then(|compiled| compiled.instantiate(Arc::new(store))) - .expect("the plan binds to the table"); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap(); - block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut fields = None; - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - let batch = batch.expect("the query runs"); - fields.get_or_insert_with(|| { - batch - .schema() - .fields - .iter() - .map(|f| f.dtype.clone()) - .collect() - }); - rows.extend(batch.to_rows()); - } - (fields.unwrap_or_default(), rows) - }) -} - /// The answer rows by their first column, each value as it prints (its /// type and value). fn by_key(rows: Vec>) -> BTreeMap { @@ -235,17 +151,6 @@ fn shown(value: &Value) -> String { } } -/// The planned output type of query 0's root. -fn planned_output(plan: &InstallablePlan) -> Vec { - let root = plan.dag.roots[0]; - let node = plan.dag.nodes.iter().find(|n| n.id == root).unwrap(); - node.output_schema - .fields - .iter() - .map(|f| f.dtype.clone()) - .collect() -} - const DISTINCT_USERS_LAST_5_MINUTES: &str = "SELECT service, COUNT(DISTINCT user_id) AS value \ FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; @@ -258,159 +163,6 @@ fn expected_distinct_users() -> BTreeMap { .collect() } -#[tokio::test] -async fn a_distinct_count_over_an_int64_id_in_a_now_window_runs_from_a_sketch() { - let plan = plan(DISTINCT_USERS_LAST_5_MINUTES, APPROXIMATE).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("SummaryAgg"), - "a distinct-count sketch answers it: {selected}" - ); - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - // The sketch is exact below its capacity. - assert_eq!(by_key(rows), expected_distinct_users()); -} - -#[tokio::test] -async fn an_exact_distinct_count_over_an_int64_id_in_a_now_window_runs() { - let plan = plan(DISTINCT_USERS_LAST_5_MINUTES, EXACT).await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - assert_eq!(by_key(rows), expected_distinct_users()); -} - -#[tokio::test] -async fn now_is_the_evaluation_time() { - let plan = plan( - "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts > now() - INTERVAL '1 minute' GROUP BY service", - EXACT, - ) - .await; - let (_, rows) = run(&plan); - // The 60 rows after the minute before the evaluation: 20 per service. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(20)))) - .collect(); - assert_eq!(by_key(rows), expected); -} - -/// SQL `percentile_cont` of `values`: linear interpolation at rank -/// `q * (n - 1)` over the sorted values. -fn percentile_cont(q: f64, mut values: Vec) -> f64 { - values.sort_by(f64::total_cmp); - let rank = q * (values.len() - 1) as f64; - let (low, high) = (rank.floor() as usize, rank.ceil() as usize); - values[low] + (values[high] - values[low]) * (rank - low as f64) -} - -#[tokio::test] -async fn an_exact_grouped_sql_percentile_runs() { - let plan = plan( - "SELECT service, approx_percentile_cont(latency, 0.99) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service", - EXACT, - ) - .await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - // Each service's latencies `i` over the last 5 minutes. The window is a - // range selector's `(T - 5m, T]`, so the row stamped exactly `T - 5m` - // (i = 899) is not read: i >= 900. - let expected: BTreeMap = (0..3i64) - .map(|s| { - let latencies = (900..1200) - .filter(|i| i % 3 == s) - .map(|i| i as f64) - .collect(); - ( - format!("s{s}"), - shown(&Value::Float64(percentile_cont(0.99, latencies))), - ) - }) - .collect(); - assert_eq!(by_key(rows), expected); -} - -/// A global percentile over an empty window is NULL (SQL), and the plan -/// types it nullable. -#[tokio::test] -async fn an_exact_global_sql_percentile_of_no_row_is_null() { - let plan = plan( - "SELECT approx_percentile_cont(latency, 0.5) AS value FROM http \ - WHERE ts > now()", - EXACT, - ) - .await; - let (fields, rows) = run(&plan); - assert_eq!( - fields, - planned_output(&plan), - "the executor's output type is the plan's" - ); - assert_eq!(rows.len(), 1); - assert!(matches!(rows[0][0], Value::Null), "{:?}", rows[0][0]); -} - -/// A 5 and a 15 minute distinct count over one table: the plan builds them -/// in tumbling windows, each tumbling window a `TimeRange` over a shifted table `Scan`, -/// which the executor runs as filters on `ts`. Both answer exactly (the -/// sketch is exact below its capacity). -#[tokio::test] -async fn distinct_counts_over_5_and_15_minute_windows_run_from_tumbling_windows() { - let fifteen = DISTINCT_USERS_LAST_5_MINUTES.replace("5 minutes", "15 minutes"); - let plan = plan_all(&[DISTINCT_USERS_LAST_5_MINUTES, &fifteen], APPROXIMATE).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("TimeShift"), - "tumbling windows: {selected}" - ); - assert_eq!(by_key(run_query(&plan, 0).1), expected_distinct_users()); - // The last 15 minutes (i >= 300): every user `i % 50` of each service - // (3 and 50 are coprime) and its 7 recent ones. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(57)))) - .collect(); - assert_eq!(by_key(run_query(&plan, 1).1), expected); -} - -/// Exact row counts over 5 and 15 minutes: a count tells a row read twice -/// or missed at a tumbling window's edge, which a distinct count cannot. Each tumbling window is -/// `(T - s - w, T - s]`, so the 15-minute tumbling windows read every row after -/// `T - 15m` exactly once: 300 per service (the row at `T - 15m` is not -/// read), and 100 in the last 5 minutes. -#[tokio::test] -async fn row_counts_over_5_and_15_minute_windows_read_each_row_once() { - let five = "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service"; - let fifteen = five.replace("5 minutes", "15 minutes"); - let plan = plan_all(&[five, &fifteen], EXACT).await; - let selected = format!("{:?}", plan.dag.nodes); - assert!( - selected.contains("TimeShift"), - "tumbling windows: {selected}" - ); - for (query, rows_per_service) in [(0, 100), (1, 300)] { - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(rows_per_service)))) - .collect(); - assert_eq!(by_key(run_query(&plan, query).1), expected, "query {query}"); - } -} - /// Runs `ids` of `dag` in `scope` with each of `inputs` bound to the node /// its key names and `table` bound as `http`: the batches of each id's /// output. @@ -641,106 +393,3 @@ async fn a_window_equal_to_its_interval_is_built_at_ingestion_time_from_table_ro .collect(); assert_eq!(by_key(run_data_plane(&plan).remove(0)), expected); } - -/// Each run's requested `(start, end]` time range, in order. -type RequestedTimeRanges = std::sync::Mutex>>; - -/// A store whose `http` table records each run's requested time range and -/// returns every row (it ignores the request). -struct RecordingStore { - rows: InMemoryDataStore, - requested_time_ranges_ms: Arc, -} -impl DataStore for RecordingStore { - fn read_raw_data( - &self, - source: &Source, - request: &RawDataRequest, - context: RunContext, - ) -> Result, asap_executor::Error> { - self.requested_time_ranges_ms - .lock() - .unwrap() - .push(request.time_range_ms); - self.rows.read_raw_data(source, request, context) - } - fn read_materialized_view( - &self, - view: &MaterializedView, - request: &MaterializedViewRequest, - context: RunContext, - ) -> Result, asap_executor::Error> { - self.rows.read_materialized_view(view, request, context) - } -} - -// The store's `http` table of a `ts >= now() - INTERVAL '5 minutes'` query is -// asked for `(T - 5m, T]` at evaluation time `T`; it returns all 20 minutes -// of rows, and the answer is the one over the window. -#[tokio::test] -async fn a_data_store_is_asked_for_the_query_window() { - // The user's query, planned exactly over the `http` table. - let plan = plan( - "SELECT service, COUNT(*) AS value FROM http \ - WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY service", - EXACT, - ) - .await; - // The plan with every node at query time: it reads the table at the Scan. - let reference = plan.dag.with_every_node_at_query_time().unwrap(); - // The schema the plan's Scan reads. - let schema = reference - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { schema, .. }) => { - Some(Arc::new(schema.clone())) - } - _ => None, - }) - .unwrap(); - // Each run's requested time range, as the store saw it. - let requested_time_ranges_ms = Arc::new(RequestedTimeRanges::default()); - // The deployment's store: `http` → its rows, recording each request. - let store = RecordingStore { - rows: InMemoryDataStore::default() - .with_raw_data( - Source::Table { - table_ref: "http".into(), - }, - schema.clone(), - vec![Batch::try_new(schema, rows()).unwrap()], - ) - .unwrap(), - requested_time_ranges_ms: requested_time_ranges_ms.clone(), - }; - // Compiled once (no reader opened), then instantiated for this run. - let roots = [reference.roots[0] as u64]; - let compiled = compile(&reference, &[], &roots).unwrap(); - let bound = compiled.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }, - Limits::default(), - ) - .unwrap(); - // The answer rows: each service and its count. - let rows = block_on(async { - let mut output = bound.execute(&roots, context).unwrap().remove(0); - let mut rows = Vec::new(); - while let Some(batch) = output.next().await { - rows.extend(batch.unwrap().to_rows()); - } - rows - }); - assert_eq!( - *requested_time_ranges_ms.lock().unwrap(), - vec![Some((EVALUATION_MS - 5 * MINUTE_MS, EVALUATION_MS))] - ); - // The 300 rows of the window, 100 per service. - let expected: BTreeMap = (0..3) - .map(|s| (format!("s{s}"), shown(&Value::Int64(100)))) - .collect(); - assert_eq!(by_key(rows), expected); -} diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index bc7658d28..d4d2dc649 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -4,20 +4,18 @@ //! plan selection still selects an exact plan for at least one of their //! queries (#580). -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; +use asap_types::ir::{ASAPOp, OperatorNode}; use std::rc::Rc; -use asap_accuracy::test_support::{default_size_params, DEFAULT_DELTA}; -use asap_frontend_promql::{lower_promql_query_workload, promql_metric_names}; +use asap_frontend_promql::promql_metric_names; use asap_frontend_sql::SqlCatalog; -use asap_plan_selection::reference_pricer::WholePlanSelection; -use asap_plan_selection::RejectionCause; + use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_planner::{PlanOutput, PlanningModels, QueryPlan}; -use asap_types::ir::operator::agg_intent::default_quantile; + use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; + use asap_types::workload::SqlDialect; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, @@ -86,166 +84,6 @@ async fn plan_promql(queries: &[(&str, f64)]) -> PlanOutput { e2e_plan(input).await.expect("workload plans") } -/// Explicit exact, all-KLL, and all-DDS assignments for composition checks. -/// These fixtures test candidate support and rejection independently of which -/// candidates a bounded search happens to visit. -fn composed_summary_candidates( - queries: &[(&str, f64)], -) -> ( - Vec, - WholePlanSelection, -) { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let workload = promql_workload(queries); - let roots = lower_promql_query_workload(&workload).expect("lowers"); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut assignments = vec![vec![0; inventory.targets.len()]]; - for family in [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] { - let choices: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|candidate| { - *candidate == HowToComputeAggregateMeasure::Summary(family) - }) - .unwrap_or(0) - }) - .collect(); - assert_eq!( - choices.iter().filter(|choice| **choice != 0).count(), - 2, - "both quantile references must explicitly use {family:?}" - ); - assignments.push(choices); - } - let entries = workload.query_workload.entries().collect::>(); - let data = workload.data_workload.as_ref().unwrap(); - let models = PlanningModels::builtin(); - let candidates: Vec<_> = assignments - .into_iter() - .enumerate() - .map(|(index, choices)| { - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - let mut candidate = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "explicit-composition", - &roots, - &entries, - data, - models.deployment_capabilities.query_time_results_retained, - &mut |_, _, _| Some(asap_types::physical::Materialization::NotMaterialized), - ) - .unwrap() - .expect("supported physical computation"); - candidate.id = format!("composition-{index}"); - candidate - }) - .collect(); - let selection = select_plan(&candidates, &entries, data, models) - .expect("the exact candidate meets the requested accuracy"); - (candidates, selection) -} - -/// Construct a legal shared-KLL candidate explicitly, independent of the -/// plan the search selects. -fn shared_kll_candidate(workload: &PlanningWorkload) -> asap_types::physical::PhysicalWorkloadPlan { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::physical::Materialization; - let roots = lower_promql_query_workload(workload).unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::generate_dag_of_logical_memo_groups(&roots), - ) - .unwrap(); - let mut choices = vec![0; inventory.targets.len()]; - for index in 0..choices.len() { - choices[index] = (1..inventory.targets[index].choice_count()).find(|choice| { - let mut trial = vec![0; choices.len()]; - trial[index] = *choice; - compose_logical_candidate(&inventory, &trial).unwrap().iter().any(|root| { - root.operator_refs().into_iter().flat_map(OperatorNode::reachable).any(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if matches!(kind.params(), SketchParams::Kll { .. })) - }) - }) - }).expect("KLL implementation exists"); - } - let roots: Vec<_> = compose_logical_candidate(&inventory, &choices).unwrap(); - // Every identical sub-DAG of the two queries made one node. - let roots: Vec<_> = asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .into_iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, node), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect(); - let mut physical = - asap_physical_asap_optimization::test_support::optimize_physical_asap_chosen( - "shared-kll", - &roots, - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - false, - &mut |_, _, _| Some(Materialization::NotMaterialized), - ) - .unwrap() - .unwrap(); - physical.id = "shared-kll".into(); - select_plan( - std::slice::from_ref(&physical), - &workload.query_workload.entries().collect::>(), - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin(), - ) - .expect("shared candidate meets both accuracy targets"); - physical -} - -fn shared_kll_output(workload: &PlanningWorkload) -> PlanOutput { - let candidate = shared_kll_candidate(workload); - PlanOutput::new( - candidate - .roots - .into_iter() - .enumerate() - .map(|(entry_index, root)| { - let QueryRoot::Operator(root) = root else { - panic!("operator fixture") - }; - QueryPlan { entry_index, root } - }) - .collect(), - ) -} - -/// Whether `root` reads a sketch state. -fn reads_sketch(root: &Rc) -> bool { - OperatorNode::reachable(root).into_iter().any(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - }) - ) - }) -} - async fn plan_sql(queries: &[&str]) -> PlanOutput { let workload = PlanningWorkload { query_workload: QueryWorkload { @@ -330,153 +168,6 @@ fn unique_deployments(output: &PlanOutput) -> usize { seen.len() } -/// An explicitly connected KLL candidate shares one physical producer and -/// satisfies both outputs' accuracy contracts. -#[test] -fn quantiles_with_equal_params_share_one_producer() { - let workload = promql_workload(&[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ]); - let cheapest = shared_kll_candidate(&workload); - let states: Vec>> = cheapest - .roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect() - }) - .collect(); - assert_eq!(states[0].len(), 1, "{}", cheapest.id); - assert!(same_states(&states), "{}", cheapest.id); -} - -/// Shared producer identity survives physicalization for a high-volume source; -/// this checks feasibility, independently of the cost optimizer's winner. -#[tokio::test] -async fn quantiles_with_equal_params_deploy_one_producer() { - let mut workload = promql_workload(&[ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[5m])", 0.01), - ]); - let data = workload.data_workload.as_mut().unwrap(); - data.arrival = DataArrival::Ingesting { - rows_per_second: 666_666.67, - }; - let lat = data.sources.get_mut("lat").unwrap(); - lat.series_count = Some(1_000_000); - lat.sample_interval = Some(DurationMs(1_500)); - let output = shared_kll_output(&workload); - assert!(same_states(&states(&output))); - assert!(!Rc::ptr_eq(&output.plans[0].root, &output.plans[1].root)); - assert_eq!(unique_deployments(&output), 1); -} - -/// A different window or label selector is a different producer, even when -/// one query asks for a stricter accuracy than the other. -#[tokio::test] -#[ignore = "plan selection still selects an exact plan for at least one query: with the built-in cost model its query-time summary costs more than exact (#580)"] -async fn different_producers_are_not_shared() { - for queries in [ - [ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[10m])", 0.01), - ], - [ - ("quantile_over_time(0.5, lat[5m])", 0.01), - ("quantile_over_time(0.99, lat[10m])", 0.001), - ], - [ - ("quantile_over_time(0.5, lat{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.001), - ], - [ - ("quantile_over_time(0.5, lat{job=\"a\"}[5m])", 0.01), - ("quantile_over_time(0.99, lat{job=\"b\"}[5m])", 0.01), - ], - ] { - let output = plan_promql(&queries).await; - assert!(!same_states(&states(&output)), "{queries:?}"); - assert_eq!(unique_deployments(&output), 2, "{queries:?}"); - for (plan, (_, epsilon)) in output.plans.iter().zip(queries) { - assert_eq!(kll_k(plan), kll_k_for(epsilon), "{queries:?}"); - } - } -} - -/// The KLL `k` of the one state a plan deploys. -fn kll_k(plan: &QueryPlan) -> u32 { - let states = plan_states(plan); - let [deployment] = states.as_slice() else { - panic!("one state: {:?}", states.len()); - }; - let asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &deployment.operator - else { - panic!("sketch state: {:?}", deployment.operator); - }; - let SketchParams::Kll { k, .. } = kind.params() else { - panic!("KLL state: {kind:?}"); - }; - *k -} - -/// The KLL `k` sized for `epsilon`. -fn kll_k_for(epsilon: f64) -> u32 { - let SketchParams::Kll { k, .. } = default_size_params( - SketchAlgorithm::Kll, - &default_quantile(0.5), - epsilon, - DEFAULT_DELTA, - ) else { - unreachable!() - }; - k -} - -/// p50 at ε=0.01 and p99 at ε=0.001 over the same input share one KLL sized -/// for the strictest consumer; each reader's guarantee meets its own target. -#[tokio::test] -#[ignore = "plan selection still selects an exact plan for at least one query: with the built-in cost model its query-time summary costs more than exact (#580)"] -async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { - let p50 = ("quantile_over_time(0.5, lat[5m])", 0.01); - let p99 = ("quantile_over_time(0.99, lat[5m])", 0.001); - assert!(kll_k_for(0.001) > kll_k_for(0.01)); - - let output = plan_promql(&[p50, p99]).await; - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); - for (plan, (_, epsilon)) in output.plans.iter().zip([p50, p99]) { - assert_eq!(kll_k(plan), kll_k_for(0.001)); - let guarantee = plan.root.guarantee.as_ref().expect("certified"); - assert!( - guarantee.spec.error.evaluate().unwrap() <= epsilon, - "{guarantee:?}" - ); - } - - // Alone, the looser query keeps its own, smaller KLL. - let alone = plan_promql(&[p50]).await; - assert_eq!(kll_k(&alone.plans[0]), kll_k_for(0.01)); -} - -/// Cross-series quantiles name their KLL state after the input column, not the -/// quantile, so p50 and p99 over one selector share it. -#[tokio::test] -async fn cross_series_p50_and_p99_share_one_producer() { - let output = shared_kll_output(&promql_workload(&[ - ("quantile(0.5, lat)", 0.01), - ("quantile(0.99, lat)", 0.01), - ])); - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); -} - /// Two identical ungrouped queries: their identical producers (rate, then /// sum) are shared. #[tokio::test] @@ -547,124 +238,6 @@ async fn sql_p50_and_p99_share_one_producer() { } } -/// The operator sub-DAG a standalone scalar query reads is planned like an -/// operator root (#511 §2.3): the scalar root is rebuilt over its selected -/// sub-DAG, and entries keep their order around a literal scalar query. -#[tokio::test] -async fn scalar_queries_read_planned_sub_dags() { - let output = plan_promql(&[ - ("1", 0.01), - ("scalar(quantile(0.99, rate(x[5m])))", 0.01), - ("quantile(0.99, rate(x[5m]))", 0.01), - ]) - .await; - assert_eq!(output.entry_indices(), vec![0, 1, 2]); - let scalar_entries: Vec<_> = output.scalar_roots.iter().map(|(i, _)| *i).collect(); - assert_eq!(scalar_entries, vec![0, 1]); - assert_eq!(output.plans.len(), 1); - assert_eq!(output.plans[0].entry_index, 2); - - assert_eq!(output.scalar_plans.len(), 1); - let read = &output.scalar_plans[0]; - assert_eq!(read.entry_index, 1); - let refs = output.scalar_roots[1].1.operator_refs(); - assert_eq!(refs.len(), 1); - assert!(Rc::ptr_eq(refs[0], &read.root)); - - // The export keys each scalar query by its entry and points the scalar - // read at an exported node. - let dag = output.physical_asap_dag().unwrap(); - let keys: Vec<_> = dag.scalar_roots.iter().map(|(i, _)| *i).collect(); - assert_eq!(keys, vec![0, 1]); - assert_eq!(dag.roots.len(), 1); - let [read] = dag.scalar_roots[1].1.operator_refs()[..] else { - panic!("one read"); - }; - assert!(dag.nodes.iter().any(|node| node.id == *read)); -} - -/// A combination of approximate values keeps its summary candidates: each -/// reference of `scalar(q99) - scalar(q50)` is planned against the entry's -/// target, so a candidate reads a sketch in both. A difference of rank (KLL) -/// or relative (DDSketch) errors composes to no guarantee in the MVP (#669) -/// and the built-in models have no empirical output-level evidence, so plan -/// selection rejects it with that reason. -#[test] -fn combined_scalar_references_keep_summary_candidates() { - let (candidates, selection) = composed_summary_candidates(&[( - "scalar(quantile(0.99, rate(x[5m]))) - scalar(quantile(0.5, rate(x[5m])))", - 0.01, - )]); - for both in candidates.iter().skip(1) { - let refs = both.roots[0].operator_refs(); - assert_eq!(refs.len(), 2); - assert!( - refs.into_iter().all(reads_sketch), - "both references use the explicit sketch family" - ); - let rejection = selection - .rejected - .iter() - .find(|r| r.id == both.id) - .expect("not selected"); - assert_eq!( - rejection.cause, - RejectionCause::NoAccuracyGuarantee, - "{rejection:?}" - ); - assert!( - rejection.reason.contains("no guarantee"), - "{}", - rejection.reason - ); - } - assert_eq!( - selection.selected, candidates[0].id, - "the exact assignment is selected" - ); -} - -/// Candidate generation supports summary implementations of the vector -/// operands, but a difference or a comparison of -/// estimates composes to no guarantee in the MVP (#669; a comparison's band -/// guarantee is later work) and the built-in models have no empirical -/// output-level evidence (plan selection, #576), so it is not selected -/// automatically: the exact plan is. -#[tokio::test] -async fn combined_vector_quantiles_keep_unselected_summary_candidates() { - for query in [ - "quantile(0.99, rate(x[5m])) - quantile(0.5, rate(x[5m]))", - "quantile(0.99, rate(x[5m])) > quantile(0.5, rate(x[5m]))", - ] { - let (candidates, selection) = composed_summary_candidates(&[(query, 0.01)]); - for summary in candidates.iter().skip(1) { - let QueryRoot::Operator(root) = &summary.roots[0] else { - panic!("{query}: operator root"); - }; - assert!(reads_sketch(root), "{query}: explicit summary candidate"); - assert!(root.guarantee.is_none(), "{query}"); - let rejection = selection - .rejected - .iter() - .find(|rejection| rejection.id == summary.id) - .expect("uncertified vector computation is rejected"); - assert_eq!( - rejection.cause, - RejectionCause::NoAccuracyGuarantee, - "{query}: {rejection:?}" - ); - assert!(rejection.reason.contains("no guarantee"), "{rejection:?}"); - } - assert_eq!( - selection.selected, candidates[0].id, - "{query}: exact assignment selected" - ); - - let output = plan_promql(&[(query, 0.01)]).await; - assert!(!reads_sketch(&output.plans[0].root), "{query}"); - } -} - /// Front ends build structurally identical references as separate nodes. /// The selected graph may reuse a producer or choose separate physical /// candidates. Every distinct selected reference must be exported exactly once. @@ -698,71 +271,3 @@ async fn identical_scalar_references_satisfy_the_plan_contract() { } } } - -/// A root whose selected candidate is an exact state is a query result: it -/// is read through `FinalizeExactAccumulator`, so its output is plain values, -/// not accumulator state. -#[tokio::test] -async fn exact_state_roots_are_finalized() { - for query in [ - "sum_over_time(m[1m])", - "sum(rate(x[5m]))", - "sum(sum_over_time(m[1m]) + sum_over_time(n[1m]))", - ] { - let output = plan_promql(&[(query, 0.01)]).await; - let root = &output.plans[0].root; - assert!( - matches!(root.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })), - "{query}: {root:#?}" - ); - let dag = output.physical_asap_dag().unwrap(); - let root = dag - .nodes - .iter() - .find(|node| node.id == dag.roots[0]) - .unwrap(); - assert!( - root.output_schema - .fields - .iter() - .all(|field| field.is_plain()), - "{query}: {:?}", - root.output_schema - ); - // With nothing materialized, the root and the exact state it reads - // run at query time. - use asap_types::ir::properties::{DataPrimitive, ExecutionDataState, ExecutionTiming}; - assert_eq!(root.output_state, ExecutionDataState::QUERY_ROWS, "{query}"); - let state = dag - .edges - .iter() - .find(|edge| edge.consumer == root.id) - .map(|edge| edge.producer) - .unwrap(); - let state = dag.nodes.iter().find(|node| node.id == state).unwrap(); - assert_eq!( - state.output_state, - ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive: DataPrimitive::SummaryState, - }, - "{query}" - ); - } -} - -/// The reference pricer over `candidates`, every candidate kept. -fn select_plan( - candidates: &[asap_types::physical::PhysicalWorkloadPlan], - entries: &[asap_types::workload::QueryWorkloadEntry], - data: &asap_types::workload::DataWorkload, - models: PlanningModels<'_>, -) -> Result { - asap_plan_selection::reference_pricer::select_among_plans( - candidates, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - ) -} diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index ac56e5b5f..f5c940069 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -15,10 +15,8 @@ use serde::{Deserialize, Serialize}; use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::schema::Schema; -use crate::physical::materialization::Materialization; /// Index of a node in [`FlatDag::nodes`]. pub type NodeId = usize; @@ -30,8 +28,6 @@ pub struct FlatNode { pub(crate) result_kind: OperatorResultKind, pub(crate) schema: Schema, pub(crate) guarantee: Option, - pub(crate) timing: Option, - pub(crate) materialization: Materialization, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -78,8 +74,6 @@ fn visit( result_kind: node.result_kind, schema: node.schema.clone(), guarantee: node.guarantee.clone(), - timing: node.timing, - materialization: node.materialization, }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id); @@ -165,7 +159,6 @@ mod tests { #[test] fn json_round_trips() { let (dag, _) = flatten(&[QueryRoot::Operator(filter(values()))]); - assert!(dag.nodes.iter().all(|n| n.timing.is_none())); let json = serde_json::to_string(&dag).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), dag); } diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index 6a3c2de29..8dc00ba18 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -6,9 +6,9 @@ //! ([`PhysicalASAPDAG::set_retention`]). //! //! A **materialized view** is a node whose output is kept across -//! evaluations: its materialization is `MatAtIngestion` or -//! `MatAtQueryKeptAcrossEvaluations`, and it carries its `retention_ms`. A -//! `MatAtQuery` node is not a view: its output is discarded after one +//! evaluations: its lifetime is `AcrossEvaluations`, at ingestion time or +//! at query time, and it carries its `retention_ms`. A node materialized for +//! `OneEvaluation` is not a view: its output is discarded after one //! evaluation. A query-time `Scan` carries how long its source's rows are //! kept for it (`None` when it reads them unbounded). //! @@ -79,7 +79,7 @@ use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::operator::operator_properties::Source; use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, SummaryCoverage}; -use crate::physical::materialization::Materialization; +use crate::physical::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is /// refreshed from, when, and for how long it is kept. @@ -235,13 +235,13 @@ impl RawDataWindow { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum MaterializedViewRefresh { - /// Refreshed at ingestion time (`MatAtIngestion`): the runtime runs the + /// Refreshed at ingestion time (materialized at ingestion time): the runtime runs the /// node in an ingestion run once its inputs are complete (its windows' /// samples have arrived and the views it reads are refreshed), and /// keeps the result. AtIngestion, /// Refreshed at query time and kept across evaluations - /// (`MatAtQueryKeptAcrossEvaluations`): when an evaluation reads this + /// (query-time `AcrossEvaluations`): when an evaluation reads this /// view for a selection it has not been refreshed for yet, the runtime /// runs the node in a query run over its inputs, keeps the result, and /// binds it like a view refreshed at ingestion. @@ -398,11 +398,9 @@ impl PhysicalASAPDAG { /// covers how far back its data reaches. O(views · nodes) for the /// coverages, plus O(nodes + edges). pub fn materialized_views(&self) -> Result, InstallableDagError> { - if let Some(node) = self - .nodes - .iter() - .find(|node| node.materialization.is_materialized_view() && node.retention_ms.is_none()) - { + if let Some(node) = self.nodes.iter().find(|node| { + matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) && node.retention_ms.is_none() + }) { return Err(InstallableDagError::ViewWithoutRetention { node_id: node.id }); } self.validate()?; @@ -730,7 +728,7 @@ impl PhysicalASAPDAG { let views: HashSet = self .nodes .iter() - .filter(|node| node.materialization.is_materialized_view()) + .filter(|node| matches!(node.lifetime, Lifetime::AcrossEvaluations(_))) .map(|node| node.id) .collect(); // The IR of each node, which coverage is derived from. @@ -764,7 +762,8 @@ impl PhysicalASAPDAG { { window_by_view.insert(id, window); } - if matches!(node.materialization, Materialization::MatAtIngestion(_)) { + let refreshed_at_ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; + if refreshed_at_ingestion { // An ingestion run reads each view for the window ending at // its end, so a view read must end where the raw windows do. let mut end_offsets_ms: Vec = inputs @@ -796,9 +795,10 @@ impl PhysicalASAPDAG { coverage_by_node.insert(id, coverage.clone()); let view = MaterializedView { node_id: id, - refresh: match node.materialization { - Materialization::MatAtIngestion(_) => MaterializedViewRefresh::AtIngestion, - _ => MaterializedViewRefresh::OnFirstRead, + refresh: if refreshed_at_ingestion { + MaterializedViewRefresh::AtIngestion + } else { + MaterializedViewRefresh::OnFirstRead }, coverage: ViewCoverage { definition_id: coverage.definition_id(), diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 026dc620e..b424f16de 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -14,9 +14,3 @@ pub use schema::error::SchemaDerivationError; pub mod flat; pub mod materialized_view; pub mod physical_export; -pub use properties::timing::{ - apply_materialization_timings, apply_materialization_timings_to_roots, data_state, - is_filtered_build, timing_under_consumer, MaterializationAssignment, NodeToTimedNodeCache, -}; -#[cfg(any(test, feature = "test-support"))] -pub use properties::timing::{planned_data_state, validate_at_ingestion}; diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index d2ae4f643..fbadb664d 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -1,7 +1,9 @@ //! The unified operator node: one node type before and after ASAP //! optimization. A node is an operator plus the common planning properties -//! every traversal needs (output category and schema, accuracy guarantee, -//! execution timing). +//! every traversal needs (output category and schema, accuracy guarantee). +//! Physical properties (timing, lifetime, schedule) are not on the node: +//! each physical candidate holds its own +//! ([`crate::stages::PhysicalCandidate::properties`]). use std::collections::HashSet; use std::rc::Rc; @@ -10,12 +12,10 @@ use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; use super::non_asap::NonASAPOp; -use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::properties::summary_coverage::{NodeToSummaryCoverageCache, SummaryCoverage}; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; -use crate::physical::materialization::Materialization; /// The output category of an operator, derived from the operation and its /// inputs. Matching column schemas do not make categories interchangeable. @@ -91,19 +91,13 @@ impl Operator { /// /// `schema` and `result_kind` are derived from `operator` and its children /// at construction and retained. `guarantee` is `None` until accuracy -/// assessment establishes one (`None` never means exact). `timing` is `None` -/// until a materialization assignment is applied; export rejects an executable -/// node without one. `materialization` is the option that assignment chose -/// for this node (`NotMaterialized` until then, and for every node whose -/// output is not stored). +/// assessment establishes one (`None` never means exact). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, pub guarantee: Option, - pub timing: Option, - pub materialization: Materialization, } impl OperatorNode { @@ -127,8 +121,6 @@ impl OperatorNode { result_kind, schema, guarantee: None, - timing: None, - materialization: Materialization::NotMaterialized, } } @@ -144,16 +136,6 @@ impl OperatorNode { self } - pub fn with_timing(mut self, timing: Option) -> Self { - self.timing = timing; - self - } - - pub fn with_materialization(mut self, materialization: Materialization) -> Self { - self.materialization = materialization; - self - } - /// What this summary state covers; `None` for a node that is not a /// `SummaryAgg` or a valid `SummaryMerge`. Derived on each call, since /// the public fields can be edited in place. @@ -237,8 +219,7 @@ impl OperatorNode { /// the old derived schema are retained, for either operator category. A /// change in output arity with such overrides needs an explicit new /// naming assignment. - /// `guarantee`, `timing` and `materialization` depend on the inputs and - /// are cleared. + /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, f: impl FnMut(&Rc) -> Rc, @@ -302,35 +283,11 @@ impl OperatorNode { out } - /// Validate assigned phases without imposing any particular runtime implementation. - #[cfg(any(test, feature = "test-support"))] - pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError> { - self.validate_structure()?; - for node in Self::reachable(self) { - let timing = node.timing.ok_or_else(|| { - SchemaDerivationError::InvalidScalarSignature( - "execution timing is unassigned".into(), - ) - })?; - if timing == crate::ir::properties::ExecutionTiming::IngestionTime - && node.children().iter().any(|child| { - child.timing != Some(crate::ir::properties::ExecutionTiming::IngestionTime) - }) - { - return Err(SchemaDerivationError::InvalidScalarSignature( - "ingestion-time operation depends on a query-time or unassigned input".into(), - )); - } - } - Ok(()) - } - /// Validate the whole DAG reachable from this node: every operator's /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field /// names and qualifiers; all structural metadata must match derivation. - /// `timing` may be `None`. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { // Each summary state's coverage is derived once, not once per merge // above it; `reachable` keeps every node alive while it is used. diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 8416fa701..ca69afbfd 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -11,6 +11,7 @@ //! build over `TimeRange(m·w)` ([`merged_window_build`]): one chain kept for //! the longest window serves every query whose window, on the same tumbling windows, //! ends at the same time and is shorter. +#[cfg(any(test, feature = "test-support"))] use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; @@ -18,9 +19,8 @@ use std::time::Duration; use super::asap::ASAPOp; use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; +#[cfg(any(test, feature = "test-support"))] use super::operator_properties::TimeShift; -use super::query::QueryRoot; -use crate::ir::properties::ExecutionTiming; use crate::ir::SchemaDerivationError; fn invalid(message: &str) -> SchemaDerivationError { @@ -122,6 +122,7 @@ fn newest_tumbling_windows_build( build_over(chain, range(Rc::clone(child), window)?) } +#[cfg(any(test, feature = "test-support"))] /// Tumbling window `index` (0 = newest) of `chain`: its `SummaryAgg` over the range /// `TimeRange(tumbling_window_width_ms)` ending `index · tumbling_window_width_ms` before the /// chain's input. @@ -145,6 +146,7 @@ pub(crate) fn tumbling_window( ) } +#[cfg(any(test, feature = "test-support"))] /// `child` evaluated `back_ms` earlier: one `TimeShift`, folded into a /// `TimeShift` directly below. fn shifted( @@ -228,12 +230,7 @@ fn build_over( /// reads each tumbling window of each chain it read, newest first, as plain /// `SummaryAgg` nodes the executor runs; a merge of the newest `n` tumbling windows /// reads the first `n`, the same nodes a merge of the whole chain reads. Every other node keeps its -/// guarantee, timing and materialization. Each tumbling window takes its chain's -/// timing and materialization, and the range and shift it reads take the -/// chain's timing: a chain materialized at ingestion time expands into -/// tumbling windows each built and stored at ingestion time. Expanding before timings -/// are applied leaves the tumbling windows untimed. A chain read by anything but a -/// `SummaryMerge` is an error. +/// guarantee. A chain read by anything but a `SummaryMerge` is an error. #[cfg(any(test, feature = "test-support"))] pub fn expand_tumbling_window_chains( root: &Rc, @@ -241,110 +238,7 @@ pub fn expand_tumbling_window_chains( expand(root, &mut NodeToExpandedNodeCache::default()) } -/// A workload with its tumbling-window chains expanded -/// ([`expand_tumbling_window_chains_in_workload`]). -#[derive(Debug, Clone)] -pub struct TumblingWindowChainExpansion { - /// The workload's roots, in order, with every chain expanded. - pub roots: Vec, - /// Each expanded chain with its tumbling windows some merge reads, newest first, - /// in the order the chains were first reached. - pub chains: Vec, - /// Every other node of the input workload the expansion reached, with - /// the node that replaces it (itself when nothing below it changed), - /// so a caller can carry what it knows about a node to the expanded - /// workload. - pub expanded_nodes: Vec<(Rc, Rc)>, -} - -/// One expanded chain: the `SummaryTumblingWindowChain` node of the input workload and the -/// `SummaryAgg` tumbling windows that replace it. -#[derive(Debug, Clone)] -pub struct ExpandedTumblingWindowChain { - /// The chain node, as the input workload holds it. - pub chain: Rc, - /// Its tumbling windows in the expanded workload that some merge reads, newest - /// first. - pub tumbling_windows: Vec>, -} - -/// `expand_tumbling_window_chains` over every root of one workload, with one cache, so -/// a node several roots share stays one node and a chain several merges -/// read expands into one set of tumbling windows. A standalone scalar root has the -/// operators it reads expanded. Also returns each chain's tumbling windows, so a -/// caller can carry what it knows about a chain (e.g. its retention) to -/// them. -pub fn expand_tumbling_window_chains_in_workload( - roots: &[QueryRoot], -) -> Result { - let mut cache = NodeToExpandedNodeCache::default(); - let roots = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => expand(node, &mut cache).map(QueryRoot::Operator), - QueryRoot::Scalar(expr) => { - let mut error = None; - let expr = expr.map_operator_refs(&mut |node| { - expand(node, &mut cache).unwrap_or_else(|e| { - error.get_or_insert(e); - Rc::clone(node) - }) - }); - match error { - Some(e) => Err(e), - None => Ok(QueryRoot::Scalar(expr)), - } - } - }) - .collect::, _>>()?; - let chains = cache - .chains - .into_iter() - .map(|chain| { - let tumbling_window_count = match chain.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => *tumbling_window_count, - _ => unreachable!("only chains are recorded"), - }; - let tumbling_windows = (0..tumbling_window_count) - .map_while(|index| { - cache - .tumbling_window_of - .get(&(Rc::as_ptr(&chain), index)) - .cloned() - }) - .collect(); - ExpandedTumblingWindowChain { - chain, - tumbling_windows, - } - }) - .collect(); - Ok(TumblingWindowChainExpansion { - roots, - chains, - expanded_nodes: cache.expanded, - }) -} - -/// The untimed `node` with `timing` written into it and into every untimed -/// node below it: the nodes [`tumbling_window`] created, over the chain's input, which -/// the timing pass already timed. -fn timed_like(node: &Rc, timing: ExecutionTiming) -> OperatorNode { - OperatorNode::with_schema( - node.operator.map_children(|child| match child.timing { - Some(_) => Rc::clone(child), - None => Rc::new(timed_like(child, timing)), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()) - .with_materialization(node.materialization) - .with_timing(Some(timing)) -} - +#[cfg(any(test, feature = "test-support"))] /// What one expansion has built: each expanded node by its input's /// address, and each tumbling window by its chain's address and index, so a chain two /// merges read expands into one set of tumbling windows. @@ -358,6 +252,7 @@ struct NodeToExpandedNodeCache { expanded: Vec<(Rc, Rc)>, } +#[cfg(any(test, feature = "test-support"))] fn expand( node: &Rc, cache: &mut NodeToExpandedNodeCache, @@ -390,13 +285,6 @@ fn expand( continue; } let tumbling_window = Rc::new(tumbling_window(child, index)?); - let tumbling_window = match child.timing { - Some(timing) => Rc::new( - timed_like(&tumbling_window, timing) - .with_materialization(child.materialization), - ), - None => tumbling_window, - }; if index == 0 { cache.chains.push(Rc::clone(child)); } @@ -416,9 +304,7 @@ fn expand( }))?; Rc::new( OperatorNode::with_schema(merge.operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), + .with_guarantee(node.guarantee.clone()), ) } _ => { @@ -436,9 +322,7 @@ fn expand( .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), node.schema.clone(), ) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index ccb07e256..62af7acd7 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -1,11 +1,11 @@ //! Physical ASAP DAG (planner-layering stage 2 output). //! -//! The flattened operators of [`super::flat`], plus the materialization and -//! execution timing (data state) of every node and edge. The input must -//! already be timed ([`crate::ir::properties::timing::apply_materialization_timings`]); export -//! reads each node's timing and materialization and does not re-run -//! data-state validation. The deployment executes the plan as given -//! (#509 Assumption 8): it does not change materialization or timing. +//! The flattened operators of [`super::flat`], plus the lifetime and +//! execution timing (data state) of every node and edge. The realizer +//! (`AssignmentRealizer::realize`) builds it from a `DagOfSelectedCandidates`, +//! each node's timing and lifetime from the `PhysicalCandidate` it realizes. +//! The deployment executes the plan as given +//! (#509 Assumption 8): it does not change lifetime or timing. use std::collections::{HashMap, HashSet}; use std::rc::Rc; @@ -13,24 +13,16 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::flat::{flatten, NodeId}; +use super::flat::NodeId; use super::scalar::ScalarExpr; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::Reduction; -use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::{ - ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; + +use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::guarantee::AccuracyGuarantee; -#[cfg(any(test, feature = "test-support"))] -use crate::ir::properties::timing::{ - apply_materialization_timings_to_roots, MaterializationAssignment, -}; -use crate::ir::properties::timing::{data_state, is_materialization_point}; -use crate::ir::schema::{FieldDataType, Schema}; -use crate::physical::materialization::Materialization; +use crate::ir::schema::{FieldDataType, Schema, SketchParams}; +use crate::physical::{InvalidPhysicalProperties, Lifetime}; /// A node's operator, with each child replaced by its node id. pub type PhysicalASAPOperatorPayload = Operator; @@ -71,22 +63,28 @@ pub struct PhysicalASAPDAGNode { pub payload: PhysicalASAPOperatorPayload, /// Phase is a placement choice for every operator, independent of payload kind. pub output_state: ExecutionDataState, - /// Whether and where the deployment stores this node's output: the - /// option the materialization assignment chose for the sub-DAG this node - /// produces. Any node may be stored (#509 §Materialization); a node with - /// no assignment is `NotMaterialized`. - pub materialization: Materialization, + /// How long and where the deployment materializes this node's output: the + /// lifetime the physical properties assignment chose for the sub-DAG this + /// node produces. Any node may be materialized (#509 §Materialization); a node + /// with no assignment has lifetime `None`. With `output_state.timing` it + /// is the node's [`PhysicalProperties`]. + pub lifetime: Lifetime, + /// θ of a summary build (`SummaryAgg`, `SummaryTumblingWindowChain`): its + /// candidate's `LogicalProperties::parameters`, which the executor builds + /// the sketch from. `None` on every other node and for an exact build. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option, pub output_schema: Schema, pub guarantee: Option, - /// For a materialized view (a node whose materialization keeps its - /// output across evaluations), how far back from the newest evaluation + /// For a materialized view (a node whose lifetime materializes its output + /// across evaluations), how far back from the newest evaluation /// time its output must stay available once refreshed; for a query-time /// `Scan`, how far back its source's rows must stay available (`None` /// when it reads them unbounded). `None` on every other node, and on /// every node of a DAG exported before installation sets it /// ([`PhysicalASAPDAG::set_retention`]). A - /// node field rather than part of [`Materialization`], because Stage 2 - /// enumerates materializations before Stage 3 knows retention. + /// node field rather than part of [`Lifetime`], because Stage 2 + /// enumerates lifetimes before Stage 3 knows retention. #[serde(default, skip_serializing_if = "Option::is_none")] pub retention_ms: Option, } @@ -171,14 +169,19 @@ pub struct PhysicalASAPDAG { #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum PhysicalASAPDAGValidationError { - #[error( - "node {node:?} runs at {timing:?}, which its materialization {materialization:?} does not" - )] - MaterializationTimingMismatch { + /// A node whose timing and lifetime cannot go together: a planner bug. + #[error("node {node:?}: {reason}")] + InvalidPhysicalProperties { node: PhysicalASAPNodeId, - timing: ExecutionTiming, - materialization: Materialization, + #[source] + reason: InvalidPhysicalProperties, }, + /// A summary build (or maintained population) that is not materialized but does + /// not run at query time: a planner bug. + #[error( + "node {node:?} is a summary build that is not materialized but does not run at query time" + )] + UnmaterializedSummaryBuildNotAtQueryTime { node: PhysicalASAPNodeId }, #[error("ingestion node {consumer:?} depends on query node {producer:?}")] QueryDependencyInIngestion { producer: PhysicalASAPNodeId, @@ -337,39 +340,9 @@ pub(crate) fn offset_below<'a>( Some(end_offset_ms) } -/// Why `PhysicalASAPDAG::with_every_node_at_query_time` failed. -#[derive(Debug, Error)] -pub enum ReferencePlanError { - /// Bad input: the DAG is not a valid physical ASAP DAG. - #[error("invalid physical ASAP DAG: {0}")] - InvalidDag(#[from] PhysicalASAPDAGValidationError), - /// Bad input: the plan cannot run at query time (a data-state rule of - /// the timing pass fails). - #[error("the plan does not run at query time: {0}")] - Timing(#[from] ExecutionDataStateError), -} - impl PhysicalASAPDAG { - /// This DAG's plan with nothing materialized: the same operators, - /// summaries and summary parameters, with every sub-DAG - /// `NotMaterialized` and so every node at query time (the default - /// [`MaterializationAssignment`]). Node ids are renumbered, and `roots` - /// and `scalar_roots` keep their order. - /// - /// The correctness reference of a selected plan: a deployment runtime - /// compares the answers of the plan it installed with the executor - /// running this one over raw data, which needs the raw data the - /// installed plan may not keep. - #[cfg(any(test, feature = "test-support"))] - pub fn with_every_node_at_query_time(&self) -> Result { - let roots = self.untimed_query_roots()?; - let timed = - apply_materialization_timings_to_roots(&roots, &MaterializationAssignment::default())?; - Ok(compile_physical_asap_workload(&timed)?) - } - - /// Every node of this DAG as IR, by node id, untimed and - /// `NotMaterialized` with its exported guarantee: what a caller derives + /// Every node of this DAG as IR, by node id, with its exported + /// guarantee (an operator node has no physical properties): what a caller derives /// IR properties of a plan file's node from, such as its /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). /// Shared inputs stay shared. Validates the DAG first. @@ -411,7 +384,7 @@ impl PhysicalASAPDAG { /// - a `TimeRange(w)`, range or instant: `(T - s - w, T - s]`; /// - a `SummaryTumblingWindowChain` of `n` tumbling windows of width `w`: its whole window `(T - s - /// n·w, T - s]`, or only its newest tumbling window `(T - s - w, T - s]` for a - /// chain in `kept_chains` (kept across evaluations, so each + /// chain in `materialized_chains` (materialized across evaluations, so each /// evaluation builds one tumbling window). /// /// `Unbounded` when a root, any other reader, or a shift with an `@` @@ -419,9 +392,9 @@ impl PhysicalASAPDAG { pub fn scan_window( &self, scan: PhysicalASAPNodeId, - kept_chains: &HashSet, + materialized_chains: &HashSet, ) -> ScanWindow { - self.scan_windows([scan], kept_chains)[&scan] + self.scan_windows([scan], materialized_chains)[&scan] } /// [`Self::scan_window`] of each of `scans`, by id, with the DAG @@ -429,7 +402,7 @@ impl PhysicalASAPDAG { pub fn scan_windows( &self, scans: impl IntoIterator, - kept_chains: &HashSet, + materialized_chains: &HashSet, ) -> HashMap { let payloads = self.payloads_by_id(); let mut consumers: HashMap> = HashMap::new(); @@ -461,7 +434,7 @@ impl PhysicalASAPDAG { Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, .. - })) if kept_chains.contains(&consumer) => { + })) if materialized_chains.contains(&consumer) => { u128::from(*tumbling_window_width_ms) } Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { @@ -567,54 +540,35 @@ impl PhysicalASAPDAG { .collect() } - /// The IR this DAG exports, one root per query in batch order, every - /// node untimed and `NotMaterialized`: the inverse of - /// [`compile_physical_asap_workload`] up to `Rc` identity and timing. - #[cfg(any(test, feature = "test-support"))] - fn untimed_query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { - let rebuilt = self.operator_nodes()?; - let mut operator_roots = self.roots.iter(); - let mut scalar_roots = self.scalar_roots.iter().peekable(); - let mut roots = Vec::with_capacity(self.roots.len() + self.scalar_roots.len()); - for position in 0..self.roots.len() + self.scalar_roots.len() { - match scalar_roots.peek() { - Some((at, expr)) if *at == position => { - roots.push(QueryRoot::Scalar( - expr.map_operator_refs(&mut |id| Rc::clone(&rebuilt[id])), - )); - scalar_roots.next(); - } - _ => { - let id = operator_roots.next().expect("validated root positions"); - roots.push(QueryRoot::Operator(Rc::clone(&rebuilt[id]))); - } - } - } - Ok(roots) - } - pub fn validate(&self) -> Result<(), PhysicalASAPDAGValidationError> { let mut nodes = HashMap::new(); for node in &self.nodes { if nodes.insert(node.id, node).is_some() { return Err(PhysicalASAPDAGValidationError::DuplicateNodeId(node.id)); } - // A sub-DAG output runs when its materialization says; any other - // node is not stored and runs when its consumer runs. - if (is_materialization_point(&node.payload) - || node.materialization != Materialization::NotMaterialized) - && node.materialization.timing() != node.output_state.timing + // A node's timing and lifetime must go together; a summary + // build that is not materialized runs at query time. Any other node that + // is not materialized runs when its consumer runs. + if node.output_state.timing == ExecutionTiming::IngestionTime + && matches!(node.lifetime, Lifetime::OneEvaluation(_)) + { + return Err(PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: node.id, + reason: InvalidPhysicalProperties::OneEvaluationAtIngestion, + }); + } + if is_materializable_state_output(&node.payload) + && node.output_state.timing == ExecutionTiming::IngestionTime + && node.lifetime == Lifetime::None { return Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { + PhysicalASAPDAGValidationError::UnmaterializedSummaryBuildNotAtQueryTime { node: node.id, - timing: node.output_state.timing, - materialization: node.materialization, }, ); } if node.retention_ms.is_some() - && !node.materialization.is_materialized_view() + && !matches!(node.lifetime, Lifetime::AcrossEvaluations(_)) && !super::materialized_view::is_query_time_scan(node) { return Err(PhysicalASAPDAGValidationError::UnexpectedRetention(node.id)); @@ -800,65 +754,66 @@ impl PhysicalASAPDAG { } } -// ── Compilation from the IR ────────────────────────────────────────────── - -/// Compiler-local identity assignment. It deliberately retains `Rc` handles -/// and is not serialized; deployed artifacts persist the physical ASAP node ID -/// together with their physical materialization/query IDs. -#[derive(Debug, Clone)] -pub struct PhysicalASAPNodeIdentityMap { - nodes_by_id: Vec>, -} - -impl PhysicalASAPNodeIdentityMap { - #[cfg(any(test, feature = "test-support"))] - pub fn node_id(&self, node: &Rc) -> Option { - self.nodes_by_id - .iter() - .position(|candidate| Rc::ptr_eq(candidate, node)) - } - - pub fn operator_node(&self, id: PhysicalASAPNodeId) -> Option<&Rc> { - self.nodes_by_id.get(id) +/// Each input of `operator` with its edge role: the operator inputs, then the +/// nodes read by its scalar expressions (the order of [`Operator::children`]). +/// A node referenced several times from the scalar expressions is one +/// `ScalarRef` input, as cost lowering reads it once; ids are per node +/// pointer, so this dedupes by pointer. +fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { + use EdgeRole::*; + let inputs: Vec = match operator { + Operator::NonASAP(op) => match op { + NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. } | NonASAPOp::BinaryOp { .. } => { + vec![Left, Right] + } + NonASAPOp::Concat { children, .. } => vec![Input; children.len()], + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::PromqlVectorFromScalar(_) => vec![], + _ => vec![Input], + }, + Operator::ASAP(op) => match op { + ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], + ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], + _ => vec![Input], + }, + }; + let children = operator.children(); + let (direct, scalar_refs) = children.split_at(inputs.len()); + let mut edges: Vec<(C, EdgeRole)> = direct.iter().map(|c| **c).zip(inputs).collect(); + for producer in scalar_refs { + if !edges.contains(&(**producer, ScalarRef)) { + edges.push((**producer, ScalarRef)); + } } + edges } -#[derive(Debug, Clone)] -pub struct PhysicalASAPDAGCompilation { - pub dag: PhysicalASAPDAG, - pub node_ids: PhysicalASAPNodeIdentityMap, -} - -#[cfg(any(test, feature = "test-support"))] -pub fn compile_physical_asap_dag( - root: &Rc, -) -> Result { - Ok(compile_physical_asap_dag_with_node_ids(root)?.dag) +/// Whether `operator`'s output is a sub-DAG output that materialization may +/// store: a `SummaryAgg`, a `SummaryTumblingWindowChain` or a `MaintainPopulation`. +pub(crate) fn is_materializable_state_output(operator: &Operator) -> bool { + matches!( + operator, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::MaintainPopulation { .. } + ) + ) } -/// Export the timed DAG below `root`. Every reachable node must carry a -/// timing (see [`crate::ir::properties::timing::apply_materialization_timings`]); the data-state -/// rules were checked by that pass and are not re-run here. +/// Test-only: `roots` exported as one DAG with every node at query time and +/// not materialized, and no θ (every `parameters` is `None`), with each +/// exported node's operator node by node id. Executor tests hand-build DAGs +/// this way; planning exports only through the realizer +/// (`AssignmentRealizer::realize`). #[cfg(any(test, feature = "test-support"))] -pub(crate) fn compile_physical_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - compile_physical_asap_workload_with_node_ids(&[QueryRoot::Operator(Rc::clone(root))]) -} - -/// Export a timed batch as one DAG: operator queries become `roots` and -/// standalone scalar queries `scalar_roots`, keyed by their position in the -/// batch. -pub fn compile_physical_asap_workload( - roots: &[QueryRoot], -) -> Result { - Ok(compile_physical_asap_workload_with_node_ids(roots)?.dag) -} - -pub fn compile_physical_asap_workload_with_node_ids( - roots: &[QueryRoot], -) -> Result { - let (flat, nodes_by_id) = flatten(roots); +pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result<(PhysicalASAPDAG, Vec>), PhysicalASAPDAGValidationError> { + use crate::ir::properties::execution_data_state::DataPrimitive; + use crate::ir::QueryRoot; + let (flat, nodes_by_id) = super::flat::flatten(roots); let mut scalar_roots = Vec::new(); let roots = flat .roots @@ -876,33 +831,36 @@ pub fn compile_physical_asap_workload_with_node_ids( let mut edges = Vec::new(); for (id, flat_node) in flat.nodes.into_iter().enumerate() { let node = &nodes_by_id[id]; - let output_state = data_state(node).ok_or(ExecutionDataStateError::UntimedNode { - operator: node.operator.kind_name(), - })?; + let primitive = match &node.operator { + Operator::ASAP(op) if op.produced_state().is_some() => DataPrimitive::SummaryState, + Operator::ASAP(ASAPOp::MaintainPopulation { .. }) => DataPrimitive::SummaryState, + _ if node.result_kind == crate::ir::OperatorResultKind::State => { + DataPrimitive::SummaryState + } + _ => DataPrimitive::Raw, + }; + let output_state = ExecutionDataState { + timing: ExecutionTiming::QueryTime, + primitive, + }; // Children come before their parents, so every producer is already in `nodes`. for (producer, role) in edge_roles(&flat_node.operator) { - let producer_state = nodes[producer].output_state; - let ingestion_dependency = producer_state.timing == ExecutionTiming::IngestionTime - && output_state.timing == ExecutionTiming::IngestionTime; edges.push(PhysicalASAPDAGEdge { producer, consumer: id, role, intermediate_schema: nodes[producer].output_schema.clone(), - data_state: producer_state, + data_state: nodes[producer].output_state, grouping: grouping_compatibility(&nodes[producer].payload, &flat_node.operator), - window: if ingestion_dependency { - WindowEdgeCompatibility::RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual - } else { - WindowEdgeCompatibility::NotApplicable - }, + window: WindowEdgeCompatibility::NotApplicable, }); } nodes.push(PhysicalASAPDAGNode { id, payload: flat_node.operator, output_state, - materialization: flat_node.materialization, + lifetime: Lifetime::None, + parameters: None, output_schema: flat_node.schema, guarantee: flat_node.guarantee, retention_ms: None, @@ -914,52 +872,16 @@ pub fn compile_physical_asap_workload_with_node_ids( roots, scalar_roots, }; - dag.validate().map_err(ExecutionDataStateError::from)?; - Ok(PhysicalASAPDAGCompilation { - dag, - node_ids: PhysicalASAPNodeIdentityMap { nodes_by_id }, - }) -} - -/// Each input of `operator` with its edge role: the operator inputs, then the -/// nodes read by its scalar expressions (the order of [`Operator::children`]). -/// A node referenced several times from the scalar expressions is one -/// `ScalarRef` input, as cost lowering reads it once; ids are per node -/// pointer, so this dedupes by pointer. -fn edge_roles(operator: &Operator) -> Vec<(C, EdgeRole)> { - use EdgeRole::*; - let inputs: Vec = match operator { - Operator::NonASAP(op) => match op { - NonASAPOp::Join { .. } | NonASAPOp::SetOp { .. } | NonASAPOp::BinaryOp { .. } => { - vec![Left, Right] - } - NonASAPOp::Concat { children, .. } => vec![Input; children.len()], - NonASAPOp::Scan { .. } - | NonASAPOp::Values { .. } - | NonASAPOp::PromqlVectorFromScalar(_) => vec![], - _ => vec![Input], - }, - Operator::ASAP(op) => match op { - ASAPOp::SummarySubtract { .. } | ASAPOp::SummaryJoin { .. } => vec![Left, Right], - ASAPOp::SummaryMerge { children, .. } => vec![Input; children.len()], - _ => vec![Input], - }, - }; - let children = operator.children(); - let (direct, scalar_refs) = children.split_at(inputs.len()); - let mut edges: Vec<(C, EdgeRole)> = direct.iter().map(|c| **c).zip(inputs).collect(); - for producer in scalar_refs { - if !edges.contains(&(**producer, ScalarRef)) { - edges.push((**producer, ScalarRef)); - } - } - edges + dag.validate()?; + Ok((dag, nodes_by_id)) } +#[cfg(any(test, feature = "test-support"))] fn grouping_compatibility( producer: &Operator, consumer: &Operator, ) -> GroupingEdgeCompatibility { + use crate::ir::operator::operator_properties::Reduction; let ( Operator::ASAP(ASAPOp::SummaryAgg { reduction: producer, diff --git a/crates/types/src/ir/properties/execution_data_state.rs b/crates/types/src/ir/properties/execution_data_state.rs index e3a2cda8e..c4c22f04c 100644 --- a/crates/types/src/ir/properties/execution_data_state.rs +++ b/crates/types/src/ir/properties/execution_data_state.rs @@ -2,14 +2,9 @@ //! //! [`ExecutionTiming`] says when a node's value is produced (ingestion vs. //! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] -//! the edge carries (raw values vs. summary state). [`Materialization`] is -//! the per-sub-DAG choice a node's timing is derived from. The rules that assign -//! and check them over a DAG live in [`crate::ir::properties::timing`], which reports -//! violations as [`ExecutionDataStateError`]. - -use thiserror::Error; - -use crate::physical::Materialization; +//! the edge carries (raw values vs. summary state). A node's timing comes +//! from the [`crate::physical::PhysicalProperties`] of the physical candidate +//! it realizes. /// When a post-ASAP value is produced. #[derive( @@ -78,121 +73,6 @@ impl std::fmt::Display for ExecutionDataState { } } -/// A plan-construction-time data_state violation. Typed (not a string) so a -/// strategy can degrade to a conservative fallback on the specific variant -/// it expects, and so tests can assert the *reason* a plan was rejected. -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum ExecutionDataStateError { - #[error("invalid maintained-population maintenance/evaluation contract")] - InvalidMaintainedPopulation, - /// A query-time value (a `SummaryEstimate` or query-time operator output) - /// placed beneath a summary built at ingestion time — the one shape issue - /// #171's data_state split exists to make unrepresentable. - #[error( - "query-time value under an ingestion-time summary: {edge} received a {child} input, but a \ - summary built at ingestion time can only consume ingestion-time rows (or exact \ - accumulator state)" - )] - QueryTimeValueUnderIngestionSummary { - edge: &'static str, - child: ExecutionDataState, - }, - /// A summary build at query time over an input (rows or exact-accumulator - /// state) produced at ingestion time and not stored there (the input node - /// is not `MatAtIngestion`): at query time that input no longer exists. - /// Bad input (a hand-built or candidate-fixed plan), not an internal bug. - #[error( - "a query-time {edge} reads {operator} output produced at ingestion time that is not \ - stored (not MatAtIngestion)" - )] - UnstoredIngestionInputUnderQueryTimeSummary { - edge: &'static str, - operator: &'static str, - }, - /// Any other edge whose child data_state the parent does not accept - /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a - /// sketch's opaque state fed into a query-time operator). - #[error("{edge} does not accept a {child} input")] - IllegalChildDataState { - edge: &'static str, - child: ExecutionDataState, - }, - /// A `SummaryAgg` whose child is summary state of a family other than an - /// exact accumulator — re-accumulating opaque sketch/sample/… state on - /// the update path has no defined semantics here. - #[error( - "SummaryAgg.child carries {family} summary state; only exact accumulator state can be \ - composed into another maintained summary" - )] - UnsupportedStateComposition { family: String }, - /// One shared node assigned two different execution timings by its - /// consumers; no single execution of it can serve both. - #[error("shared node is assigned conflicting timings: {first} and {second}")] - ConflictingTiming { - first: ExecutionDataState, - second: ExecutionDataState, - }, - /// A plan root that runs at ingestion time without being stored there - /// (a timing fixed by a candidate builder): no query reads its rows. A - /// root assigned `MatAtIngestion` is valid; the query reads the stored - /// output. - #[error( - "{operator} root runs at ingestion time but is not MatAtIngestion: no query reads its \ - output" - )] - UnstoredIngestionRoot { operator: &'static str }, - /// An explicit materialization on a node whose timing a candidate builder - /// fixed (e.g. a query-time `FinalizeExactAccumulator` read boundary) - /// that needs the other timing. - #[error( - "{operator} is fixed at {} by its candidate; materialization {materialization:?} needs \ - the other timing", - fixed.as_str() - )] - MaterializationConflictsWithFixedTiming { - operator: &'static str, - fixed: ExecutionTiming, - materialization: Materialization, - }, - /// A summary build with a filter placed at ingestion time. An - /// ingestion-time build reads raw data samples, which carry none of the - /// query's columns the filter reads, so only a query-time build can - /// apply it. - #[error("{operator} has a filter, which only a query-time build can apply")] - FilteredSummaryAtIngestion { operator: &'static str }, - /// An ingestion-time `BinaryOp` the update path cannot evaluate: not - /// arithmetic without vector matching, operands that are not the same - /// per-series rows, or a schema other than non-null timestamps, at most - /// one series identity and one Float64 value. - #[error("unsupported ingestion-time binary schema or operator")] - InvalidIngestionTimeBinary, - #[error("checked division requires one valid guard on a read-time division operator")] - InvalidCheckedDivision, - /// An exact operator whose input columns are not all `Plain` at its - /// declared data_state. - #[error("exact operator consumes non-plain column {column:?} ({dtype})")] - NonPlainOperand { column: String, dtype: String }, - /// An ASAP operator with no execution timing contract yet in an - /// executable plan: the reserved `SummarySubtract`, `SummaryDelete`, - /// `SummaryJoin` and `Extension`. - #[error("{operator} has no execution timing contract yet")] - UnimplementedOperator { operator: &'static str }, - /// A node assigned `MatAtIngestion` over a sub-DAG that can only run at - /// query time (#509 Algorithm 3, line 13). - #[error("{operator}: upstream of ingestion-time node not at ingestion time")] - UpstreamNotAtIngestion { operator: &'static str }, - /// A node reached by export without a timing: the materialization timing pass - /// was not applied to the DAG first. - #[error( - "{operator} node has no execution timing; apply materialization timings before export" - )] - UntimedNode { operator: &'static str }, - /// The exported physical DAG failed its own validation, e.g. an empty - /// batch or a hand-timed plan the timing pass would have rejected. - #[error("invalid physical ASAP DAG: {0}")] - InvalidPhysicalDAG(#[from] crate::ir::physical_export::PhysicalASAPDAGValidationError), -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/ir/properties/mod.rs b/crates/types/src/ir/properties/mod.rs index 455a489f2..c7730ba8f 100644 --- a/crates/types/src/ir/properties/mod.rs +++ b/crates/types/src/ir/properties/mod.rs @@ -7,21 +7,16 @@ //! ([`AccuracyGuarantee`]). //! - [`execution_data_state`] — when a node's value is produced //! ([`ExecutionTiming`]) and what its edge carries ([`ExecutionDataState`]). -//! - [`timing`] — execution timing written into every node from a -//! materialization assignment, and its validation. //! - [`summary_coverage`] — what a summary state covers. pub mod accuracy; pub mod execution_data_state; pub mod guarantee; pub mod summary_coverage; -pub mod timing; pub use accuracy::{ default_metric_for_root, AccuracyMetric, AccuracyScope, AccuracySpec, AccuracyTarget, NormScale, }; -pub use execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; +pub use execution_data_state::{DataPrimitive, ExecutionDataState, ExecutionTiming}; pub use guarantee::{ AccuracyError, AccuracyGuarantee, BoundExpr, CompositionOperator, GuaranteeSource, ProbabilityExpr, VALUES_PER_RESULT, diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 5cf1812ed..f371a9931 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -282,7 +282,7 @@ impl SummaryCoverage { /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, /// of its sub-DAG's operators, output kinds and schemas, children /// first, without planning metadata (`timing`, `guarantee`, - /// `materialization`). Two equal definitions (the equality a merge + /// `lifetime`). Two equal definitions (the equality a merge /// requires) get equal ids on any node and in any process, so a summary /// store can key states by this id and a concrete selection. pub(crate) fn definition_id(&self) -> String { diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs deleted file mode 100644 index 07684c9fc..000000000 --- a/crates/types/src/ir/properties/timing.rs +++ /dev/null @@ -1,1640 +0,0 @@ -//! Execution timing: written into every node from a materialization -//! assignment, then validated against each operator's kind and its consuming -//! edges. -//! -//! The logical DAG carries no timing. Materialization (#509 -//! §Materialization) gives each sub-DAG one of its options: -//! `NotMaterialized`, `MatAtIngestion(medium)`, `MatAtQuery(medium)` or -//! `MatAtQueryKeptAcrossEvaluations(medium)`. -//! [`MaterializationAssignment`] records that option per sub-DAG, named by -//! the node whose output would be stored (any node: a summary state, a -//! population, or e.g. the tumbling windows of #509 Example 4), and -//! [`apply_materialization_timings`] expands it into a timing on every node: -//! -//! - `SummaryEstimate` and `EvaluatePopulation` run at query time; -//! - a node assigned `MatAtIngestion` runs at ingestion time, and one -//! assigned `MatAtQuery` or `MatAtQueryKeptAcrossEvaluations` at query -//! time. A `SummaryAgg` or -//! `MaintainPopulation` without an assignment is `NotMaterialized` and runs -//! at query time. Every node upstream of an ingestion-time node also runs -//! at ingestion time, so `MatAtIngestion` (or an ingestion timing fixed by -//! a candidate builder) over something that can only exist at query time -//! is an error ([`ExecutionDataStateError::UpstreamNotAtIngestion`]); -//! - a PromQL `BinaryOp` over per-series inputs that are not the same series -//! rows runs at query time, since maintenance cannot align them; -//! - every other node is `NotMaterialized` and runs when its consumer runs: -//! everything that feeds an ingestion-time node runs at ingestion time, -//! everything above an evaluation at query time. -//! -//! A node whose timing a candidate builder fixed (e.g. the query-time -//! `FinalizeExactAccumulator` of `finalize_query_candidate`) keeps that -//! timing. An explicit assignment on it that agrees is kept; one that needs -//! the other timing is an error -//! ([`ExecutionDataStateError::MaterializationConflictsWithFixedTiming`]). -//! A root may be `MatAtIngestion`: the query reads its stored output. A root -//! of rows that runs at ingestion time without being stored there is an -//! error ([`ExecutionDataStateError::UnstoredIngestionRoot`]). -//! -//! A node reached from two consumers that need different timings cannot be -//! executed once for both; Stage 2's `split_shared_by_timing` copies such a -//! sub-DAG for one side (by [`timing_under_consumer`]) before the assignment -//! is applied, and the pass itself -//! rejects a conflict it still finds. -//! -//! ## Edge rules (checked after the write) -//! -//! | Consumer | Accepts from an input | -//! |---|---| -//! | `SummaryAgg.child`, `SummaryTumblingWindowChain.child` | Rows, or exact-accumulator state, never a query-time value when the state is maintained; a query-time build reads an ingestion-time input (rows or exact-accumulator state) only as a `MatAtIngestion` input's stored output | -//! | `SummaryEstimate.summary_input` | Summary state at either timing | -//! | `SummaryMerge.children` | Summary state at either timing; at ingestion time only for a merge at ingestion time | -//! | `FinalizeExactAccumulator.child` | Exact-accumulator state | -//! | `EvaluatePopulation.child` | A timed `MaintainPopulation` | -//! | `MaintainPopulation.child` | Rows at its own timing matching the population's input; unlike a `NonASAP` consumer, a query-time population does not read a `MatAtIngestion` input's stored output | -//! | any `NonASAP` consumer | Rows (or exact-accumulator state for a projection-like operator) at the consumer's own timing, or a `MatAtIngestion` output read at query time; ingestion work never reads a query-time value | - -use std::collections::HashMap; -use std::rc::Rc; - -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; -use crate::ir::operator::non_asap::NonASAPOp; -use crate::ir::operator::operator_properties::BinaryOpKind; -use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::{ - DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, -}; -use crate::ir::schema::{DataType, FieldDataType, Schema}; -use crate::physical::materialization::Materialization; -#[cfg(any(test, feature = "test-support"))] -use crate::physical::materialization::Medium; - -/// The materialization option of each sub-DAG (#509 §Materialization: -/// "Materialization decides, for each sub-DAG, whether its output is -/// stored"). A sub-DAG is named by the node whose output would be stored (by -/// identity); any node may be assigned. A node without an assignment is not -/// stored: a `SummaryAgg` or `MaintainPopulation` takes the default, every -/// other node runs when its consumer runs. `Default` assigns -/// `NotMaterialized` to every sub-DAG. -/// -/// The assignment keeps every assigned node alive, so a later allocation -/// cannot reuse its address and match a stale entry. -#[derive(Clone, Default)] -pub struct MaterializationAssignment { - assigned: HashMap<*const OperatorNode, (Rc, Materialization)>, - unassigned: Materialization, -} - -/// Each assigned node as its operator kind and address, not its whole -/// sub-DAG. -impl std::fmt::Debug for MaterializationAssignment { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let assigned: Vec<_> = self - .assigned - .iter() - .map(|(ptr, (node, materialization))| { - ( - format!("{}@{ptr:p}", node.operator.kind_name()), - materialization, - ) - }) - .collect(); - f.debug_struct("MaterializationAssignment") - .field("assigned", &assigned) - .field("unassigned", &self.unassigned) - .finish() - } -} - -impl MaterializationAssignment { - /// Every `SummaryAgg` or `MaintainPopulation` not [`set`](Self::set) - /// explicitly takes `materialization` where the DAG allows it: a - /// `MatAtIngestion` default over a sub-DAG that can only run at query - /// time leaves that sub-DAG `NotMaterialized`. - pub fn with_default(materialization: Materialization) -> Self { - Self { - assigned: HashMap::new(), - unassigned: materialization, - } - } - - /// Assign `materialization` to the sub-DAG whose output `node` produces. - pub fn set(&mut self, node: &Rc, materialization: Materialization) { - self.assigned - .insert(Rc::as_ptr(node), (Rc::clone(node), materialization)); - } - - /// The explicit assignment of `node`, if any. - pub fn get(&self, node: &OperatorNode) -> Option { - self.assigned - .get(&(node as *const OperatorNode)) - .map(|(_, m)| *m) - } - - /// The option `node` gets when `upstream_at_query_time` says whether - /// something below it can only run at query time. - fn resolve( - &self, - node: &OperatorNode, - upstream_at_query_time: bool, - ) -> Result { - let at_ingestion = |m: Materialization| m.timing() == ExecutionTiming::IngestionTime; - match self.get(node) { - Some(m) if at_ingestion(m) && is_filtered_build(node) => { - Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) if at_ingestion(m) && upstream_at_query_time => { - Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) => Ok(m), - None if at_ingestion(self.unassigned) - && (upstream_at_query_time || is_filtered_build(node)) => - { - Ok(Materialization::NotMaterialized) - } - None => Ok(self.unassigned), - } - } -} - -/// Whether `operator`'s output is a sub-DAG output that materialization may -/// store: a `SummaryAgg`, a `SummaryTumblingWindowChain` or a `MaintainPopulation`. -pub(crate) fn is_materialization_point(operator: &Operator) -> bool { - matches!( - operator, - Operator::ASAP( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::MaintainPopulation { .. } - ) - ) -} - -/// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`) with a -/// filter, which never runs at ingestion time -/// ([`ExecutionDataStateError::FilteredSummaryAtIngestion`]). -pub fn is_filtered_build(node: &OperatorNode) -> bool { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - filter: Some(_), - .. - } | ASAPOp::SummaryTumblingWindowChain { - filter: Some(_), - .. - } - ) - ) -} - -/// Cache of one [`apply_materialization_timings`] pass: `input node → timed node`, -/// shared by every root of a workload so a node shared by two roots stays -/// one `Rc`. Re-reaching a node with a different timing is a conflict. -#[derive(Default)] -pub struct NodeToTimedNodeCache { - timed_node_of: HashMap<*const OperatorNode, Rc>, -} - -impl NodeToTimedNodeCache { - pub fn new() -> Self { - Self::default() - } - - /// The timed node produced for `input`, if the pass has reached it. - pub(crate) fn timed(&self, input: &Rc) -> Option<&Rc> { - self.timed_node_of.get(&Rc::as_ptr(input)) - } -} - -/// What `node`'s output carries: summary state from a build, a merge or a -/// node that keeps whole states (a key filter, a projection of the state -/// column), raw rows otherwise. -fn primitive(node: &OperatorNode) -> DataPrimitive { - match &node.operator { - Operator::ASAP(op) if op.produced_state().is_some() => DataPrimitive::SummaryState, - Operator::ASAP(ASAPOp::MaintainPopulation { .. }) => DataPrimitive::SummaryState, - _ if node.result_kind == crate::ir::OperatorResultKind::State => { - DataPrimitive::SummaryState - } - _ => DataPrimitive::Raw, - } -} - -/// The data state a timed node's output carries. -pub fn data_state(node: &OperatorNode) -> Option { - Some(ExecutionDataState { - timing: node.timing?, - primitive: primitive(node), - }) -} - -/// Whether the sub-DAG below `node` contains a node that can only run at -/// query time (an evaluation, or a sub-DAG `assignment` does not put at -/// ingestion time), which forces every consumer above it to query time as well. -/// A PromQL `BinaryOp` whose inputs are not the same per-series rows also runs -/// at query time. -fn forces_query_time( - node: &OperatorNode, - assignment: &MaterializationAssignment, - seen: &mut HashMap<*const OperatorNode, bool>, -) -> bool { - let key = node as *const OperatorNode; - if let Some(&cached) = seen.get(&key) { - return cached; - } - let upstream = |seen: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, seen)) - }; - let forced = match &node.operator { - _ if node.timing == Some(ExecutionTiming::QueryTime) => true, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => true, - _ if node.timing.is_none() - && (is_materialization_point(&node.operator) || assignment.get(node).is_some()) => - { - let upstream = upstream(seen); - match assignment.resolve(node, upstream) { - Ok(Materialization::NotMaterialized) - if !is_materialization_point(&node.operator) => - { - upstream - } - Ok(m) => m.timing() == ExecutionTiming::QueryTime, - Err(_) => true, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) - if node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) => - { - true - } - _ => upstream(seen), - }; - seen.insert(key, forced); - forced -} - -/// Write the timings of `assignment` into every node reachable from `root`, -/// top-down, then validate every edge. Returns the timed copy of `root`; -/// `cache` carries the sharing across the roots of one workload. -pub fn apply_materialization_timings( - root: &Rc, - assignment: &MaterializationAssignment, - cache: &mut NodeToTimedNodeCache, -) -> Result, ExecutionDataStateError> { - let mut forced = HashMap::new(); - let timed = write( - root, - ExecutionTiming::QueryTime, - assignment, - cache, - &mut forced, - )?; - // A root stored at ingestion time is read by the query; one that runs at - // ingestion time without being stored there produces rows nothing reads. - if timed.timing == Some(ExecutionTiming::IngestionTime) - && !matches!(timed.materialization, Materialization::MatAtIngestion(_)) - && data_state(&timed).map(|s| s.primitive) == Some(DataPrimitive::Raw) - { - return Err(ExecutionDataStateError::UnstoredIngestionRoot { - operator: timed.operator.kind_name(), - }); - } - validate(&timed, &mut HashMap::new())?; - Ok(timed) -} - -/// [`apply_materialization_timings`] over every root of one workload with -/// one cache, so a node several roots share is timed once and stays one -/// node. A standalone scalar root has each operator it reads timed, and is -/// rebuilt over the timed nodes. -pub fn apply_materialization_timings_to_roots( - roots: &[QueryRoot], - assignment: &MaterializationAssignment, -) -> Result, ExecutionDataStateError> { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => QueryRoot::Operator(apply_materialization_timings( - node, assignment, &mut cache, - )?), - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - apply_materialization_timings(node, assignment, &mut cache)?; - } - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - Rc::clone(cache.timed(node).expect("scalar reference was timed")) - })) - } - }) - }) - .collect() -} - -/// Validate the sub-DAG below `root` with every sub-DAG `MatAtIngestion` -/// where the DAG allows it ([`MaterializationAssignment::with_default`]) and -/// `root` consumed at `root_timing`. For planning-time legality checks of a -/// candidate before it is assembled into a workload DAG: a candidate must stay -/// executable if materialization later puts its states at ingestion time. -/// Nothing is kept. -#[cfg(any(test, feature = "test-support"))] -pub fn validate_at_ingestion( - root: &Rc, - root_timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - let assignment = all_at_ingestion(); - let mut cache = NodeToTimedNodeCache::new(); - let mut forced = HashMap::new(); - let timed = write(root, root_timing, &assignment, &mut cache, &mut forced)?; - validate(&timed, &mut HashMap::new()) -} - -/// The data state `node` produces with every sub-DAG `MatAtIngestion` where -/// the DAG allows it, when its consumer runs at `consumer` — the planning-time answer to -/// "what does this candidate's output look like", consistent with -/// [`validate_at_ingestion`]. -#[cfg(any(test, feature = "test-support"))] -pub fn planned_data_state( - node: &Rc, - consumer: ExecutionTiming, -) -> ExecutionDataState { - let mut forced = HashMap::new(); - let (timing, _) = placement(node, consumer, &all_at_ingestion(), &mut forced) - .expect("a default assignment never fails"); - ExecutionDataState { - timing, - primitive: primitive(node), - } -} - -/// The assignment of [`validate_at_ingestion`] and [`planned_data_state`]. -#[cfg(any(test, feature = "test-support"))] -fn all_at_ingestion() -> MaterializationAssignment { - MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)) -} - -/// The timing and materialization `node` takes when its consumer runs at -/// `consumer`. -fn placement( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result<(ExecutionTiming, Materialization), ExecutionDataStateError> { - // A placement fixed when the candidate was built (an exact-state read - // boundary that must run at query time, or one that feeds maintenance) - // is honored; a conflicting consumer is rejected by validation. - let upstream_at_query_time = |forced: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, forced)) - }; - if let Some(placed) = node.timing { - if placed == ExecutionTiming::IngestionTime && is_filtered_build(node) { - return Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }); - } - if placed == ExecutionTiming::IngestionTime && upstream_at_query_time(forced) { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }); - } - // An explicit assignment is never silently dropped: one that agrees - // with the fixed timing is kept, one that needs the other timing is - // an error. - return match assignment.get(node) { - Some(m) if m != Materialization::NotMaterialized && m.timing() != placed => Err( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: node.operator.kind_name(), - fixed: placed, - materialization: m, - }, - ), - Some(m) => Ok((placed, m)), - None => Ok((placed, node.materialization)), - }; - } - Ok(match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => match assignment.get(node) { - Some(m) if m.timing() == ExecutionTiming::IngestionTime => { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(m) => (ExecutionTiming::QueryTime, m), - None => (ExecutionTiming::QueryTime, Materialization::NotMaterialized), - }, - _ if is_materialization_point(&node.operator) || assignment.get(node).is_some() => { - let upstream = upstream_at_query_time(forced); - match assignment.resolve(node, upstream)? { - Materialization::NotMaterialized if !is_materialization_point(&node.operator) => { - (consumer, Materialization::NotMaterialized) - } - materialization => (materialization.timing(), materialization), - } - } - _ => (consumer, Materialization::NotMaterialized), - }) -} - -/// The timing `node` takes when its consumer runs at `consumer`, by the -/// placement rules [`apply_materialization_timings`] applies, for Stage 2's -/// `split_shared_by_timing`: an assignment that cannot be honored is -/// reported later by [`apply_materialization_timings`]. -/// `query_time_forced` caches, per node, whether something at or below it -/// can only run at query time: pass an empty map and reuse it across the -/// calls of one assignment. -pub fn timing_under_consumer( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - query_time_forced: &mut HashMap<*const OperatorNode, bool>, -) -> ExecutionTiming { - placement(node, consumer, assignment, query_time_forced) - .map_or(ExecutionTiming::QueryTime, |(timing, _)| timing) -} - -fn write( - node: &Rc, - consumer: ExecutionTiming, - assignment: &MaterializationAssignment, - cache: &mut NodeToTimedNodeCache, - forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result, ExecutionDataStateError> { - let (timing, materialization) = placement(node, consumer, assignment, forced)?; - if let Some(done) = cache.timed_node_of.get(&Rc::as_ptr(node)) { - let previous = done.timing.expect("memoized node is timed"); - if previous != timing { - return Err(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState { - timing: previous, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), - }, - second: ExecutionDataState { - timing, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), - }, - }); - } - return Ok(Rc::clone(done)); - } - let mut error = None; - let operator = - node.operator.map_children( - |child| match write(child, timing, assignment, cache, forced) { - Ok(timed) => timed, - Err(e) => { - error.get_or_insert(e); - Rc::clone(child) - } - }, - ); - if let Some(e) = error { - return Err(e); - } - let timed = Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(Some(timing)) - .with_materialization(materialization), - ); - cache - .timed_node_of - .insert(Rc::as_ptr(node), Rc::clone(&timed)); - Ok(timed) -} - -fn state_of(node: &OperatorNode) -> ExecutionDataState { - data_state(node).expect("timed node") -} - -/// Check every edge below `node` against the module-level rules. -fn validate( - node: &Rc, - seen: &mut HashMap<*const OperatorNode, ()>, -) -> Result<(), ExecutionDataStateError> { - if seen.insert(Rc::as_ptr(node), ()).is_some() { - return Ok(()); - } - let timing = node.timing.expect("timed node"); - match &node.operator { - Operator::ASAP(op) => validate_asap(node, op, timing)?, - Operator::NonASAP(op) => validate_non_asap(node, op, timing)?, - } - for child in node.children() { - validate(child, seen)?; - } - Ok(()) -} - -fn is_exact_accumulator_state(schema: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &schema.fields { - match &field.dtype { - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) => {} - other => { - return Err(ExecutionDataStateError::UnsupportedStateComposition { - family: format!("{other:?}"), - }) - } - } - } - Ok(()) -} - -fn validate_asap( - node: &OperatorNode, - op: &ASAPOp, - timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - match op { - ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. } => { - let edge = match op { - ASAPOp::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain.child", - _ => "SummaryAgg.child", - }; - let avail = state_of(child); - match avail { - ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => {} - s if s.primitive == DataPrimitive::SummaryState => { - is_exact_accumulator_state(&child.schema)? - } - other => { - return Err( - ExecutionDataStateError::QueryTimeValueUnderIngestionSummary { - edge, - child: other, - }, - ) - } - } - // A query-time build reads an ingestion-time input (rows or - // exact-accumulator state) only from where it is stored, as a - // `NonASAP` consumer does. - if timing == ExecutionTiming::QueryTime - && avail.timing == ExecutionTiming::IngestionTime - && child.materialization.timing() != ExecutionTiming::IngestionTime - { - return Err( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge, - operator: child.operator.kind_name(), - }, - ); - } - if timing == ExecutionTiming::IngestionTime - && avail.timing == ExecutionTiming::QueryTime - { - return Err( - ExecutionDataStateError::QueryTimeValueUnderIngestionSummary { - edge, - child: avail, - }, - ); - } - Ok(()) - } - ASAPOp::SummaryEstimate { summary_input, .. } => { - let s = state_of(summary_input); - if s.primitive != DataPrimitive::SummaryState { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryEstimate.summary_input", - child: s, - }); - } - if timing != ExecutionTiming::QueryTime { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryEstimate", - child: state_of(node), - }); - } - Ok(()) - } - ASAPOp::FinalizeExactAccumulator { child } => { - let s = state_of(child); - if s.primitive != DataPrimitive::SummaryState - || is_exact_accumulator_state(&child.schema).is_err() - || (timing == ExecutionTiming::IngestionTime && s.timing != timing) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "FinalizeExactAccumulator.child", - child: s, - }); - } - Ok(()) - } - ASAPOp::MaintainPopulation { child, population } => { - let valid = population.matches_node(child) - && state_of(child) - == ExecutionDataState { - timing, - primitive: DataPrimitive::Raw, - }; - if !valid { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - Ok(()) - } - ASAPOp::EvaluatePopulation { child, evaluation } => { - let valid = timing == ExecutionTiming::QueryTime - && matches!( - &child.operator, - Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) - if population.supports(evaluation) - && child.timing.is_some() - ); - if !valid { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - Ok(()) - } - ASAPOp::SummaryMerge { children, .. } => { - for child in children { - let s = state_of(child); - if s.primitive != DataPrimitive::SummaryState - || (timing == ExecutionTiming::IngestionTime && s.timing != timing) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "SummaryMerge.children", - child: s, - }); - } - } - Ok(()) - } - ASAPOp::SummarySubtract { .. } - | ASAPOp::SummaryDelete { .. } - | ASAPOp::SummaryJoin { .. } - | ASAPOp::Extension { .. } => Err(ExecutionDataStateError::UnimplementedOperator { - operator: op.kind_name(), - }), - } -} - -fn check_plain_or_exact_values(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!( - field.dtype, - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) - ) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_all_plain(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !field.is_plain() { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -/// Whether `node` is a filter that keeps whole summary states by their key -/// columns ([`selected_from`](crate::ir::operator::asap::selected_from)). -fn selects_states(node: &OperatorNode) -> bool { - crate::ir::operator::asap::selected_from(node).is_some() -} - -fn validate_non_asap( - node: &OperatorNode, - op: &NonASAPOp, - timing: ExecutionTiming, -) -> Result<(), ExecutionDataStateError> { - // Every input is rows at this node's own timing. Ingestion work never - // reads a query-time value; exact-accumulator state may pass through - // the projection-like operators unchanged. - for child in op.children() { - let s = state_of(child); - let passes_state = matches!( - op, - NonASAPOp::Project { .. } - | NonASAPOp::Filter { .. } - | NonASAPOp::Sort { .. } - | NonASAPOp::Limit { .. } - ) && s.primitive == DataPrimitive::SummaryState - && (is_exact_accumulator_state(&child.schema).is_ok() || selects_states(node)); - // A query-time consumer may read the stored output of a sub-DAG - // materialized at ingestion time. - let reads_stored = timing == ExecutionTiming::QueryTime - && child.materialization.timing() == ExecutionTiming::IngestionTime; - if (s.timing != timing && !reads_stored) - || (s.primitive != DataPrimitive::Raw && !passes_state) - { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: op.kind_name(), - child: s, - }); - } - } - match op { - // A key filter keeps whole summary states of any family. - NonASAPOp::Filter { .. } if selects_states(node) => {} - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } => check_plain_or_exact_values(&child.schema)?, - NonASAPOp::Aggregate { - reduction, - measures, - child, - .. - } => { - let mut referenced: Vec = reduction - .group_keys() - .map(|keys| keys.keys().to_vec()) - .unwrap_or_default(); - for m in measures { - referenced.extend(m.input_cols()); - } - let implicit = measures.iter().any(|m| m.input_cols().is_empty()); - for (i, field) in child.schema.fields.iter().enumerate() { - if (implicit || referenced.contains(&i)) && !field.is_plain() { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - } - NonASAPOp::BinaryOp { - operator, lhs, rhs, .. - } => { - let is_div = matches!( - operator.kind, - BinaryOpKind::Arithmetic(crate::ir::scalar::ArithmeticOpKind::Div) - ); - if (operator.checked_relative_division && operator.checked_finite_division) - || ((operator.checked_relative_division || operator.checked_finite_division) - && (timing != ExecutionTiming::QueryTime || !is_div)) - { - return Err(ExecutionDataStateError::InvalidCheckedDivision); - } - if timing == ExecutionTiming::IngestionTime { - // The opaque series identity is a non-null Utf8 key; every - // other column is a non-null Float64 value or timestamp. - let plain_float_or_ts = |schema: &Schema| { - schema.fields.iter().all(|field| { - !field.nullable - && if field.name == crate::ir::schema::PROMQL_SERIES_IDENTITY { - field.dtype == FieldDataType::Plain(DataType::Utf8) - } else { - matches!( - field.dtype, - FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) - ) - } - }) - }; - let float_count = node - .schema - .fields - .iter() - .filter(|f| matches!(f.dtype, FieldDataType::Plain(DataType::Float64))) - .count(); - if operator.vector_match.is_some() - || !matches!(operator.kind, BinaryOpKind::Arithmetic(_)) - || lhs.schema != rhs.schema - || lhs.schema != node.schema - || node - .schema - .fields - .iter() - .filter(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - .count() - > 1 - || (node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs) - .is_none_or(|rows| per_series_rows(rhs) != Some(rows))) - || !plain_float_or_ts(&node.schema) - || float_count != 1 - { - return Err(ExecutionDataStateError::InvalidIngestionTimeBinary); - } - } - } - _ => { - for child in op.children() { - check_all_plain(&child.schema)?; - } - } - } - Ok(()) -} - -/// Ingestion-time arithmetic needs the same per-series population on both sides. -fn per_series_rows(node: &OperatorNode) -> Option<&OperatorNode> { - use crate::ir::schema::ExactKind; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match &child.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), - reduction: crate::ir::operator::Reduction::PerEntity, - filter: None, - .. - }) => Some(child), - _ => None, - }, - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) => { - let rows = per_series_rows(lhs)?; - (per_series_rows(rhs) == Some(rows)).then_some(rows) - } - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ir::operator::agg_intent::AggIntent; - use crate::ir::operator::non_asap::NonASAPOp; - use crate::ir::operator::operator_properties::{Reduction, Source}; - use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use crate::ir::schema::Field; - - fn scan_with(fields: Vec) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index(fields, 0, vec![]), - })) - .unwrap() - } - - fn scan() -> Rc { - scan_with(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ]) - } - - fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) - } - - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - fn estimate(child: Rc) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }), - Schema::lifted( - vec![Field::plain("quantile_0_99", DataType::Float64, false)], - None, - ), - ) - .with_guarantee(None), - ) - } - - fn aggregate(measure: AggIntent, child: Rc) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![measure], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn max(child: Rc) -> Rc { - aggregate(AggIntent::Max { col: None }, child) - } - - /// `node` with its timing fixed in advance, as a candidate builder does - /// for an operator that must feed maintenance. - fn placed_at_ingestion(node: Rc) -> Rc { - Rc::new( - (*node) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ) - } - - /// Apply with every sub-DAG `MatAtIngestion` where the DAG allows it, the - /// placement whose edge rules these tests exercise. - fn apply(root: &Rc) -> Result, ExecutionDataStateError> { - apply_materialization_timings(root, &all_at_ingestion(), &mut NodeToTimedNodeCache::new()) - } - - const AT_INGESTION: Materialization = Materialization::MatAtIngestion(Medium::Memory); - - fn child(node: &Rc) -> Rc { - Rc::clone(node.children()[0]) - } - - /// Without a materialization decision, a summary is `NotMaterialized` and - /// it and its input run at query time; an explicit choice overrides the - /// default and is recorded on the timed node. - #[test] - fn default_assignment_materializes_nothing() { - let summary = agg(scan(), kll()); - let root = apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!( - data_state(&root), - Some(ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive: DataPrimitive::SummaryState, - }) - ); - assert_eq!(root.materialization, Materialization::NotMaterialized); - assert_eq!( - data_state(&child(&root)), - Some(ExecutionDataState::QUERY_ROWS) - ); - for (materialization, timing) in [ - (AT_INGESTION, ExecutionTiming::IngestionTime), - ( - Materialization::MatAtQuery(Medium::Disk), - ExecutionTiming::QueryTime, - ), - ] { - let mut assignment = MaterializationAssignment::default(); - assignment.set(&summary, materialization); - let root = apply_materialization_timings( - &summary, - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(root.timing, Some(timing)); - assert_eq!(root.materialization, materialization); - assert_eq!(child(&root).timing, Some(timing)); - assert_eq!( - child(&root).materialization, - Materialization::NotMaterialized - ); - } - } - - /// #509 Algorithm 3, line 13: an explicit `MatAtIngestion` over a state - /// that only exists at query time is an error, not a silent move to - /// query time. - #[test] - fn ingestion_time_node_over_query_time_upstream_is_rejected() { - let inner = agg(scan(), exact_sum()); - let outer = agg(Rc::clone(&inner), kll()); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&outer, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "SummaryAgg" - }) - ); - assignment.set(&inner, AT_INGESTION); - let outer = agg(Rc::clone(&inner), kll()); - assignment.set(&outer, AT_INGESTION); - let root = apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(child(&root).materialization, AT_INGESTION); - assert_eq!(child(&child(&root)).materialization, AT_INGESTION); - } - - #[test] - fn summary_agg_input_runs_at_ingestion_time() { - let root = apply(&agg(scan(), kll())).unwrap(); - assert_eq!( - data_state(&root), - Some(ExecutionDataState::INGESTION_SUMMARY) - ); - assert_eq!( - data_state(&child(&root)), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn exact_accumulator_state_may_feed_another_summary_agg() { - let inner = agg(scan(), exact_sum()); - assert!(apply(&estimate(agg(inner, kll()))).is_ok()); - } - - /// A state not at ingestion time leaves the summary that reads it, when - /// that summary has no explicit assignment, `NotMaterialized` at query - /// time, as an evaluation below it would. - #[test] - fn query_time_state_moves_its_consuming_summary_to_query_time() { - let inner = agg(scan(), exact_sum()); - let outer = agg(Rc::clone(&inner), kll()); - let mut assignment = all_at_ingestion(); - assignment.set(&inner, Materialization::NotMaterialized); - let root = apply_materialization_timings( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let outer = child(&root); - assert_eq!(outer.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(outer.materialization, Materialization::NotMaterialized); - assert_eq!(child(&outer).timing, Some(ExecutionTiming::QueryTime)); - } - - #[test] - fn evaluation_can_feed_summary_construction_at_query_time() { - let inner = estimate(agg(scan(), kll())); - let root = apply(&estimate(agg(inner, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); - } - - /// Any non-ASAP operator over an evaluation runs at query time. - #[test] - fn query_time_operation_over_evaluation_is_legal_and_root_is_evaluation() { - let evaluation = || estimate(agg(scan(), kll())); - let sorted = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Sort { - keys: vec![], - partition_by: Default::default(), - child: evaluation(), - })) - .unwrap(); - for root in [max(evaluation()), sorted] { - let root = apply(&root).unwrap(); - assert_eq!(data_state(&root), Some(ExecutionDataState::QUERY_ROWS)); - } - } - - #[test] - fn query_time_values_can_feed_query_time_summary_construction() { - let post = max(estimate(agg(scan(), kll()))); - let root = apply(&estimate(agg(post, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); - } - - #[test] - fn function_under_summary_agg_is_legal_but_not_at_root() { - let operation = placed_at_ingestion(max(scan())); - assert_eq!( - apply(&operation).err(), - Some(ExecutionDataStateError::UnstoredIngestionRoot { - operator: "Aggregate" - }) - ); - let root = apply(&estimate(agg(operation, kll()))).unwrap(); - let timed_operation = child(&child(&root)); - assert_eq!( - data_state(&timed_operation), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn function_over_evaluation_is_rejected() { - let operation = placed_at_ingestion(max(estimate(agg(scan(), kll())))); - assert_eq!( - apply(&estimate(agg(operation, kll()))).err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "Aggregate" - }) - ); - } - - /// One shared sub-DAG reached as ingestion-time input and as query-time - /// input cannot be executed once for both (Stage 2's - /// `split_shared_by_timing` copies it first). - #[test] - fn a_shared_subtree_reached_at_two_timings_conflicts() { - let shared = scan(); - let root = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - max(estimate(agg(Rc::clone(&shared), kll()))), - max(Rc::clone(&shared)), - ], - })) - .unwrap(); - assert_eq!( - apply(&root).err(), - Some(ExecutionDataStateError::ConflictingTiming { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - } - - /// A finalize pinned to ingestion time (as fixed-window rate candidates - /// build it) over a state that is not `MatAtIngestion` names the missing - /// upstream, not an illegal edge. - #[test] - fn ingestion_pinned_finalize_over_query_time_state_is_upstream_not_at_ingestion() { - let finalize = placed_at_ingestion( - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { - child: agg(scan(), exact_sum()), - }, - )) - .unwrap(), - ); - assert_eq!( - apply_materialization_timings( - &estimate(agg(finalize, kll())), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "FinalizeExactAccumulator" - }) - ); - } - - /// Any node may be materialized, not only a summary state: stored rows - /// at ingestion time are read by a query-time consumer, and a node that - /// can only run at query time cannot be stored at ingestion time. - #[test] - fn any_node_may_be_materialized() { - let tumbling_windows = max(scan()); - let root = max(Rc::clone(&tumbling_windows)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&tumbling_windows, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed)).timing, - Some(ExecutionTiming::IngestionTime) - ); - - let over_evaluation = max(estimate(agg(scan(), kll()))); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&over_evaluation, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &over_evaluation, - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: "Aggregate" - }) - ); - } - - /// A summary build with a filter never runs at ingestion time: an - /// ingestion-time build reads raw data samples, which carry none of the - /// query's columns the filter reads. An explicit `MatAtIngestion` is an - /// error; the all-at-ingestion default leaves it at query time. - #[test] - fn a_filtered_summary_build_never_runs_at_ingestion_time() { - use crate::ir::scalar::expr_ir::ScalarValue; - use crate::ir::{Predicate, ScalarExpr}; - let plain = agg(scan(), kll()); - let Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family, - input, - reduction, - grouping, - .. - }) = plain.operator.clone() - else { - unreachable!() - }; - let filtered = Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family, - input, - reduction, - grouping, - filter: Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - }), - plain.schema.clone(), - ) - .with_guarantee(None), - ); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&filtered, AT_INGESTION); - assert_eq!( - apply_materialization_timings( - &estimate(Rc::clone(&filtered)), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: "SummaryAgg" - }) - ); - let timed = apply(&estimate(filtered)).unwrap(); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::QueryTime)); - assert_eq!( - child(&timed).materialization, - Materialization::NotMaterialized - ); - assert!(validate_at_ingestion(&child(&timed), ExecutionTiming::QueryTime).is_ok()); - } - - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child }, - )) - .unwrap() - } - - /// The materialization the exported DAG records for `node`'s operator kind. - fn exported(root: &Rc, kind: &str) -> Materialization { - let dag = crate::ir::physical_export::compile_physical_asap_dag(root).unwrap(); - dag.validate().unwrap(); - dag.nodes - .iter() - .find(|n| n.payload.kind_name() == kind) - .unwrap() - .materialization - } - - /// A node whose timing its candidate fixed (the query-time read boundary - /// of `finalize_query_candidate`) keeps an explicit assignment that agrees - /// with that timing and rejects one that needs the other timing. - #[test] - fn fixed_timing_keeps_an_agreeing_assignment_and_rejects_a_conflict() { - let state = agg(scan(), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let stored = Materialization::MatAtQuery(Medium::Disk); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&read, stored); - let timed = - apply_materialization_timings(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(timed.materialization, stored); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), stored); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - assignment.set(&read, AT_INGESTION); - assert_eq!( - apply_materialization_timings(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - materialization: AT_INGESTION, - } - ) - ); - } - - /// The Debug output names each assigned node by its operator kind, not - /// its whole sub-DAG. - #[test] - fn debug_names_each_assigned_node_by_its_kind() { - let mut assignment = MaterializationAssignment::default(); - assignment.set(&agg(scan(), kll()), AT_INGESTION); - let text = format!("{assignment:?}"); - assert!(text.contains("SummaryAgg@"), "{text}"); - assert!(!text.contains("fields"), "{text}"); - } - - /// A node fixed at ingestion time by its candidate rejects a `MatAtQuery` - /// assignment, which needs the other timing. - #[test] - fn ingestion_fixed_node_rejects_mat_at_query() { - let fixed = placed_at_ingestion(max(scan())); - let root = max(Rc::clone(&fixed)); - let at_query = Materialization::MatAtQuery(Medium::Memory); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&fixed, at_query); - assert_eq!( - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some( - ExecutionDataStateError::MaterializationConflictsWithFixedTiming { - operator: "Aggregate", - fixed: ExecutionTiming::IngestionTime, - materialization: at_query, - } - ) - ); - } - - /// An explicit `NotMaterialized` on a node whose timing its candidate - /// fixed agrees with either timing: the node keeps its fixed timing and - /// records `NotMaterialized`. - #[test] - fn explicit_not_materialized_keeps_a_fixed_timing() { - let read = Rc::new( - (*finalize(agg(scan(), exact_sum()))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let root = max(Rc::clone(&read)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&read, Materialization::NotMaterialized); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let read = child(&timed); - assert_eq!(read.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(read.materialization, Materialization::NotMaterialized); - } - - /// A query-time summary build cannot read ingestion-time rows that are - /// not stored: at query time they no longer exist. Storing the rows node - /// at ingestion time makes the same build valid (#738). - #[test] - fn query_time_summary_over_unstored_ingestion_rows_is_rejected() { - let rows = placed_at_ingestion(max(scan())); - let summary = agg(Rc::clone(&rows), kll()); - assert_eq!( - apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge: "SummaryAgg.child", - operator: "Aggregate", - } - ) - ); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&rows, AT_INGESTION); - let timed = - apply_materialization_timings(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - } - - /// The same holds for exact-accumulator state: a query-time build cannot - /// read an ingestion-time exact state that is not stored. - #[test] - fn query_time_summary_over_unstored_ingestion_state_is_rejected() { - let state = placed_at_ingestion(agg(scan(), exact_sum())); - let summary = agg(Rc::clone(&state), kll()); - assert_eq!( - apply_materialization_timings( - &summary, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some( - ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { - edge: "SummaryAgg.child", - operator: "SummaryAgg", - } - ) - ); - - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - let timed = - apply_materialization_timings(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).materialization, AT_INGESTION); - } - - /// A root stored at ingestion time is valid: the query reads the stored - /// output. A root that runs at ingestion time without being stored there - /// is not. - #[test] - fn a_root_may_be_stored_at_ingestion_time() { - let root = max(scan()); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&root, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "Aggregate"), AT_INGESTION); - - let state = agg(scan(), exact_sum()); - let root = finalize(Rc::clone(&state)); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&state, AT_INGESTION); - assignment.set(&root, AT_INGESTION); - let timed = - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), AT_INGESTION); - - let mut assignment = MaterializationAssignment::default(); - let root = placed_at_ingestion(max(scan())); - assignment.set(&root, Materialization::NotMaterialized); - assert_eq!( - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .err(), - Some(ExecutionDataStateError::UnstoredIngestionRoot { - operator: "Aggregate" - }) - ); - } - - /// Both paired operands must be plain; an unrelated state column is not - /// an input. - #[test] - fn pearson_corr_checks_both_operand_states() { - let corr_over = |state_column: usize| { - let mut fields = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("x", DataType::Float64, false), - Field::plain("y", DataType::Float64, false), - Field::plain("unused", DataType::Float64, false), - ]; - fields[state_column].dtype = kll(); - aggregate( - AggIntent::PearsonCorr { left: 1, right: 2 }, - scan_with(fields), - ) - }; - for operand in [1, 2] { - assert!(matches!( - validate_at_ingestion(&corr_over(operand), ExecutionTiming::QueryTime), - Err(ExecutionDataStateError::NonPlainOperand { .. }) - )); - } - validate_at_ingestion(&corr_over(3), ExecutionTiming::QueryTime).unwrap(); - } - // Typed derived updates retain one opaque series identity only when both - // operands cover the same series; arbitrary labels are never admitted. - #[test] - fn ingestion_time_binary_accepts_only_well_typed_series_identity() { - use crate::ir::operator::non_asap::BinaryOperator; - use crate::ir::operator::BinaryOpKind; - use crate::ir::scalar::ArithmeticOpKind; - let identity = crate::ir::schema::PROMQL_SERIES_IDENTITY; - // A finalized per-series Sum of `metric`'s rows. - let operand = |metric: &str, schema: &Schema| { - let rows = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let mut state = schema.clone(); - state.fields[0].dtype = exact_sum(); - let sum = Rc::new(OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: rows, - family: exact_sum(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }), - state, - )); - Rc::new(OperatorNode::with_schema( - crate::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: sum }), - schema.clone(), - )) - }; - let validate = |schema: Schema, rhs: &str| { - let binary = Rc::new(OperatorNode::with_schema( - crate::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool: false, - lhs: operand("m", &schema), - rhs: operand(rhs, &schema), - }), - schema, - )); - validate_at_ingestion(&placed_at_ingestion(binary), ExecutionTiming::IngestionTime) - }; - let mut schema = Schema::lifted( - vec![ - Field::plain("value", DataType::Float64, false), - Field::plain("ts", DataType::Timestamp, false), - ], - None, - ); - schema.time_index = Some(1); - assert!(validate(schema.clone(), "m").is_ok()); - assert!( - validate(schema.clone(), "n").is_ok(), - "no identity to align" - ); - schema - .fields - .push(Field::plain(identity, DataType::Utf8, false)); - assert!(validate(schema.clone(), "m").is_ok()); - assert_eq!( - validate(schema.clone(), "n"), - Err(ExecutionDataStateError::InvalidIngestionTimeBinary), - "different selectors may cover different series" - ); - for mutation in 0..4 { - let mut invalid = schema.clone(); - match mutation { - 0 => invalid.fields[2].nullable = true, - 1 => invalid.fields[2].dtype = FieldDataType::Plain(DataType::Timestamp), - 2 => invalid.fields.push(invalid.fields[2].clone()), - _ => invalid.fields[2].name = "label".into(), - } - assert_eq!( - validate(invalid, "m"), - Err(ExecutionDataStateError::InvalidIngestionTimeBinary) - ); - } - } -} diff --git a/crates/types/src/ir/schema/family_capability.rs b/crates/types/src/ir/schema/family_capability.rs index 6af62d919..960403e36 100644 --- a/crates/types/src/ir/schema/family_capability.rs +++ b/crates/types/src/ir/schema/family_capability.rs @@ -175,8 +175,8 @@ impl FamilyCapability { HydraKind::HydraKll => SketchAlgorithm::HydraKll, }) } - FieldDataType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { - return Self::of_per_instance_sketch(kind.algorithm()) + FieldDataType::Sketch(algorithm, GroupingStrategy::PerSubpopulationInstance) => { + return Self::of_per_instance_sketch(algorithm) } FieldDataType::Plain(_) | FieldDataType::SummarySample(..) @@ -194,13 +194,10 @@ impl FamilyCapability { mod tests { use super::*; use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::{ExactParams, SketchKind, SketchParams}; + use crate::ir::schema::{ExactParams, SketchParams}; - fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::default(), - ) + fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm, GroupingStrategy::default()) } fn total() -> SketchStatistic { @@ -335,7 +332,7 @@ mod tests { /// per group: not counters, IRate or a Hydra. #[test] fn only_per_group_mergeable_families_merge_groups() { - use crate::ir::schema::{HydraKind, HydraParams}; + use crate::ir::schema::HydraKind; let kll = sketch( SketchAlgorithm::Kll, SketchParams::Kll { @@ -369,12 +366,6 @@ mod tests { kind, GroupingStrategy::SharedMultiSubpopulation { kind: HydraKind::HydraCms, - params: HydraParams::HydraCms { - width: 64, - depth: 5, - shared_rows: 3, - shared_columns: 256, - }, }, ); assert_eq!( diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 90796a76b..cce079379 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -164,11 +164,11 @@ pub enum FieldDataType { /// `Increase`). Value consumers require an explicit finalization boundary. ExactAggregate(ExactKind, ExactParams), /// Approximate sketch state (KLL/CMS/HLL/…), evaluated by a - /// `SummaryEstimate`. A [`SketchKind`] already carries the concrete - /// algorithm, params, and grouping layout committed to, not just its - /// category — a bound node needs to know it's specifically independent - /// KLL or shared Hydra-backed CMS, not merely "some sketch". - Sketch(SketchKind, GroupingStrategy), + /// `SummaryEstimate`: its family and grouping layout. The family's + /// parameters θ are not part of the state's type, so states of one family + /// and grouping are one memo group whatever their size; θ is a + /// candidate's `LogicalProperties::parameters` (concepts doc, section 3). + Sketch(SketchAlgorithm, GroupingStrategy), /// A summary sample: the subset of input rows a [`SamplingKind`] keeps, /// such as a reservoir. Not raw data samples. Serialized as `Sample`. #[serde(rename = "Sample")] diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index 913ab8728..d63742553 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -92,8 +92,6 @@ pub fn with_promql_series_identity( } let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; rebuilt.guarantee = node.guarantee.clone(); - rebuilt.timing = node.timing; - rebuilt.materialization = node.materialization; let rebuilt = Rc::new(rebuilt); cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); Ok(rebuilt) diff --git a/crates/types/src/ir/schema/sketch.rs b/crates/types/src/ir/schema/sketch.rs index a2c3a21e1..b4c4ace79 100644 --- a/crates/types/src/ir/schema/sketch.rs +++ b/crates/types/src/ir/schema/sketch.rs @@ -133,6 +133,10 @@ pub enum SketchParams { depth: u32, heap_size: u32, }, + /// The sizes of a Hydra structure shared across subpopulations + /// ([`GroupingStrategy::SharedMultiSubpopulation`]): θ, like every other + /// family's parameters, not part of the grouping. + Hydra(HydraParams), } impl PartialEq for SketchParams { @@ -203,6 +207,7 @@ impl PartialEq for SketchParams { (P::Hll { precision: a, .. }, P::Hll { precision: b, .. }) => a == b, (P::DDSketch { alpha: a }, P::DDSketch { alpha: b }) => a == b, (P::Kmv { k: a, .. }, P::Kmv { k: b, .. }) => a == b, + (P::Hydra(a), P::Hydra(b)) => a == b, // Different variants. Listed by `self`'s variant, not `_`, so a // new variant fails to compile here until it is compared above. (P::UnivMon { .. }, _) @@ -213,7 +218,8 @@ impl PartialEq for SketchParams { | (P::CmsWithHeap { .. }, _) | (P::Kmv { .. }, _) | (P::CountSketch { .. }, _) - | (P::CountSketchWithHeap { .. }, _) => false, + | (P::CountSketchWithHeap { .. }, _) + | (P::Hydra(_), _) => false, } } } @@ -490,6 +496,23 @@ pub enum HydraParams { }, } +impl HydraParams { + /// The parameters of the per-subpopulation sketch this shared structure + /// emulates, which its error bound and per-group state follow. + pub fn per_subpopulation(&self) -> SketchParams { + match *self { + HydraParams::HydraKll { k, .. } => SketchParams::Kll { + k, + failure_probability: None, + }, + HydraParams::HydraCms { width, depth, .. } => SketchParams::Cms { width, depth }, + HydraParams::HydraCountSketch { width, depth, .. } => { + SketchParams::CountSketch { width, depth } + } + } + } +} + /// How a grouped aggregate's summary state is physically instantiated /// across its `by` subpopulations — orthogonal to *which* /// `SketchKind`/`SamplingKind`/`WaveletKind`/`StatModelKind` answers the @@ -517,10 +540,8 @@ pub enum GroupingStrategy { /// shares across a *query's own* subpopulations/group-by keys, not /// across tenants in a deployment-isolation sense — a different, and /// unrelated, kind of "sharing". - SharedMultiSubpopulation { - kind: HydraKind, - params: HydraParams, - }, + /// Its sizes are θ ([`SketchParams::Hydra`]), not part of the grouping. + SharedMultiSubpopulation { kind: HydraKind }, } impl Default for GroupingStrategy { @@ -751,33 +772,6 @@ mod tests { assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); } - /// Two KLLs with the same `k` are the same sketch, whatever failure - /// probability each was sized for, so they compare equal and can be shared. - #[test] - fn kll_params_compare_equal_whatever_their_failure_probability() { - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - assert_eq!(kll(331, Some(0.001)), kll(331, None)); - assert_eq!(kll(331, Some(0.001)), kll(331, Some(0.0001))); - assert_ne!(kll(331, None), kll(269, None)); - assert_ne!( - kll(331, None), - SketchParams::Kmv { - k: 331, - relative_error: None - } - ); - let family = |params| { - crate::ir::schema::FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, params), - GroupingStrategy::default(), - ) - }; - assert_eq!(family(kll(331, Some(0.001))), family(kll(331, None))); - } - #[test] #[should_panic(expected = "SketchKind parameter mismatch")] fn sketch_kind_rejects_params_from_another_algorithm() { diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index b6e8c0157..e219b78f5 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -12,8 +12,8 @@ //! - [`workload`] — the planner inputs (#509): the query and data //! workloads and the parsed workload the query-language frontends produce //! ([`workload::parsed_workload`]). -//! - [`physical`] — Stage 2 decision data: the materialization of a -//! sub-DAG ([`physical::Materialization`]). +//! - [`physical`] — Stage 2 decision data: the physical properties of a +//! candidate ([`physical::PhysicalProperties`]: timing and lifetime). //! - [`stages`] — the types one planning stage hands the next: the Stage 1 //! → Stage 2 memo of groups and implementation candidates //! ([`stages::LogicalASAPMemoCandidates`]) and the Stage 2 → Stage 3 diff --git a/crates/types/src/physical/materialization.rs b/crates/types/src/physical/materialization.rs deleted file mode 100644 index e30e192b3..000000000 --- a/crates/types/src/physical/materialization.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! The materialization option of each sub-DAG (#509 §Materialization), the -//! Stage 2 decision a node's execution timing -//! ([`crate::ir::properties::ExecutionTiming`]) is derived -//! from. - -use crate::ir::properties::ExecutionTiming; - -/// Where a materialized output is stored (#509 §Materialization). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub enum Medium { - Memory, - Disk, -} - -/// The materialization option of one sub-DAG (#509 §Materialization, -/// Algorithm 3): whether its output is stored across query executions, and -/// whether it is computed at ingestion time or at query time. -#[derive( - Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, -)] -pub enum Materialization { - /// Runs at query time for each execution; its output is discarded. - #[default] - NotMaterialized, - /// Runs as data arrives; its output is stored until a query reads it. - MatAtIngestion(Medium), - /// Runs when a query first needs it; its output is reused by the other - /// queries of the same evaluation and discarded after it. - MatAtQuery(Medium), - /// Runs at query time and keeps its output across evaluations: a - /// tumbling-window chain whose readers evaluate together every tumbling window - /// width builds its newest tumbling window at each evaluation and keeps the older - /// ones for the next (#509 Example 4's B3). Offered only when the - /// deployment keeps query-time results. - MatAtQueryKeptAcrossEvaluations(Medium), -} - -impl Materialization { - /// When the sub-DAG runs: only `MatAtIngestion` runs at ingestion time. - pub fn timing(self) -> ExecutionTiming { - match self { - Self::MatAtIngestion(_) => ExecutionTiming::IngestionTime, - Self::NotMaterialized - | Self::MatAtQuery(_) - | Self::MatAtQueryKeptAcrossEvaluations(_) => ExecutionTiming::QueryTime, - } - } - - /// Whether a node of this materialization is a materialized view: its - /// output is kept across evaluations. - pub(crate) fn is_materialized_view(self) -> bool { - matches!( - self, - Self::MatAtIngestion(_) | Self::MatAtQueryKeptAcrossEvaluations(_) - ) - } -} diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index bf10811ac..8d3b41089 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,11 +1,15 @@ -//! #509 Stage 2 decision data: the materialization of each sub-DAG. +//! #509 Stage 2 decision data: the physical properties of each candidate. //! -//! - [`materialization`] — whether a sub-DAG's output is stored and when it -//! is computed ([`Materialization`], [`Medium`]). +//! - [`properties`] — when a candidate runs, how long its output lives and +//! on which schedule it is delivered ([`PhysicalProperties`], [`Lifetime`], +//! [`Medium`], [`Schedule`]). //! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG //! ([`PhysicalWorkloadPlan`]). -pub mod materialization; +pub mod properties; pub mod workload_plan; -pub use materialization::{Materialization, Medium}; +pub use properties::{ + EvaluationSchedule, ExecutionTiming, InvalidPhysicalProperties, Lifetime, Medium, + PhysicalProperties, Schedule, +}; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs new file mode 100644 index 000000000..6080c1184 --- /dev/null +++ b/crates/types/src/physical/properties.rs @@ -0,0 +1,228 @@ +//! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell, \sigma)$ +//! (#1200; concepts doc `memo-groups-properties-requirements.md`, sections 1 +//! and 3): when it runs ([`ExecutionTiming`]), how long its output lives +//! ([`Lifetime`]) and, for a materialized output, on which times or grid it +//! is delivered ([`Schedule`]). +//! +//! | timing | lifetime | schedule | +//! |---|---|---| +//! | query | `None` | none: runs in each context κ that reads it | +//! | query | `OneEvaluation` | `Periodic { interval, phase }` | +//! | query | `AcrossEvaluations` | `TumblingWindowGrid { phase }` | +//! | ingestion | `None` | none: runs inside the ingestion work that reads it | +//! | ingestion | `AcrossEvaluations` | `TumblingWindowGrid { phase: 0 }` | +//! | ingestion | `OneEvaluation` | invalid: there is no evaluation at ingestion time | +//! +//! There is no stored physical requirement: $r^{\mathrm{phys}}_{c,i}$ is a +//! function of the consumer's own properties +//! ([`crate::stages::satisfies_physical`]). + +pub use crate::ir::properties::ExecutionTiming; + +/// Where a materialized output is stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum Medium { + Memory, + Disk, +} + +/// $\ell_v$: how long a candidate's output lives. Only a materialized output +/// has a medium. +#[derive( + Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum Lifetime { + /// Discarded after its consumer's execution completes. + #[default] + None, + /// Materialized until every consumer within one evaluation finishes. + OneEvaluation(Medium), + /// Materialized across evaluations, so later evaluations reuse it. + AcrossEvaluations(Medium), +} + +/// $\sigma_v$: what a materialized output ($\ell \ne$ None) is delivered on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Schedule { + /// `OneEvaluation`: delivered at $\varphi + I\cdot\mathbb Z$. + Periodic { interval_ms: u64, phase_ms: u64 }, + /// `AcrossEvaluations`: the tumbling-window grid's phase, modulo the + /// grid's width. + TumblingWindowGrid { phase_ms: u64 }, +} + +/// A timing, lifetime and schedule that cannot go together (the invalid +/// combinations of the concepts doc, section 2). +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum InvalidPhysicalProperties { + #[error("an ingestion-time output cannot be materialized for one evaluation: there is no evaluation at ingestion time")] + OneEvaluationAtIngestion, + #[error("an output that is not materialized has no schedule: it runs in its consumer's execution context")] + ScheduleWithoutMaterialization, + #[error("a materialized output needs a schedule")] + MaterializedWithoutSchedule, + #[error( + "one evaluation needs a periodic schedule; across evaluations needs a tumbling-window grid" + )] + ScheduleDoesNotMatchLifetime, + #[error("a periodic schedule needs a positive interval and a phase below it")] + PhaseNotBelowInterval, + #[error("an ingestion-time tumbling-window grid is anchored at the epoch: its phase is 0")] + IngestionGridPhaseNotZero, +} + +/// $p^{\mathrm{phys}}_v = (\tau, \ell, \sigma)$. [`Self::new`] rejects the +/// invalid combinations (concepts doc, section 2). A grid schedule's +/// phase below the tumbling width is checked where the width is known: when +/// a `PhysicalCandidate` is built. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalProperties { + timing: ExecutionTiming, + lifetime: Lifetime, + schedule: Option, +} + +impl PhysicalProperties { + /// The properties `(timing, lifetime, schedule)`, unless they cannot go + /// together. + pub fn new( + timing: ExecutionTiming, + lifetime: Lifetime, + schedule: Option, + ) -> Result { + let _ = (timing, lifetime, schedule); + todo!("#1207 PR 1: reject the invalid combinations of section 2") + } + + /// $\tau_v$. + pub fn timing(&self) -> ExecutionTiming { + self.timing + } + + /// $\ell_v$. + pub fn lifetime(&self) -> Lifetime { + self.lifetime + } + + /// $\sigma_v$: `None` exactly when $\ell_v$ = None. + pub fn schedule(&self) -> Option<&Schedule> { + self.schedule.as_ref() + } + + /// Whether $\ell_v \ne$ None. Derived, not stored. + pub fn is_materialized(&self) -> bool { + self.lifetime != Lifetime::None + } +} + +/// A periodic schedule $\varphi + I\cdot\mathbb Z$: a materialized consumer's +/// own times, or a context's times $T_\kappa$ (Stage 3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EvaluationSchedule { + interval_ms: u64, + phase_ms: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + use ExecutionTiming::{IngestionTime, QueryTime}; + + const MEMORY: Medium = Medium::Memory; + + fn periodic(interval_ms: u64, phase_ms: u64) -> Option { + Some(Schedule::Periodic { + interval_ms, + phase_ms, + }) + } + + fn grid(phase_ms: u64) -> Option { + Some(Schedule::TumblingWindowGrid { phase_ms }) + } + + /// Every invalid combination of the concepts doc, section 2. + #[test] + fn new_rejects_each_invalid_combination() { + use InvalidPhysicalProperties as E; + let cases = [ + ( + IngestionTime, + Lifetime::OneEvaluation(MEMORY), + periodic(10_000, 0), + E::OneEvaluationAtIngestion, + ), + ( + QueryTime, + Lifetime::None, + periodic(10_000, 0), + E::ScheduleWithoutMaterialization, + ), + ( + QueryTime, + Lifetime::OneEvaluation(MEMORY), + None, + E::MaterializedWithoutSchedule, + ), + ( + QueryTime, + Lifetime::OneEvaluation(MEMORY), + grid(0), + E::ScheduleDoesNotMatchLifetime, + ), + ( + QueryTime, + Lifetime::AcrossEvaluations(MEMORY), + periodic(10_000, 0), + E::ScheduleDoesNotMatchLifetime, + ), + ( + QueryTime, + Lifetime::OneEvaluation(MEMORY), + periodic(10_000, 10_000), + E::PhaseNotBelowInterval, + ), + ( + QueryTime, + Lifetime::OneEvaluation(MEMORY), + periodic(0, 0), + E::PhaseNotBelowInterval, + ), + ( + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + grid(5_000), + E::IngestionGridPhaseNotZero, + ), + ]; + for (timing, lifetime, schedule, error) in cases { + assert_eq!( + PhysicalProperties::new(timing, lifetime, schedule), + Err(error) + ); + } + } + + /// The valid (τ, ℓ) pairs of section 2, each with a schedule of its kind. + #[test] + fn new_accepts_each_valid_pair() { + for (timing, lifetime, schedule) in [ + (QueryTime, Lifetime::None, None), + (IngestionTime, Lifetime::None, None), + ( + QueryTime, + Lifetime::OneEvaluation(MEMORY), + periodic(10_000, 5_000), + ), + (QueryTime, Lifetime::AcrossEvaluations(MEMORY), grid(5_000)), + ( + IngestionTime, + Lifetime::AcrossEvaluations(Medium::Disk), + grid(0), + ), + ] { + let properties = PhysicalProperties::new(timing, lifetime, schedule).unwrap(); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + } +} diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs index 47412bd9b..ce8ed9653 100644 --- a/crates/types/src/physical/workload_plan.rs +++ b/crates/types/src/physical/workload_plan.rs @@ -1,10 +1,8 @@ -//! One evaluated materialization assignment rebuilt as a physical DAG. -use std::collections::BTreeMap; - -use crate::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; +//! One realized selection as a physical DAG. +use crate::ir::physical_export::PhysicalASAPDAG; use crate::ir::QueryRoot; - -use super::Materialization; +#[cfg(any(test, feature = "test-support"))] +use crate::physical::Lifetime; /// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one /// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; @@ -15,25 +13,20 @@ pub struct PhysicalWorkloadPlan { pub id: String, /// The id of the logical candidate it implements. pub from_logical: String, - /// The timed query roots, one per query in workload order, that `dag` + /// The query roots, one per query in workload order, that `dag` /// exports. pub roots: Vec, pub dag: PhysicalASAPDAG, - /// Its materialization assignment: the option chosen for each summary - /// build (`SummaryAgg` or `SummaryTumblingWindowChain`) of `dag`, by node id, including - /// the builds left `NotMaterialized`, and each other node materialized - /// (a shared node built once per evaluation). A build one query reads - /// at ingestion time and another at query time appears once per copy. - pub materialization: BTreeMap, } impl PhysicalWorkloadPlan { - /// Whether any node is materialized: a summary build, or a shared + /// Whether any node of `dag` is materialized: a summary build, or a shared /// non-summary node built once per evaluation. #[cfg(any(test, feature = "test-support"))] pub fn materializes_any(&self) -> bool { - self.materialization - .values() - .any(|m| *m != Materialization::NotMaterialized) + self.dag + .nodes + .iter() + .any(|node| node.lifetime != Lifetime::None) } } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 8d18a64bd..f77cefedb 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -1,96 +1,79 @@ -//! Implementation candidates attached to memo groups. +//! The memo and each group's candidates +//! $v = (e_v, p^{\log}_v)$ and each candidate's input requirements +//! $r^{\log}_{c,i}$ (concepts doc `memo-groups-properties-requirements.md`, +//! section 3; #1203). use std::collections::HashSet; -use std::rc::Rc; use super::{MemoGroup, MemoGroupId}; -use crate::ir::{OperatorNode, QueryRoot}; +use crate::ir::schema::{SketchAlgorithm, SketchParams}; +use crate::ir::Operator; +use crate::ir::QueryRoot; -/// The memo: the memo groups, each query's root over them, and each group's -/// candidate implementations. +/// One candidate: index `index` in group `group`'s candidates +/// ([`MemoGroup::candidates`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CandidateForMemoGroupId { + pub group: MemoGroupId, + pub index: usize, +} + +/// The memo, Stage 0's and Stage 1's one type: the memo groups, each with +/// its candidates ([`MemoGroup::candidates`]: empty after Stage 0), and each +/// query's root over them. #[derive(Debug, Clone, Default)] pub struct Memo { pub groups: Vec, - /// Each query's root over groups, by index in the workload (the - /// initial logical memo's query roots); groups added later are never a - /// query's root. + /// Each query's root over groups, by index in the workload. pub query_roots: Vec>, - /// Each group's implementation candidates, by group index. - pub candidates: Vec>, } -/// One way to implement a group: its node as Stage 2 implements it, over -/// child groups. -#[derive(Debug, Clone)] -pub struct CandidateForMemoGroup { - /// The groups a fused computation also implements, beyond the group - /// whose candidate it is (its owner, implicit from its position in - /// [`Memo::candidates`]); empty otherwise. - pub fused_groups: Vec, - /// The implementation's own node(s): its root node, whose descendants - /// down to [`Self::input_groups`] are its private nodes. - pub root_node: Rc, - /// The groups it reads, whose implementations are chosen separately. - pub input_groups: Vec, - /// How it was made: by Pass 1, or by which Pass 2 transformation. - pub origin: OriginOfCandidateForMemoGroup, - /// The number of equal shares of its measures' accuracy budget its - /// summaries are sized for: 1 for the whole budget, `n` when it is one - /// of `n` approximate inputs of a query (a ratio's operand). This positive - /// generation parameter is already encoded in concrete sketch parameters; - /// it is not an equality constraint on the selected number of readers or - /// approximate inputs. Recompose and validate accuracy at query roots. - /// Larger allocations may be valid but costlier than necessary. - pub accuracy_budget_parts: usize, -} +impl Memo { + /// Hash-consing: the group with this expression, added if new. + pub fn find_or_add_group(&mut self, expression: Operator) -> MemoGroupId { + let _ = expression; + todo!("#1207 PR 2: hash-consing over groups") + } -/// Where an implementation candidate came from. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum OriginOfCandidateForMemoGroup { - /// Pass 1: exact, exact accumulator, or a summary family and size. - Pass1, - /// One build serving several reader groups (summary capability). - SharedBuild, - /// A summary readout factored from a composite implementation. - SummaryResult, - /// Canonical value preprocessing feeding a summary build. - SummaryInput, - /// Tumbling windows, or the newest tumbling windows of another group's chain. - TumblingWindows, - /// A coarse group regrouping a finer group. - Rollup, - /// A top-k read from another group's ranking. - TopKReuse, - /// Its metric's scans widened to a shared label set, read through a - /// `Project` (scan widening). - WidenedScans, -} + /// Adds `candidate` to `group`'s candidates. + pub fn add_candidate( + &mut self, + group: MemoGroupId, + candidate: CandidateForMemoGroup, + ) -> CandidateForMemoGroupId { + let _ = (group, candidate); + todo!("#1207 PR 2") + } + + /// The candidate `id`. + pub fn candidate(&self, id: CandidateForMemoGroupId) -> &CandidateForMemoGroup { + let _ = id; + todo!("#1207 PR 2") + } -impl Memo { /// The groups, each after every group it or one of its candidates /// reads, the lower index first among those ready: a dependency order for - /// candidate traversal, whatever order transformations appended - /// groups in. `None` when those reads form a cycle, a group reading - /// itself included. + /// candidate traversal, whatever order rules appended groups in. `None` + /// when those reads form a cycle, a group reading itself included. pub fn topological_order(&self) -> Option> { let count = self.groups.len(); let mut reads: Vec> = vec![HashSet::new(); count]; for (g, group) in self.groups.iter().enumerate() { - let candidates = self.candidates.get(g).map_or(&[][..], Vec::as_slice); - for child in group - .child_groups - .iter() - .chain(candidates.iter().flat_map(|c| &c.input_groups)) - { + for child in group.child_groups.iter().chain( + group + .candidates + .iter() + .flat_map(|c| c.expression.operator.children()), + ) { // A group reading itself never becomes ready: a cycle. reads[g].insert(child.0 as usize); } } - let mut readers: Vec> = vec![Vec::new(); count]; + let mut consumers: Vec> = vec![Vec::new(); count]; let mut waiting: Vec = reads.iter().map(HashSet::len).collect(); for (g, read) in reads.iter().enumerate() { for &child in read { - readers[child].push(g); + consumers[child].push(g); } } let mut ready: std::collections::BinaryHeap> = (0..count) @@ -100,13 +83,174 @@ impl Memo { let mut order = Vec::with_capacity(count); while let Some(std::cmp::Reverse(g)) = ready.pop() { order.push(MemoGroupId(g as u32)); - for &reader in &readers[g] { - waiting[reader] -= 1; - if waiting[reader] == 0 { - ready.push(std::cmp::Reverse(reader)); + for &consumer in &consumers[g] { + waiting[consumer] -= 1; + if waiting[consumer] == 0 { + ready.push(std::cmp::Reverse(consumer)); } } } (order.len() == count).then_some(order) } } + +/// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. +#[derive(Debug, Clone)] +pub struct CandidateForMemoGroup { + /// $e_v$. + pub expression: CandidateExpression, + /// $p^{\log}_v$. + pub logical_properties: LogicalProperties, + /// $r^{\log}_{c,i}$, one per input group: per child of + /// `expression.operator`, in [`Operator::children`] order. + pub input_requirements: Vec, + /// The groups a fused computation also implements, beyond the group + /// whose candidate it is; empty otherwise. + pub fused_groups: Vec, + /// Which rule added it. Not in the formulation: kept for + /// `stage_pipeline --text` and the viewer (approved 2026-11-05). + pub origin: OriginOfCandidateForMemoGroup, +} + +/// Which rule added a candidate. Nothing in planning reads it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OriginOfCandidateForMemoGroup { + /// Pass 1: exact, exact accumulator, or a summary family. + Pass1, + /// Canonical value preprocessing feeding a summary build. + SummaryInput, + /// A tumbling-window layout of a summary state group. + TumblingWindows, + /// A coarse group regrouping a finer group. + Rollup, + /// A top-k read from another group's ranking. + TopKReuse, + /// Its metric's scans widened to a shared label set, read through a + /// `Project` (scan widening). + WidenedScans, +} + +/// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with +/// its arguments, whose children are groups. The input groups $G_{v,i}$ are +/// [`Operator::children`], in that order. +#[derive(Debug, Clone, PartialEq)] +pub struct CandidateExpression { + pub operator: Operator, +} + +/// $p^{\log}_v = (\boldsymbol\theta_v, \omega_v)$. The family is the +/// operator's argument, not a property. [`Self::new`] rejects parameters of +/// another family (or on an exact operator) and a window on an operator that +/// cannot be windowed. +#[derive(Debug, Clone)] +pub struct LogicalProperties { + /// $\boldsymbol\theta_v$, from the family's predefined list; `None` for + /// an exact operator. + parameters: Option, + /// $\omega_v$. + window_layout: Option, +} + +impl LogicalProperties { + pub fn new( + expression: &CandidateExpression, + parameters: Option, + window_layout: Option, + ) -> Result { + let _ = (expression, parameters, window_layout); + todo!("#1207 PR 1: reject parameters of another family and invalid windows") + } + + /// $\boldsymbol\theta_v$. + pub fn parameters(&self) -> Option<&SketchParams> { + self.parameters.as_ref() + } + + /// $\omega_v$. + pub fn window_layout(&self) -> Option<&WindowLayout> { + self.window_layout.as_ref() + } +} + +/// $\omega_v$. The window count is the group's query range ÷ width; +/// retention is $h_v$ (Stage 3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum WindowLayout { + Tumbling { width_ms: u64 }, +} + +/// Logical properties or a logical requirement whose dimensions cannot go +/// together: bad input to [`LogicalProperties::new`] or +/// [`LogicalRequirement::new`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum InvalidLogicalProperties { + #[error("the parameters belong to another family than the operator's")] + ParametersOfAnotherFamily, + #[error("an exact operator has no summary parameters")] + ParametersOnExactOperator, + #[error("the operator cannot be windowed")] + WindowOnOperatorThatCannotBeWindowed, + #[error("a tumbling window has a positive width")] + ZeroWindowWidth, + #[error("the parameter floor is not in its family's predefined list")] + FloorNotInParameterList, +} + +/// The family's predefined $\boldsymbol\theta$ list (#1207 step 6), each with +/// its error bound derived through `estimators::local_guarantee` for a given +/// statistic. +pub fn parameter_list(family: SketchAlgorithm) -> &'static [SketchParams] { + let _ = family; + todo!("#1207 PR 1: the predefined θ list per family") +} + +/// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. +/// [`Self::new`] rejects a floor that is not in the family's predefined list. +#[derive(Debug, Clone)] +pub struct LogicalRequirement { + /// $\boldsymbol\theta^{\min}_{c,i}$: the smallest $\boldsymbol\theta$ in + /// the list whose $\varepsilon$, for the statistic the consumer reads, is + /// at most $\text{cap}_{c,i}$ = min(validity cap of $g_c$, pruning cap). + parameter_floor: Option, +} + +impl LogicalRequirement { + pub fn new(parameter_floor: Option) -> Result { + let _ = parameter_floor; + todo!("#1207 PR 1: reject a floor not in the family's predefined list") + } + + /// $\boldsymbol\theta^{\min}_{c,i}$. + pub fn parameter_floor(&self) -> Option<&SketchParams> { + self.parameter_floor.as_ref() + } + + /// $g_c$'s linear bound: $\beta_{c,i}$ and $b_{c,i}$ per input, read + /// from the operator and its arguments. An operator without an error + /// contract is reported as unsupported; no factor is guessed. + pub fn error_propagation(consumer: &Operator) -> ErrorPropagation { + let _ = consumer; + todo!("#1207 PR 4: the error contract of each operator") + } +} + +/// The local error propagation of one consumer: its output error is at +/// most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error stays +/// within its validity cap (concepts doc, section 4; approved 2026-11-05). +#[derive(Debug, Clone, PartialEq)] +pub struct ErrorPropagation { + /// $\beta_{c,i}$ per input, for that input's metric. + pub factor_per_input: Vec, + /// $b_{c,i}$ per input: the cap under which the linear bound holds; + /// `None` when the bound holds everywhere (e.g. a ratio caps only its + /// denominator). + pub validity_cap_per_input: Vec>, +} + +/// $\alpha^{\log}_{u,c,i}$: $\boldsymbol\theta_u \succeq +/// \boldsymbol\theta^{\min}_{c,i}$, compared per family on the parameters +/// that set $\varepsilon$. No floor, or an exact producer, always satisfies. +pub fn satisfies_logical(producer: &LogicalProperties, requirement: &LogicalRequirement) -> bool { + let _ = (producer, requirement); + todo!("#1207 PR 1: per-family dominance") +} diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index d9b4899c8..6a8205011 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -1,14 +1,24 @@ -//! Stage 1, Pass 2's output, which Stage 2 implements. +//! Stage 1, Pass 2's output, which Stage 2 expands: the memo after every +//! rule, and the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201). -use super::Memo; +use super::{CandidateForMemoGroupId, Memo}; -/// Stage 1, Pass 2's output: the Pass 1 groups plus the groups Pass 2 adds -/// (shared producers, shared summary states), each with its Pass 1 -/// candidates plus every Pass 2 rule's (by `origin`), including the -/// candidates reading a shared summary state. +/// $(u, c, i)$ over logical candidates: `producer` is a candidate of +/// $G_{c,i}$, the group input `input_index_of_consumer` of `consumer` reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LogicalEdge { + pub producer: CandidateForMemoGroupId, + pub consumer: CandidateForMemoGroupId, + pub input_index_of_consumer: usize, + /// $\alpha^{\log}_{u,c,i}$. + pub satisfies_logical_requirement: bool, +} + +/// Stage 1, Pass 2's output. #[derive(Debug, Clone)] pub struct LogicalASAPMemoCandidates { - /// The expanded memo: groups, query roots (passed through from Pass 1) - /// and each group's candidates. + /// The memo after the last rule. pub memo: Memo, + /// Every $(u, c, i)$ with $u \in G_{c,i}$, grouped by $(c, i)$. + pub edges: Vec, } diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 765ef09ca..67df23b50 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -1,8 +1,10 @@ -//! Memo groups: the distinct logical sub-expressions of the workload. +//! Memo groups: the distinct logical sub-expressions of the workload, each +//! with its candidates. -use std::collections::{BTreeSet, HashMap}; +use std::collections::BTreeSet; use std::rc::Rc; +use super::CandidateForMemoGroup; use crate::ir::{OperatorNode, QueryRoot}; /// A group, by index in the memo's groups. @@ -26,6 +28,8 @@ pub struct MemoGroup { /// maintain it; later stages read it through /// [`Self::reached_by_queries`]. pub queries: Vec, + /// Its candidates: empty after Stage 0, filled by Stage 1. + pub candidates: Vec, } impl MemoGroup { @@ -40,57 +44,6 @@ impl MemoGroup { } } -/// Stage 0's output, Pass 1's input: the workload's memo groups, one per -/// distinct logical sub-expression, without implementation choices or -/// solver state. Stage 1 builds it by hash-consing the query roots. -#[derive(Debug, Clone)] -pub struct DagOfLogicalMemoGroups { - pub groups: Vec, - /// Each query's root over groups, by index in the workload: its - /// operator's group, or its scalar expression with every operator - /// reference replaced by that operator's group. - pub query_roots: Vec>, - /// Each node of the original workload, by address, to its group. - group_of: HashMap<*const OperatorNode, MemoGroupId>, -} - -impl DagOfLogicalMemoGroups { - /// The memo groups `groups` (children before parents), each query's - /// root over them, and `group_of`: each node of the original workload, - /// by address, to its group. - pub fn new( - groups: Vec, - query_roots: Vec>, - group_of: HashMap<*const OperatorNode, MemoGroupId>, - ) -> Self { - Self { - groups, - query_roots, - group_of, - } - } - - /// Group of a node from the original workload, not a rewritten candidate. - pub fn group_for_node(&self, node: &Rc) -> Option { - self.group_of.get(&Rc::as_ptr(node)).copied() - } - - /// Each query's root over the groups' logical nodes, in workload order: - /// a group several queries read is one node. - pub fn roots_over_groups(&self) -> Vec { - let node = |g: &MemoGroupId| Rc::clone(&self.groups[g.0 as usize].logical_node); - self.query_roots - .iter() - .map(|root| match root { - QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))) - } - }) - .collect() - } -} - /// The groups some query's root reads, ascending: the groups of /// `query_roots` that are a query's result or a scalar query's operand. pub fn root_groups(query_roots: &[QueryRoot]) -> BTreeSet { diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 109ce200e..b106be284 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -1,7 +1,6 @@ -//! The Stage 0 → Stage 1 boundary: the workload's memo groups -//! ([`DagOfLogicalMemoGroups`]). Inside Stage 1, Pass 1 → Pass 2: the memo -//! groups with their Pass 1 candidates -//! ([`PerComputationLogicalMemoCandidates`]). The Stage 1 → Stage 2 boundary: the memo of logical groups and their +//! The Stage 0 → Stage 1 boundary: the workload's memo groups, without +//! candidates ([`Memo`]). Inside Stage 1, Pass 1 → Pass 2: the memo with +//! its Pass 1 candidates. The Stage 1 → Stage 2 boundary: the memo of logical groups and their //! implementation candidates that Stage 1 (Pass 1 and Pass 2) produces and //! Stage 2 implements (#1047, #1054). The Stage 2 → Stage 3 boundary: the //! physical candidates Stage 2 builds and Stage 3 selects from. Stage @@ -16,20 +15,20 @@ mod physical; mod pruning; mod realizer; -pub use candidates::{CandidateForMemoGroup, Memo, OriginOfCandidateForMemoGroup}; -pub use logical_memo_candidates::LogicalASAPMemoCandidates; -pub use memo::{queries_rooted_at, root_groups, DagOfLogicalMemoGroups, MemoGroup, MemoGroupId}; +pub use candidates::{ + parameter_list, satisfies_logical, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, ErrorPropagation, InvalidLogicalProperties, LogicalProperties, + LogicalRequirement, Memo, OriginOfCandidateForMemoGroup, WindowLayout, +}; +pub use logical_memo_candidates::{LogicalASAPMemoCandidates, LogicalEdge}; +pub use memo::{queries_rooted_at, root_groups, MemoGroup, MemoGroupId}; pub use per_computation_logical_memo_candidates::{ - ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, PerComputationLogicalMemoCandidates, + ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, }; pub use physical::{ - output_references, CandidateForMemoGroupId, DagOfSelectedCandidates, - OperatorNodeOfCandidateForMemoGroup, OperatorNodeOfCandidateForMemoGroupId, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - ProducerCandidates, ProducerSetInterner, ReaderConstraint, SelectedEdge, SelectedNodeId, + output_references, satisfies_physical, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, + SelectedEdge, SelectedNodeId, }; -#[cfg(any(test, feature = "test-support"))] -pub use physical::{producer_storage_metrics, ProducerStorageMetrics}; pub use pruning::{PruneReason, PrunedCandidate}; pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/per_computation_logical_memo_candidates.rs b/crates/types/src/stages/per_computation_logical_memo_candidates.rs index 307f4bdc4..137360946 100644 --- a/crates/types/src/stages/per_computation_logical_memo_candidates.rs +++ b/crates/types/src/stages/per_computation_logical_memo_candidates.rs @@ -1,27 +1,11 @@ -//! Stage 1, Pass 1's output, which Pass 2 extends (the memo groups with -//! their Pass 1 candidates), and the computation-target types Pass 1 builds -//! those candidates from. +//! The computation-target types Stage 1, Pass 1 builds its candidates from. use std::rc::Rc; -use super::{Memo, MemoGroupId, PrunedCandidate}; +use super::MemoGroupId; use crate::ir::schema::{ExactKind, SketchAlgorithm}; use crate::ir::{OperatorNode, Predicate}; -/// Stage 1, Pass 1's output: the workload's memo groups, each with its -/// Pass 1 candidates, and the candidates Pass 1 pruned. It is also Pass 2's -/// input. -#[derive(Debug, Clone)] -pub struct PerComputationLogicalMemoCandidates { - /// The groups moved from the [`DagOfLogicalMemoGroups`](super::DagOfLogicalMemoGroups), - /// its query roots, and every group's candidates (origin - /// [`Pass1`](super::OriginOfCandidateForMemoGroup::Pass1)). - pub memo: Memo, - /// The ways to compute a target's measure that Pass 1 pruned, each on - /// its target's aggregate group, in pruning order. - pub pruned_candidates: Vec>, -} - /// How one Pass 1 target is computed: exact or a summary. /// /// Each target has exactly one exact option: `ExactAccumulator` when the @@ -84,8 +68,6 @@ pub struct ComputationTarget { /// Index into the aggregate's measures. pub measure_index: usize, /// The measure's unranked ways to compute it, the exact option first. - /// The ones Pass 1 pruned are in - /// [`PerComputationLogicalMemoCandidates::pruned_candidates`]. pub ways_to_compute: Vec, /// The measure's own filter; only rows that pass it update the state /// (`SummaryAgg.filter`). @@ -114,10 +96,8 @@ pub struct FusedComputation { /// computations absorb. None of them is computed when this is picked. pub absorbed_targets: Vec, /// How the fused computation is computed: exact or a summary. Empty - /// when Pass 1 pruned every one (today only for - /// [`PruneReason::UnsupportedByDeployment`](super::PruneReason::UnsupportedByDeployment), - /// recorded in [`PerComputationLogicalMemoCandidates::pruned_candidates`]); the fusion is - /// then not offered. + /// when Pass 1 pruned every one (today only for a family the deployment + /// does not support); the fusion is then not offered. pub ways_to_compute: Vec, } diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs index c4537eaa4..69f6bc760 100644 --- a/crates/types/src/stages/physical/graph.rs +++ b/crates/types/src/stages/physical/graph.rs @@ -1,72 +1,12 @@ //! Selections over Stage 2's physical candidates, and their structural //! validation. use super::memo_candidates::PhysicalASAPMemoCandidates; -use super::producers::ProducerCandidates; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; /// Index into [`PhysicalASAPMemoCandidates::candidates`](super::PhysicalASAPMemoCandidates::candidates). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct PhysicalCandidateId(pub usize); -/// A rule on which output references may read a candidate, applied to each -/// node of a [`DagOfSelectedCandidates`] that runs it: the output references -/// that reach the node are its readers. Output references -/// index the group references of the query roots -/// ([`output_references`](super::output_references)); several references of -/// one query stay distinct. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum ReaderConstraint { - /// Each node is read by at least `minimum` queries. Each bucket of - /// `query_outputs` is one query's output references; together they - /// partition every output exactly once. Empty buckets represent - /// literal-only queries. - MinimumQueryReaders { - candidate: PhysicalCandidateId, - query_outputs: Vec>, - minimum: usize, - }, - /// A non-summary query materialization: each node produces an - /// output reference, or has an immediate selected consumer reached by - /// at most one query (`query_outputs` as in `MinimumQueryReaders`). - HighestSharedQueryMaterialization { - candidate: PhysicalCandidateId, - query_outputs: Vec>, - }, - /// Only `outputs` may read a node. - AllowedOutputs { - candidate: PhysicalCandidateId, - outputs: Vec, - }, - /// No node is read by both `left` and `right`. - IncompatibleOutputs { - candidate: PhysicalCandidateId, - left: usize, - right: usize, - }, -} - -impl ReaderConstraint { - /// The candidate it constrains. - pub fn candidate(&self) -> PhysicalCandidateId { - match self { - Self::MinimumQueryReaders { candidate, .. } - | Self::HighestSharedQueryMaterialization { candidate, .. } - | Self::AllowedOutputs { candidate, .. } - | Self::IncompatibleOutputs { candidate, .. } => *candidate, - } - } - - /// The candidate it constrains, to renumber. - pub fn candidate_mut(&mut self) -> &mut PhysicalCandidateId { - match self { - Self::MinimumQueryReaders { candidate, .. } - | Self::HighestSharedQueryMaterialization { candidate, .. } - | Self::AllowedOutputs { candidate, .. } - | Self::IncompatibleOutputs { candidate, .. } => candidate, - } - } -} - /// Index into [`DagOfSelectedCandidates::candidate_of_node`]: one run of a /// candidate. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -78,9 +18,9 @@ pub struct SelectedNodeId(pub usize); pub struct SelectedEdge { pub producer: SelectedNodeId, pub consumer: SelectedNodeId, - /// Index in the consumer's inputs (its candidate's - /// `producer_candidates_per_input`). - pub input: usize, + /// Index in the consumer's inputs: its logical candidate's + /// [`Operator::children`](crate::ir::operator::Operator::children). + pub input_index_of_consumer: usize, } /// The selected plan as a DAG with one node per run of a candidate: what @@ -102,47 +42,58 @@ pub struct DagOfSelectedCandidates { impl DagOfSelectedCandidates { /// Structural check against `physical`, instead of trusting rounded /// solver values: every node runs an existing candidate, every input of - /// every node has exactly one edge from a node of an allowed producer - /// candidate, the graph is acyclic, every node is read by some output - /// reference, each output reference comes from one of its - /// `query_output_candidates`, and every node keeps its candidate's - /// reader constraints. Costs and resource budgets are Stage 3's to - /// check. + /// every node has exactly one edge from a node of a producer on one of + /// its admissible edges, the graph is acyclic, every node is reached by + /// some output reference, each output reference comes from a candidate + /// of its group. Costs and accuracy are Stage 3's to check. pub fn validate(&self, physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let query_output_candidates: Vec> = (0..physical.output_reference_count()) - .map(|output| physical.query_output_candidates(output).collect()) + let memo = &physical.logical.memo; + let output_groups: Vec<_> = super::output_references(&physical.query_roots) + .into_iter() + .map(|(_, group)| group) + .collect(); + let query_output_candidates: Vec> = output_groups + .iter() + .map(|&group| { + (0..physical.candidates.len()) + .map(PhysicalCandidateId) + .filter(|id| physical.candidates[id.0].logical.group == group) + .collect() + }) .collect(); let query_output_candidates: Vec<&[_]> = query_output_candidates.iter().map(Vec::as_slice).collect(); + let edges: Vec<_> = physical.edges().collect(); self.validate_against( physical.candidates.len(), |candidate| { - physical.candidates[candidate.0] - .producer_candidates_per_input + memo.candidate(physical.candidates[candidate.0].logical) + .expression + .operator + .children() .len() }, - |candidate, input| { - physical.candidates[candidate.0] - .producer_candidates_per_input - .get(input) + |candidate, input_index_of_consumer, producer| { + physical + .admissible_edges_into(candidate, input_index_of_consumer) + .iter() + .any(|edge| edges[edge.0].1.producer == producer) }, &query_output_candidates, - &physical.reader_constraints, ) } /// [`Self::validate`] over a catalog given by its parts: /// `candidate_count` candidates; `input_count(c)` inputs of candidate - /// `c`; `producers_of_input(c, i)` the candidates that may produce its - /// input `i` (`None` past its last input); `query_output_candidates[r]` - /// the candidates that may produce output reference `r`. - fn validate_against<'a>( + /// `c`; `admissible(c, i, u)` whether an admissible edge `(u, c, i)` + /// exists; `query_output_candidates[r]` the candidates that may produce + /// output reference `r`. + fn validate_against( &self, candidate_count: usize, input_count: impl Fn(PhysicalCandidateId) -> usize, - producers_of_input: impl Fn(PhysicalCandidateId, usize) -> Option<&'a ProducerCandidates>, + admissible: impl Fn(PhysicalCandidateId, usize, PhysicalCandidateId) -> bool, query_output_candidates: &[&[PhysicalCandidateId]], - reader_constraints: &[ReaderConstraint], ) -> Result<(), String> { let invalid = || "decoded candidate selection violates dependency or root constraints".to_string(); @@ -176,15 +127,15 @@ impl DagOfSelectedCandidates { if producer >= count || consumer >= count { return Err(invalid()); } - let Some(producer_candidates) = - producers_of_input(self.candidate_of_node[consumer], edge.input) - else { + let Some(target) = producers[consumer].get_mut(edge.input_index_of_consumer) else { return Err(invalid()); }; - let Some(target) = producers[consumer].get_mut(edge.input) else { - return Err(invalid()); - }; - if target.is_some() || !producer_candidates.contains(&self.candidate_of_node[producer]) + if target.is_some() + || !admissible( + self.candidate_of_node[consumer], + edge.input_index_of_consumer, + self.candidate_of_node[producer], + ) { return Err(invalid()); } @@ -228,54 +179,6 @@ impl DagOfSelectedCandidates { if (0..count).any(|node| !output_reach.iter().any(|reach| reach.contains(&node))) { return Err(invalid()); } - let queries_reaching = |node: usize, query_outputs: &[Vec]| { - query_outputs - .iter() - .filter(|outputs| { - outputs - .iter() - .any(|&output| output_reach[output].contains(&node)) - }) - .count() - }; - let output_nodes: HashSet = self - .node_of_output_reference - .iter() - .map(|node| node.0) - .collect(); - let mut nodes_of_candidate: HashMap> = HashMap::new(); - for (node, candidate) in self.candidate_of_node.iter().enumerate() { - nodes_of_candidate.entry(*candidate).or_default().push(node); - } - for constraint in reader_constraints { - let Some(nodes) = nodes_of_candidate.get(&constraint.candidate()) else { - continue; - }; - let violated = nodes.iter().any(|&node| match constraint { - ReaderConstraint::MinimumQueryReaders { - query_outputs, - minimum, - .. - } => queries_reaching(node, query_outputs) < *minimum, - ReaderConstraint::HighestSharedQueryMaterialization { query_outputs, .. } => { - !output_nodes.contains(&node) - && !consumers[node] - .iter() - .any(|&consumer| queries_reaching(consumer, query_outputs) <= 1) - } - ReaderConstraint::AllowedOutputs { outputs, .. } => { - (0..output_reach.len()).any(|output| { - !outputs.contains(&output) && output_reach[output].contains(&node) - }) - } - ReaderConstraint::IncompatibleOutputs { left, right, .. } => { - output_reach[*left].contains(&node) && output_reach[*right].contains(&node) - } - }); - if violated { - return Err("decoded selection violates selected-reader compatibility".into()); - } - } Ok(()) } } diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 1d500fc27..213115d2a 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -1,131 +1,140 @@ -//! Stage 2's output: every operator node of every implementation candidate, -//! each with the timings and materializations Stage 2 allows, and which -//! candidates may produce each input. Stage 3 selects among them; Stage 2 -//! prices nothing. -use std::rc::Rc; +//! Stage 2's output: each logical candidate's physical alternatives +//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell, \sigma)$, and +//! the physical edges $\mathcal E$ with $\alpha^{\log}$ and +//! $\alpha^{\mathrm{phys}}$ (#1200, #1201). Stage 3 selects among them; +//! Stage 2 prices nothing. -use super::graph::{PhysicalCandidateId, ReaderConstraint}; -use super::producers::ProducerCandidates; -use crate::ir::properties::ExecutionTiming; -use crate::ir::{OperatorNode, QueryRoot}; -use crate::physical::Materialization; -use crate::stages::{MemoGroupId, PrunedCandidate}; +use std::collections::HashMap; -/// One implementation candidate: index `index` in group `group`'s -/// candidates ([`Memo::candidates`](crate::stages::Memo::candidates)). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct CandidateForMemoGroupId { - pub group: MemoGroupId, - pub index: usize, -} +use super::graph::PhysicalCandidateId; +use crate::ir::QueryRoot; +use crate::physical::{EvaluationSchedule, PhysicalProperties}; +use crate::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PrunedCandidate, +}; -/// Index into -/// [`PhysicalASAPMemoCandidates::operator_nodes_of_candidates_for_memo_groups`]. +/// Index into [`PhysicalASAPMemoCandidates::edges`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct OperatorNodeOfCandidateForMemoGroupId(pub usize); +pub struct PhysicalEdgeId(pub usize); -/// One operator node of one or more candidates for a memo group, before -/// Stage 2 chose its timing and materialization. -#[derive(Debug, Clone)] -pub struct OperatorNodeOfCandidateForMemoGroup { - /// Every candidate of the group this node can implement, ascending: - /// identical terminal readers of different candidates are factored into - /// one node, and the selected producers decide which one it implements. - /// A private node belongs to the one candidate it was lowered from. - pub candidates_for_memo_group: Vec, - /// Whether it is its candidates' root; false for an implementation's - /// private nodes. - pub is_implementation_root: bool, - /// How many equal shares of a query's accuracy budget this node's sketch sizes meet - /// (`CandidateForMemoGroup::accuracy_budget_parts`). For a factored node, the largest among its - /// candidates: equal nodes have equal sizes. 1 for an exact node. - pub accuracy_budget_parts: usize, - /// The operator node, with Stage 2's physical implementation; its - /// children are the expected inputs, which the selected producers replace. - pub physical_node: Rc, +/// One physical alternative of a logical candidate: what Stage 3 selects. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalCandidate { + /// $e_v$ and $p^{\log}_v$. + pub logical: CandidateForMemoGroupId, + /// $p^{\mathrm{phys}}_v$. + pub properties: PhysicalProperties, } -impl OperatorNodeOfCandidateForMemoGroup { - /// The memo group whose candidates this node implements. - /// `candidates_for_memo_group` is never empty, and all its entries are - /// candidates of this one group. - pub fn group(&self) -> MemoGroupId { - let group = self.candidates_for_memo_group[0].group; - debug_assert!(self - .candidates_for_memo_group - .iter() - .all(|candidate| candidate.group == group)); - group +impl PhysicalCandidate { + /// Whether this producer delivers on `times` (the concepts doc's + /// alignment table, section 2). Reads the grid width from the candidate's + /// logical window layout in `memo`. + pub fn delivers_on(&self, memo: &Memo, times: &EvaluationSchedule) -> bool { + let _ = (memo, times); + todo!("#1207 PR 1: the alignment table of section 2") } } -/// One operator node with one timing and materialization: what Stage 3 -/// selects. A selection may run it more than once, each run a node of the -/// [`DagOfSelectedCandidates`](super::DagOfSelectedCandidates). -#[derive(Debug, Clone)] -pub struct PhysicalCandidate { - /// The operator node it runs. - pub operator_node: OperatorNodeOfCandidateForMemoGroupId, - /// When it runs. A node that is not materialized under an ingestion-time - /// build still runs at ingestion time, so both are kept. - pub timing: ExecutionTiming, - /// Whether and where its result is kept. - pub materialization: Materialization, - /// Per input of its operator node (in `map_children` order), the - /// candidates that may produce it. Depends on its timing. - pub producer_candidates_per_input: Vec, - /// Whether it may produce a query's output: an implementation root at - /// query time whose output matches its group's logical node, in a group - /// some query's root reads. - pub may_produce_query_output: bool, - /// One entry per workload query: whether this tumbling-window chain may be kept - /// across evaluations when that query is its only reader. Filled only - /// for a `MatAtQueryKeptAcrossEvaluations(Memory)` tumbling-window chain in a - /// deployment that keeps query-time results; empty otherwise. - pub may_keep_across_evaluations_per_sole_reader_query: Vec, +/// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime (the concepts doc's +/// table, section 2), and, for a materialized consumer, that the producer +/// delivers on the consumer's own schedule. Takes candidates rather than +/// bare properties because a tumbling grid's width lives in the logical +/// window layout. +pub fn satisfies_physical( + memo: &Memo, + producer: &PhysicalCandidate, + consumer: &PhysicalCandidate, +) -> bool { + let _ = (memo, producer, consumer); + todo!("#1207 PR 1: the timing x lifetime table and the alignment table of section 2") +} + +/// $(u, c, i)$ over physical candidates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PhysicalEdge { + pub producer: PhysicalCandidateId, + pub consumer: PhysicalCandidateId, + pub input_index_of_consumer: usize, + /// $\alpha^{\log}_{u,c,i}$. + pub satisfies_logical_requirement: bool, + /// $\alpha^{\mathrm{phys}}_{u,c,i}$. + pub satisfies_physical_requirement: bool, } /// A physical candidate Stage 2 pruned, named by what it would have been: /// candidates are renumbered after pruning, so a pruned one has no -/// [`PhysicalCandidateId`]. Unique among a [`PhysicalASAPMemoCandidates`]' -/// `pruned_candidates`: an operator node has at most one candidate per -/// timing and materialization. +/// [`PhysicalCandidateId`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalCandidateKey { - /// Its operator node: an index into the output's - /// `operator_nodes_of_candidates_for_memo_groups`, which pruning keeps - /// whole. - pub operator_node: OperatorNodeOfCandidateForMemoGroupId, - pub timing: ExecutionTiming, - pub materialization: Materialization, + pub logical: CandidateForMemoGroupId, + pub properties: PhysicalProperties, } /// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] pub struct PhysicalASAPMemoCandidates { - /// One per operator node of an implementation, before Stage 2 chose its - /// timing and materialization. - pub operator_nodes_of_candidates_for_memo_groups: Vec, - /// One per (operator node, timing, materialization): what Stage 3 - /// selects. Indexed by [`PhysicalCandidateId`]. + /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, + /// $r^{\log}_{c,i}$, and the logical edges. + pub logical: LogicalASAPMemoCandidates, + /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. pub candidates: Vec, /// Each query's root over groups, by index in the workload, passed - /// through from Pass 2. Its group references, in order, are the output - /// references of [`Self::query_output_candidates`]. + /// through from Pass 2. pub query_roots: Vec>, - /// Legality that depends on which readers are selected. - pub reader_constraints: Vec, - /// The candidates Stage 2 pruned, each on its operator node's memo - /// group, in pruning order: configurations it does not generate, then - /// candidates no buildable input, query output or reader count can use. - /// Empty when built by [`Self::new`]. + /// The candidates Stage 2 pruned, in pruning order. pub pruned_candidates: Vec>, - /// By group index: the candidates whose operator node is an - /// implementation root of that group, ascending. - implementation_roots_of_group: Vec>, - /// Each output reference's group: [`output_references`] of - /// `query_roots`, built once. - output_reference_groups: Vec, + /// $\mathcal E$, grouped by (consumer, input); indexed by + /// [`PhysicalEdgeId`]. + edges: Vec, + /// $P_{c,i}$ after $\alpha$: the admissible edges into each (consumer, input). + admissible_edges_into_input: HashMap<(PhysicalCandidateId, usize), Vec>, + /// The admissible edges out of each candidate, by [`PhysicalCandidateId`]. + admissible_edges_out_of_candidate: Vec>, +} + +impl PhysicalASAPMemoCandidates { + /// Stage 2's output over `candidates` and `edges` (grouped by + /// (consumer, input)), with the admissible-edge indexes built once. + pub fn new( + logical: LogicalASAPMemoCandidates, + candidates: Vec, + query_roots: Vec>, + edges: Vec, + ) -> Self { + let _ = (logical, candidates, query_roots, edges); + todo!("#1207 PR 5: index the admissible edges") + } + + /// $\mathcal E$: every edge, in (consumer, input) order. + pub fn edges(&self) -> impl Iterator + '_ { + self.edges + .iter() + .enumerate() + .map(|(id, edge)| (PhysicalEdgeId(id), *edge)) + } + + /// $P^{\mathrm{adm}}_{c,i}$: the admissible edges into input + /// `input_index_of_consumer` of `consumer` + /// ($\alpha^{\log} = \alpha^{\mathrm{phys}} = 1$). + pub fn admissible_edges_into( + &self, + consumer: PhysicalCandidateId, + input_index_of_consumer: usize, + ) -> &[PhysicalEdgeId] { + let _ = ( + consumer, + input_index_of_consumer, + &self.admissible_edges_into_input, + ); + todo!("#1207 PR 5") + } + + /// The admissible edges out of `producer`: every consumer that may share it. + pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { + let _ = (producer, &self.admissible_edges_out_of_candidate); + todo!("#1207 PR 5") + } } /// The output references of `query_roots`: each operator reference, in @@ -145,82 +154,3 @@ pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, } references } - -impl PhysicalASAPMemoCandidates { - /// Builds the private per-group index over `candidates`. Stage 3's - /// component search uses it to build one smaller catalog per component. - pub fn new( - operator_nodes_of_candidates_for_memo_groups: Vec, - candidates: Vec, - query_roots: Vec>, - reader_constraints: Vec, - ) -> Self { - let mut implementation_roots_of_group: Vec> = Vec::new(); - for (id, candidate) in candidates.iter().enumerate() { - let node = &operator_nodes_of_candidates_for_memo_groups[candidate.operator_node.0]; - if node.is_implementation_root { - let group = node.group().0 as usize; - if implementation_roots_of_group.len() <= group { - implementation_roots_of_group.resize(group + 1, Vec::new()); - } - implementation_roots_of_group[group].push(PhysicalCandidateId(id)); - } - } - let output_reference_groups = output_references(&query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - Self { - operator_nodes_of_candidates_for_memo_groups, - candidates, - query_roots, - reader_constraints, - pruned_candidates: Vec::new(), - implementation_roots_of_group, - output_reference_groups, - } - } - - /// The implementation roots of `group`, ascending. - pub fn implementation_roots(&self, group: MemoGroupId) -> &[PhysicalCandidateId] { - self.implementation_roots_of_group - .get(group.0 as usize) - .map_or(&[], Vec::as_slice) - } - - /// The candidates that may produce output reference `output_reference` - /// (each group reference of `query_roots`, in order; a scalar query has - /// several): its group's roots with `may_produce_query_output`. - pub fn query_output_candidates( - &self, - output_reference: usize, - ) -> impl Iterator + '_ { - let roots = self - .output_reference_groups - .get(output_reference) - .map_or(&[][..], |&group| self.implementation_roots(group)); - roots - .iter() - .copied() - .filter(|id| self.candidates[id.0].may_produce_query_output) - } - - /// The number of output references: group references of `query_roots`. - pub fn output_reference_count(&self) -> usize { - self.output_reference_groups.len() - } - - /// The operator node candidate `candidate` runs. - pub fn physical_node(&self, candidate: PhysicalCandidateId) -> &Rc { - &self.operator_nodes_of_candidates_for_memo_groups - [self.candidates[candidate.0].operator_node.0] - .physical_node - } - - /// The memo group whose candidates candidate `candidate` implements. - pub fn memo_group_of(&self, candidate: PhysicalCandidateId) -> MemoGroupId { - self.operator_nodes_of_candidates_for_memo_groups - [self.candidates[candidate.0].operator_node.0] - .group() - } -} diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 7f029925a..3a49d0eb1 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -1,18 +1,11 @@ -//! The Stage 2 → Stage 3 boundary: the physical candidates Stage 2 builds -//! from the memo and Stage 3 selects from (#1047, #1054). +//! The Stage 2 → Stage 3 boundary: the physical candidates and edges Stage 2 +//! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201). mod graph; mod memo_candidates; -mod producers; -pub use graph::{ - DagOfSelectedCandidates, PhysicalCandidateId, ReaderConstraint, SelectedEdge, SelectedNodeId, -}; +pub use graph::{DagOfSelectedCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId}; pub use memo_candidates::{ - output_references, CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateKey, + output_references, satisfies_physical, PhysicalASAPMemoCandidates, PhysicalCandidate, + PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, }; -#[cfg(any(test, feature = "test-support"))] -pub use producers::{producer_storage_metrics, ProducerStorageMetrics}; -pub use producers::{ProducerCandidates, ProducerSetInterner}; diff --git a/crates/types/src/stages/physical/producers.rs b/crates/types/src/stages/physical/producers.rs deleted file mode 100644 index 0e7547434..000000000 --- a/crates/types/src/stages/physical/producers.rs +++ /dev/null @@ -1,206 +0,0 @@ -//! Shared storage for ordered compatible producers. Sharing this storage never -//! ties together the independent selections of different consumer input slots. -use super::graph::PhysicalCandidateId; -use std::collections::HashSet; -use std::ops::{Deref, DerefMut}; -use std::rc::Rc; - -#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] -pub struct ProducerCandidates(Rc>); - -impl From> for ProducerCandidates { - fn from(values: Vec) -> Self { - Self(Rc::new(values)) - } -} - -impl FromIterator for ProducerCandidates { - fn from_iter>(values: T) -> Self { - Self::from(values.into_iter().collect::>()) - } -} - -impl Deref for ProducerCandidates { - type Target = Vec; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for ProducerCandidates { - fn deref_mut(&mut self) -> &mut Self::Target { - Rc::make_mut(&mut self.0) - } -} - -impl<'a> IntoIterator for &'a ProducerCandidates { - type Item = &'a PhysicalCandidateId; - type IntoIter = std::slice::Iter<'a, PhysicalCandidateId>; - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> IntoIterator for &'a mut ProducerCandidates { - type Item = &'a mut PhysicalCandidateId; - type IntoIter = std::slice::IterMut<'a, PhysicalCandidateId>; - fn into_iter(self) -> Self::IntoIter { - self.iter_mut() - } -} - -impl IntoIterator for ProducerCandidates { - type Item = PhysicalCandidateId; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - Rc::unwrap_or_clone(self.0).into_iter() - } -} - -/// Exact ordered-content interning. No sorting, deduplication, or compatibility -/// inference occurs here; malformed input remains visible to graph validation. -#[derive(Default)] -pub struct ProducerSetInterner { - sets: HashSet, -} - -impl ProducerSetInterner { - pub fn intern(&mut self, values: impl Into) -> ProducerCandidates { - let values = values.into(); - if let Some(existing) = self.sets.get(&values) { - return existing.clone(); - } - self.sets.insert(values.clone()); - values - } -} - -/// Count physical producer-list storage separately from logical connections. -/// Distinct input positions remain distinct even when their storage is shared. -#[cfg(any(test, feature = "test-support"))] -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct ProducerStorageMetrics { - pub logical_connections: usize, - pub unique_producer_sets: usize, - pub stored_producer_ids: usize, -} - -#[cfg(any(test, feature = "test-support"))] -pub fn producer_storage_metrics( - physical: &super::PhysicalASAPMemoCandidates, -) -> ProducerStorageMetrics { - let mut allocations = HashSet::new(); - let mut metrics = ProducerStorageMetrics::default(); - for producer_candidates in physical - .candidates - .iter() - .flat_map(|candidate| &candidate.producer_candidates_per_input) - { - metrics.logical_connections += producer_candidates.len(); - if allocations.insert(Rc::as_ptr(&producer_candidates.0)) { - metrics.unique_producer_sets += 1; - metrics.stored_producer_ids += producer_candidates.len(); - } - } - metrics -} - -#[cfg(test)] -mod tests { - use super::*; - - // Two slots can choose different producers even when their compatible lists - // share one allocation; storage compression adds no selection equality. - #[test] - fn shared_producer_storage_keeps_input_choices_independent() { - use super::super::graph::{DagOfSelectedCandidates, SelectedEdge, SelectedNodeId}; - use super::super::memo_candidates::{ - CandidateForMemoGroupId, OperatorNodeOfCandidateForMemoGroup, - OperatorNodeOfCandidateForMemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, - }; - use crate::ir::properties::ExecutionTiming; - use crate::ir::QueryRoot; - use crate::physical::Materialization; - use crate::stages::MemoGroupId; - let mut interner = ProducerSetInterner::default(); - let choices = interner.intern(vec![PhysicalCandidateId(0), PhysicalCandidateId(1)]); - use crate::ir::scalar::expr_ir::ScalarValue; - use crate::ir::schema::{DataType, Field, Schema}; - use crate::ir::{NonASAPOp, Operator, OperatorNode, ScalarExpr}; - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(1.0))]], - schema: Schema::lifted(vec![Field::plain("value", DataType::Float64, false)], None), - })) - .unwrap(); - let physical = PhysicalASAPMemoCandidates::new( - (0..3) - .map(|group| OperatorNodeOfCandidateForMemoGroup { - candidates_for_memo_group: vec![CandidateForMemoGroupId { - group: MemoGroupId(group), - index: 0, - }], - is_implementation_root: true, - accuracy_budget_parts: 1, - physical_node: Rc::clone(&node), - }) - .collect(), - (0..3) - .map(|id| PhysicalCandidate { - operator_node: OperatorNodeOfCandidateForMemoGroupId(id), - timing: ExecutionTiming::QueryTime, - materialization: Materialization::NotMaterialized, - producer_candidates_per_input: if id == 2 { - vec![choices.clone(), choices.clone()] - } else { - vec![] - }, - may_produce_query_output: id == 2, - may_keep_across_evaluations_per_sole_reader_query: Vec::new(), - }) - .collect(), - vec![QueryRoot::Operator(MemoGroupId(2))], - Vec::new(), - ); - let selected = DagOfSelectedCandidates { - candidate_of_node: (0..3).map(PhysicalCandidateId).collect(), - edges: (0..2) - .map(|input| SelectedEdge { - producer: SelectedNodeId(input), - consumer: SelectedNodeId(2), - input, - }) - .collect(), - node_of_output_reference: vec![SelectedNodeId(2)], - }; - selected.validate(&physical).unwrap(); - assert_eq!( - producer_storage_metrics(&physical), - ProducerStorageMetrics { - logical_connections: 4, - unique_producer_sets: 1, - stored_producer_ids: 2 - } - ); - } - - // Equal ordered lists share one allocation; mutation of one input detaches - // it and cannot silently change another input or an interner key. - #[test] - fn interned_inputs_detach_on_mutation() { - let mut interner = ProducerSetInterner::default(); - let original = interner.intern(vec![PhysicalCandidateId(2), PhysicalCandidateId(1)]); - let mut changed = interner.intern(vec![PhysicalCandidateId(2), PhysicalCandidateId(1)]); - assert!(Rc::ptr_eq(&original.0, &changed.0)); - changed.retain(|id| id.0 == 1); - assert_eq!( - original.as_slice(), - &[PhysicalCandidateId(2), PhysicalCandidateId(1)] - ); - assert_eq!(changed.as_slice(), &[PhysicalCandidateId(1)]); - assert!(!Rc::ptr_eq(&original.0, &changed.0)); - let repeated = interner.intern(original.clone()); - assert!(Rc::ptr_eq(&original.0, &repeated.0)); - let reversed = interner.intern(vec![PhysicalCandidateId(1), PhysicalCandidateId(2)]); - assert!(!Rc::ptr_eq(&original.0, &reversed.0)); - } -} diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index f857460f4..82984ba3b 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,8 +1,8 @@ //! What a stage removed, and why: one shape for every stage (#1047 P6). //! Each stage removes only what its own knowledge rules out (option C): -//! Pass 1 what the measure or the deployment cannot compute, Stage 2 what -//! cannot be implemented, built or kept. Stage 0 fails with an error and -//! Pass 2 only adds candidates, so neither prunes. Stage 3 prunes before +//! Stage 2 what cannot be implemented, built or kept. Stage 0 fails with an +//! error, Pass 1 records nothing it leaves out (#1205 will show it), and +//! Pass 2 only adds candidates. Stage 3 prunes before //! its selection MILP (dominated and uncostable candidates); its //! rejections of whole plans keep their own cause (`RejectionCause` in plan //! selection). @@ -10,9 +10,9 @@ use super::{MemoGroupId, PhysicalCandidateId}; /// One candidate a stage removed from memo group `group`, and why. `C` is -/// the stage's candidate type: Pass 1's way to compute a measure -/// (`HowToComputeAggregateMeasure`), Stage 2's -/// [`PhysicalCandidateKey`](super::PhysicalCandidateKey). +/// the stage's candidate type: Stage 2's +/// [`PhysicalCandidateKey`](super::PhysicalCandidateKey), Stage 3's +/// [`PhysicalCandidateId`]. #[derive(Debug, Clone, PartialEq)] pub struct PrunedCandidate { pub group: MemoGroupId, @@ -20,54 +20,11 @@ pub struct PrunedCandidate { pub reason: PruneReason, } -/// Why a stage removed a candidate. The first seven are Pass 1's: the -/// candidate cannot compute its measure, the deployment cannot run it, or -/// no query that reads it can meet its accuracy target with it. -/// Then Stage 2's: the physical candidate cannot be built, used or -/// kept. The last two are Stage 3's, before its selection MILP. +/// Why a stage removed a candidate: first Stage 2's (the physical candidate +/// cannot be built, used or kept), then Stage 3's, before its selection +/// MILP. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PruneReason { - /// Pass 1: the measure requires an exact answer and the summary is - /// approximate. - RequiresExact, - /// Pass 1: `count` counts rows, so a possibly-null `avg` input changes - /// the result. - NullableAvgInput, - /// Pass 1: frequency sketches do not implement SQL row-count updates. - /// The exact counter is kept; a fused raw-row top-k has its own keyed - /// update contract. - NoRelationalCountSketchRealization, - /// Pass 1: the deployment's executor cannot build the summary's family - /// or answer the measure's readout from it (`summary_supported` in - /// Stage 1). - UnsupportedByDeployment, - /// Pass 1: the quantile's φ is outside [0, 1], where PromQL answers -Inf - /// or +Inf: no sketch estimates that, and the executor's sketches refuse - /// such a φ, so only the exact answer remains. - QuantileOutsideUnitInterval, - /// Pass 1: a PromQL top-k's heap (Count-Min or CountSketch with a heap) - /// over its own input, an instant vector with one row per ranked series - /// or group: the heap reads every row and adds nothing to it, so an - /// exact top-k over the same rows, which keeps only `k` of them, is - /// exact and no costlier. Only the whole-expression heap over the inner - /// aggregate's input, where an item's rows repeat, is offered (Pass 1's - /// fused computation). - TopKHeapOverOneRowPerItem, - /// Pass 1: for every query that reads its memo group, the query's - /// output has no error bound that can meet the query's accuracy target - /// at any summary size: guarantee propagation has no rule above it - /// (e.g. a rank error through `avg`, or a value `HAVING` above an - /// estimate), its bound stays unknown once the data workload's - /// declarations are applied (e.g. a relative error through `avg` over - /// samples not declared non-negative), its bound is in a metric the - /// target does not accept (the deployment's accuracy model decides), or - /// its failure probability is per value and the target's is over a - /// whole result whose number of values the data does not bound. Kept - /// when another aggregate over the same source offers the same family, - /// which Pass 2 may build from it. Pruned only when the deployment's - /// accuracy model and evidence cannot accept an output without its - /// guarantee. - NoErrorBoundForQueryTarget, /// Stage 2: the executor cannot run its operator node, an exact /// per-entity `Aggregate` (a per-series range reduction over raw data /// samples) with more than one measure, a measure without an exact @@ -86,7 +43,7 @@ pub enum PruneReason { /// candidates, once each candidate's allowed readers are applied. ReachesNoQueryOutput, /// Stage 2: it is shared within one query evaluation - /// (`MatAtQuery`), which needs at least two reading queries, and fewer + /// (`OneEvaluation`), which needs at least two reading queries, and fewer /// queries can reach it. TooFewQueryReaders, /// Stage 3: the candidate it names, in the same memo group with the diff --git a/crates/types/src/stages/realizer.rs b/crates/types/src/stages/realizer.rs index 8c2ff17d6..ab1258f16 100644 --- a/crates/types/src/stages/realizer.rs +++ b/crates/types/src/stages/realizer.rs @@ -38,7 +38,7 @@ impl From<&str> for PhysicalCandidateError { #[derive(Debug, Clone)] pub struct RealizedPlan { /// The selection rebuilt as one physical plan, one operator node per - /// selected node, each with its candidate's timing and materialization. + /// selected node, each with its candidate's timing and lifetime. pub plan: PhysicalWorkloadPlan, /// For each tumbling-window chain of `plan` kept beyond one evaluation /// (maintained at ingestion time, or built at query time and kept @@ -68,7 +68,7 @@ pub struct RealizedPlan { /// recomposed, and every output contract (result kind and schema) met; a /// realized assignment also passes Stage 2's timing and dependency rules /// and the physical DAG validator, with every node keeping its -/// candidate's timing and materialization. +/// candidate's timing and lifetime. /// /// It checks structure and execution legality only. It does not check cost, /// and it does not check accuracy targets: Stage 3 does both. diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index d267ee3eb..8e138c2f2 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -2,19 +2,16 @@ //! crates' tests), exposed under the `test-support` feature so the normal //! public API stays as narrow as its production users need. Each forwards //! to the crate-private function of the same name. -use std::rc::Rc; use crate::ir::materialized_view::RawDataWindow; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; -use crate::ir::physical_export::PhysicalASAPDAGCompilation; use crate::ir::properties::summary_coverage::SummaryCoverage; -use crate::ir::properties::timing::NodeToTimedNodeCache; -use crate::ir::properties::ExecutionDataStateError; + use crate::ir::schema::Schema; use crate::ir::{ASAPOp, NonASAPOp}; use crate::ir::{OperatorNode, SchemaDerivationError}; -use crate::physical::Materialization; +use std::rc::Rc; /// `SummaryCoverage::definition_id`. pub fn definition_id(coverage: &SummaryCoverage) -> String { @@ -34,13 +31,6 @@ pub fn tumbling_window( crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index) } -/// `physical_export::compile_physical_asap_dag_with_node_ids`. -pub fn compile_physical_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root) -} - /// `Operator::output_schema`. pub fn output_schema(operator: &Operator) -> Result { operator.output_schema() @@ -61,20 +51,38 @@ pub fn matches_node(input: &CurrentSeriesInput, node: &OperatorNode) -> bool { input.matches_node(node) } -/// `NodeToTimedNodeCache::timed`. -pub fn timed<'a>( - cache: &'a NodeToTimedNodeCache, - input: &Rc, -) -> Option<&'a Rc> { - cache.timed(input) -} - /// `RawDataWindow::lookback_ms`. pub fn lookback_ms(window: &RawDataWindow) -> u64 { window.lookback_ms() } -/// `Materialization::is_materialized_view`. -pub fn is_materialized_view(materialization: Materialization) -> bool { - materialization.is_materialized_view() +/// `physical_export::physical_asap_dag_at_query_time_with_node_ids`: `roots` +/// exported with every node at query time, not materialized and without θ, +/// with each exported node's operator node by node id. +pub fn physical_asap_dag_at_query_time_with_node_ids( + roots: &[crate::ir::QueryRoot], +) -> Result< + ( + crate::ir::physical_export::PhysicalASAPDAG, + Vec>, + ), + crate::ir::physical_export::PhysicalASAPDAGValidationError, +> { + crate::ir::physical_export::physical_asap_dag_at_query_time_with_node_ids(roots) +} + +/// [`physical_asap_dag_at_query_time_with_node_ids`] of the single root `root`, without +/// the node ids. +pub fn physical_asap_dag_at_query_time( + root: &Rc, +) -> Result< + crate::ir::physical_export::PhysicalASAPDAG, + crate::ir::physical_export::PhysicalASAPDAGValidationError, +> { + Ok( + physical_asap_dag_at_query_time_with_node_ids(&[crate::ir::QueryRoot::Operator( + Rc::clone(root), + )])? + .0, + ) } diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index a50afb0ed..71f33a0e6 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -476,9 +476,7 @@ pub fn with_confirmed_weights_at( let rebuilt = if changed { Rc::new( OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_materialization(node.materialization), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index e95aeb9cf..fa1718e0e 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,7 +2,6 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. -mod materialized_view; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; diff --git a/crates/types/tests/materialized_view.rs b/crates/types/tests/materialized_view.rs deleted file mode 100644 index 0576ca187..000000000 --- a/crates/types/tests/materialized_view.rs +++ /dev/null @@ -1,512 +0,0 @@ -//! A DAG's materialized views and their retention -//! (`asap_types::ir::materialized_view`): two one-minute KLL tumbling windows of `m` -//! kept at ingestion, merged into a view refreshed on first read. - -use std::collections::HashMap; -use std::rc::Rc; -use std::time::Duration; - -use asap_types::ir::materialized_view::{ - InstallableDagError, MaterializedViewRefresh, RefreshWindow, -}; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; -use asap_types::ir::operator::TimeShift; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, -}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, -}; -use asap_types::ir::{ - apply_materialization_timings, apply_materialization_timings_to_roots, ASAPOp, - MaterializationAssignment, NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, QueryRoot, - TimeRangeKind, -}; -use asap_types::physical::{Materialization, Medium}; - -/// A KLL of `m`'s values over `(T - shift - 1 min, T - shift]`. -fn tumbling_window(scan: &Rc, shift_ms: i64) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: Rc::clone(scan), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_secs(60), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: range, - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -/// `m` with columns `ts`, `value` and `job`. -fn scan_m() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -/// The tumbling windows at offsets 0 and 1 min kept at ingestion, their merge kept -/// across evaluations, exported: `(dag, merge's node id)`. -fn tumbling_windows_and_kept_merge() -> (PhysicalASAPDAG, usize) { - let scan = scan_m(); - let tumbling_windows = [tumbling_window(&scan, 0), tumbling_window(&scan, 60_000)]; - let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: tumbling_windows.to_vec(), - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - assignment.set( - &merge, - Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), - ); - let timed = - apply_materialization_timings(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - let merge_id = dag.roots[0]; - (dag, merge_id) -} - -/// A view without `retention_ms` (a DAG before installation) has no -/// materialized views to list. -#[test] -fn a_view_without_retention_is_refused() { - let (dag, _) = tumbling_windows_and_kept_merge(); - assert!(matches!( - dag.materialized_views(), - Err(InstallableDagError::ViewWithoutRetention { .. }) - )); -} - -/// The merge, refreshed on first read for each step of a one-hour range, -/// is kept for its two minutes plus the hour; each tumbling window it reads is kept -/// that long too, since the merge may be refreshed for its oldest step. -/// The tumbling windows read `m` at ingestion, so no raw data is kept for queries. -#[test] -fn retention_propagates_from_a_view_refreshed_on_first_read_to_its_inputs() { - let (mut dag, merge_id) = tumbling_windows_and_kept_merge(); - let spans = HashMap::from([(merge_id, 3_600_000)]); - dag.set_retention(&HashMap::new(), &spans).unwrap(); - let views = dag.materialized_views().unwrap(); - let (merge, tumbling_windows) = views.split_last().unwrap(); - assert_eq!(merge.node_id, merge_id); - assert_eq!(merge.refresh(), MaterializedViewRefresh::OnFirstRead); - assert_eq!(merge.retention_ms(), 120_000 + 3_600_000); - assert_eq!(tumbling_windows.len(), 2); - for tumbling_window in tumbling_windows { - assert_eq!( - tumbling_window.refresh(), - MaterializedViewRefresh::AtIngestion - ); - assert_eq!(tumbling_window.retention_ms(), merge.retention_ms()); - let node = dag - .nodes - .iter() - .find(|n| n.id == tumbling_window.node_id) - .unwrap(); - assert_eq!(node.retention_ms, Some(tumbling_window.retention_ms())); - } - assert_eq!(dag.raw_data_retention().unwrap(), []); -} - -/// A view kept for less than its data reaches back (the older tumbling window, which -/// reaches 2 minutes back, kept for 1 minute) is refused. -#[test] -fn a_view_kept_shorter_than_its_reach_is_refused() { - let (mut dag, _) = tumbling_windows_and_kept_merge(); - let views = dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let older = views - .iter() - .find(|v| { - v.inputs - .raw_data_windows() - .first() - .map(|w| w.end_offset_ms()) - == Some(60_000) - }) - .unwrap() - .node_id; - dag.nodes - .iter_mut() - .find(|n| n.id == older) - .unwrap() - .retention_ms = Some(60_000); - assert_eq!( - dag.materialized_views(), - Err(InstallableDagError::ViewRetentionTooShort { node_id: older }) - ); -} - -/// A quantile sketch of `algorithm` (KLL k = 200 or DDSketch α = 0.01). -fn quantile_sketch(algorithm: SketchAlgorithm) -> FieldDataType { - let params = match algorithm { - SketchAlgorithm::Kll => SketchParams::Kll { - k: 200, - failure_probability: None, - }, - _ => SketchParams::DDSketch { alpha: 0.01 }, - }; - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) -} - -/// `tumbling_window_count` tumbling windows of `tumbling_window_width_ms` of `family` per `job` over `input`. -fn chain( - input: &Rc, - family: FieldDataType, - tumbling_window_width_ms: u64, - tumbling_window_count: u32, -) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(input), - family, - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - tumbling_window_width_ms, - tumbling_window_count, - })) - .unwrap() -} - -/// Three queries over one `Scan` of `m`, each merging a chain kept at -/// ingestion: a KLL of six 10 s tumbling windows, a DDSketch of thirty 10 s tumbling windows, and -/// a KLL of five 60 s tumbling windows. The DAG Stage 3 costs (chains unexpanded) and -/// the installed DAG (chains expanded into tumbling windows, retention set). -fn three_chains_over_one_scan() -> (PhysicalASAPDAG, PhysicalASAPDAG) { - let scan = scan_m(); - let chains = [ - chain(&scan, quantile_sketch(SketchAlgorithm::Kll), 10_000, 6), - chain( - &scan, - quantile_sketch(SketchAlgorithm::DDSketch), - 10_000, - 30, - ), - chain(&scan, quantile_sketch(SketchAlgorithm::Kll), 60_000, 5), - ]; - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = chains - .iter() - .map(|chain| { - assignment.set(chain, Materialization::MatAtIngestion(Medium::Memory)); - QueryRoot::Operator( - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(chain)], - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap(), - ) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let unexpanded = compile_physical_asap_workload(&timed).unwrap(); - let expanded = expand_tumbling_window_chains_in_workload(&timed).unwrap(); - let mut installed = compile_physical_asap_workload(&expanded.roots).unwrap(); - installed - .set_retention(&HashMap::new(), &HashMap::new()) - .unwrap(); - (unexpanded, installed) -} - -/// The width and sketch of the tumbling windows `node` of `dag` builds. -fn tumbling_window_shape(dag: &PhysicalASAPDAG, node: usize) -> (u64, SketchAlgorithm) { - let payload = &dag.nodes.iter().find(|n| n.id == node).unwrap().payload; - let (family, width_ms) = match payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - tumbling_window_width_ms, - .. - }) => (family, *tumbling_window_width_ms), - Operator::ASAP(ASAPOp::SummaryAgg { family, child, .. }) => { - let range = &dag.nodes.iter().find(|n| n.id == *child).unwrap().payload; - let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = range else { - panic!("a tumbling window reads a TimeRange") - }; - (family, range.as_millis() as u64) - } - other => panic!("not a tumbling window: {other:?}"), - }; - let FieldDataType::Sketch(kind, _) = family else { - panic!("a sketch") - }; - (width_ms, *kind.algorithm()) -} - -/// A chain kept at ingestion has its newest tumbling window's refresh window on the -/// DAG Stage 3 costs, and each of its expanded tumbling windows has the same one. -#[test] -fn a_chain_and_each_of_its_tumbling_windows_have_one_refresh_window() { - let (unexpanded, installed) = three_chains_over_one_scan(); - let window_of = |width_ms| RefreshWindow { - width_ms, - phase_ms: 0, - }; - let chains: Vec = unexpanded - .nodes - .iter() - .filter(|node| { - matches!( - node.payload, - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) - }) - .map(|node| node.id) - .collect(); - assert_eq!(chains.len(), 3); - for chain in chains { - let (width_ms, _) = tumbling_window_shape(&unexpanded, chain); - assert_eq!(unexpanded.refresh_window(chain), Some(window_of(width_ms))); - } - let views = installed.materialized_views().unwrap(); - assert_eq!(views.len(), 6 + 30 + 5); - for view in &views { - let (width_ms, _) = tumbling_window_shape(&installed, view.node_id); - assert_eq!( - installed.refresh_window(view.node_id), - Some(window_of(width_ms)) - ); - } -} - -/// The 10 s KLL and 10 s DDSketch tumbling windows are one refresh group, the 60 s -/// KLL tumbling windows another; the merges, at query time, are in none. -#[test] -fn tumbling_windows_of_one_width_form_one_refresh_group() { - let (_, installed) = three_chains_over_one_scan(); - let groups = installed.ingestion_refresh_groups().unwrap(); - let shapes: Vec<_> = groups - .iter() - .map(|group| { - let mut shapes: Vec<_> = group - .views - .iter() - .map(|view| tumbling_window_shape(&installed, view.node_id)) - .collect(); - shapes.sort_by_key(|(width_ms, algorithm)| (*width_ms, format!("{algorithm:?}"))); - shapes.dedup(); - (group.refresh_window, shapes) - }) - .collect(); - assert_eq!( - shapes, - [ - ( - Some(RefreshWindow { - width_ms: 10_000, - phase_ms: 0 - }), - vec![ - (10_000, SketchAlgorithm::DDSketch), - (10_000, SketchAlgorithm::Kll) - ] - ), - ( - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }), - vec![(60_000, SketchAlgorithm::Kll)] - ), - ] - ); - assert_eq!(groups[0].views.len(), 6 + 30); - assert_eq!(groups[1].views.len(), 5); -} - -/// A tumbling window whose window ends 30 s after a multiple of its 60 s width (a -/// `TimeShift(90 s)` below its range) is refreshed on another grid than a -/// tumbling window at no offset: another refresh group. -#[test] -fn a_shifted_tumbling_window_has_its_own_phase() { - let scan = scan_m(); - let tumbling_windows = [tumbling_window(&scan, 0), tumbling_window(&scan, 90_000)]; - let mut assignment = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - assignment.set( - tumbling_window, - Materialization::MatAtIngestion(Medium::Memory), - ); - } - let roots: Vec = tumbling_windows - .iter() - .map(|tumbling_window| QueryRoot::Operator(Rc::clone(tumbling_window))) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let mut windows: Vec<(u64, u64)> = dag - .ingestion_refresh_groups() - .unwrap() - .iter() - .map(|group| { - let window = group.refresh_window.unwrap(); - (window.width_ms, window.phase_ms) - }) - .collect(); - windows.sort_unstable(); - assert_eq!(windows, [(60_000, 0), (60_000, 30_000)]); -} - -/// Two views kept at ingestion over rows with no time window (`Values`) -/// have no refresh window: each is a group of its own. -#[test] -fn views_of_unbounded_windows_are_groups_of_their_own() { - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Values { - rows: vec![], - schema: scan_m().schema.clone(), - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - let roots: Vec = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - .into_iter() - .map(|algorithm| { - let build = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(&rows), - family: quantile_sketch(algorithm), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![2]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap(); - assignment.set(&build, Materialization::MatAtIngestion(Medium::Memory)); - QueryRoot::Operator(build) - }) - .collect(); - let timed = apply_materialization_timings_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let groups = dag.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 2); - for group in &groups { - assert_eq!(group.refresh_window, None); - assert_eq!(group.views.len(), 1); - } -} - -/// A rollup across jobs of a tumbling window, both kept at ingestion, reads the tumbling window's -/// window: one refresh group, the tumbling window listed before the rollup that reads -/// it, so a run whose roots hold both computes the tumbling window in the run. -#[test] -fn a_view_reading_a_view_of_its_group_follows_it() { - let scan = scan_m(); - let tumbling_window = tumbling_window(&scan, 0); - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&tumbling_window)], - newest_tumbling_windows: None, - group_by: Some(vec![]), - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - for view in [&tumbling_window, &rollup] { - assignment.set(view, Materialization::MatAtIngestion(Medium::Memory)); - } - let timed = apply_materialization_timings_to_roots( - &[QueryRoot::Operator(Rc::clone(&rollup))], - &assignment, - ) - .unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); - dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); - let groups = dag.ingestion_refresh_groups().unwrap(); - assert_eq!(groups.len(), 1); - let [tumbling_window_view, rollup_view] = groups[0].views.as_slice() else { - panic!("{groups:#?}") - }; - assert_eq!(rollup_view.node_id, dag.roots[0]); - assert_eq!( - rollup_view.inputs.materialized_views, - [tumbling_window_view.node_id] - ); - assert_eq!( - groups[0].refresh_window, - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }) - ); -} - -/// On the DAG Stage 3 costs (chains unexpanded), a node above a chain has -/// the window of all the chain's tumbling windows: a merge of six 10 s tumbling windows, refreshed -/// at ingestion, has refresh window (60 s, 0). -#[test] -fn a_node_above_an_unexpanded_chain_has_the_window_of_its_tumbling_windows() { - let (unexpanded, _) = three_chains_over_one_scan(); - let merge_of_kll_10s = unexpanded.roots[0]; - assert_eq!( - unexpanded.refresh_window(merge_of_kll_10s), - Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 0 - }) - ); -} - -/// A non-chain view (a KLL over `TimeRange(1 min)` shifted 15 s) has the -/// same refresh window, (60 s, 15 s), on a DAG of its nodes alone (no -/// edges, what Stage 3 builds from candidates): `view_windows` walks each -/// operator's inputs. -#[test] -fn a_view_has_its_refresh_window_on_a_dag_without_edges() { - let view = tumbling_window(&scan_m(), 15_000); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&view, Materialization::MatAtIngestion(Medium::Memory)); - let timed = apply_materialization_timings(&view, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); - let view_id = dag.roots[0]; - let window = Some(RefreshWindow { - width_ms: 60_000, - phase_ms: 15_000, - }); - assert_eq!(dag.refresh_window(view_id), window); - dag.edges.clear(); - assert_eq!(dag.refresh_window(view_id), window); -} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index e2f701ffb..0d9d2e1b3 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -1,18 +1,13 @@ -//! A timed plan exports as a PhysicalASAPDAG that keeps timing. +//! A plan with its nodes' physical properties exports as a PhysicalASAPDAG +//! that keeps them. use asap_types::ir::operator::{Reduction, Source}; -use asap_types::ir::physical_export::{ - compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, - PhysicalASAPDAGValidationError, -}; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; +use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; +use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; -use asap_types::ir::{ - apply_materialization_timings, ASAPOp, MaterializationAssignment, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, QueryRoot, -}; -use asap_types::physical::{Materialization, Medium}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; +use asap_types::physical::{InvalidPhysicalProperties, Lifetime, Medium}; use std::rc::Rc; /// Scan(t) → SummaryAgg(sum by key) → FinalizeExactAccumulator, untimed. @@ -43,107 +38,22 @@ fn plan() -> Rc { .unwrap() } -const ON_DISK: Materialization = Materialization::MatAtIngestion(Medium::Disk); - -/// `plan()` timed with its summary `MatAtIngestion` on disk. -fn timed_at_ingestion() -> Rc { - let root = plan(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(root.children()[0], ON_DISK); - apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new()).unwrap() -} - -/// A summary materialized at ingestion time is read at query time, and the -/// exported DAG says where to store it. -#[test] -fn timed_plan_exports_with_timing() { - let timed = timed_at_ingestion(); - let dag = compile_physical_asap_dag(&timed).unwrap(); - dag.validate().unwrap(); - - let timings: Vec<_> = dag.nodes.iter().map(|n| n.output_state.timing).collect(); - assert_eq!( - timings, - vec![ - ExecutionTiming::IngestionTime, - ExecutionTiming::IngestionTime, - ExecutionTiming::QueryTime - ] - ); - let materializations: Vec<_> = dag.nodes.iter().map(|n| n.materialization).collect(); - assert_eq!( - materializations, - vec![ - Materialization::NotMaterialized, - ON_DISK, - Materialization::NotMaterialized - ] - ); - // Timing keeps what the summary covers. - assert!(timed.children()[0].coverage().is_some()); - - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); -} - -/// A query-time producer cannot feed an ingestion-time consumer. -#[test] -fn query_time_input_to_ingestion_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - dag.nodes[0].output_state.timing = ExecutionTiming::QueryTime; - *dag.edges[0].data_state_mut() = dag.nodes[0].output_state; - assert!(matches!( - dag.validate(), - Err(PhysicalASAPDAGValidationError::QueryDependencyInIngestion { .. }) - )); -} - -/// A summary's timing must be the one its materialization implies: the -/// deployment cannot run a `MatAtIngestion` state at query time. -#[test] -fn materialization_inconsistent_with_timing_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - dag.nodes[1].output_state.timing = ExecutionTiming::QueryTime; - *dag.edges[1].data_state_mut() = dag.nodes[1].output_state; - assert_eq!( - dag.validate(), - Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: 1, - timing: ExecutionTiming::QueryTime, - materialization: ON_DISK, - } - ) - ); - dag.nodes[1].materialization = Materialization::MatAtQuery(Medium::Disk); - dag.validate().unwrap(); -} - -/// Untimed plans cannot be exported as physical plans. -#[test] -fn untimed_plan_is_rejected() { - assert!(compile_physical_asap_dag(&plan()).is_err()); -} - /// Two queries reading one summary state export once, with one root per query. #[test] fn batch_shares_the_summary_and_keeps_one_root_per_query() { - use asap_types::ir::physical_export::compile_physical_asap_workload; let first = plan(); let state = first.children()[0].clone(); let second = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state, })) .unwrap(); - let assignment = MaterializationAssignment::default(); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = [first, second] - .iter() - .map(|root| apply_materialization_timings(root, &assignment, &mut cache).unwrap()) + let roots: Vec = [first, second] + .into_iter() + .map(QueryRoot::Operator) .collect(); - let roots: Vec = timed.into_iter().map(QueryRoot::Operator).collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) + .map(|(dag, _)| dag) + .unwrap(); dag.validate().unwrap(); assert_eq!(dag.roots.len(), 2); assert_eq!(dag.nodes.len(), 4, "scan and summary are exported once"); @@ -155,22 +65,18 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { fn scalar_roots_export_in_batch_order_and_reference_nodes() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - let timed = apply_materialization_timings( - &plan(), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); + let read = plan(); let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&timed))), + left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&read))), right: Box::new(ScalarExpr::literal_f64(1.0)), semantics: ExprSemantics::Promql, }; - let dag = compile_physical_asap_workload(&[ + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[ QueryRoot::Scalar(scalar), - QueryRoot::Operator(Rc::clone(&timed)), + QueryRoot::Operator(Rc::clone(&read)), ]) + .map(|(dag, _)| dag) .unwrap(); assert_eq!(dag.nodes.len(), 3, "the read plan is exported once"); assert_eq!(dag.roots.len(), 1); @@ -187,7 +93,7 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { scalar_only.roots.clear(); scalar_only.validate().unwrap(); let valid_scalar_only = scalar_only.clone(); - // Materialization is checked on the nodes a scalar reads, too. + // Physical properties are checked on the nodes a scalar reads, too. let state = scalar_only .nodes .iter() @@ -200,16 +106,14 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { ) }) .unwrap(); - scalar_only.nodes[state].materialization = ON_DISK; + scalar_only.nodes[state].output_state.timing = ExecutionTiming::IngestionTime; + scalar_only.nodes[state].lifetime = Lifetime::OneEvaluation(Medium::Disk); assert_eq!( scalar_only.validate(), - Err( - PhysicalASAPDAGValidationError::MaterializationTimingMismatch { - node: scalar_only.nodes[state].id, - timing: ExecutionTiming::QueryTime, - materialization: ON_DISK, - } - ) + Err(PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + node: scalar_only.nodes[state].id, + reason: InvalidPhysicalProperties::OneEvaluationAtIngestion, + }) ); let mut missing = dag.clone(); @@ -276,13 +180,7 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { child: series("a"), })) .unwrap(); - let timed = apply_materialization_timings( - &project, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time(&project).unwrap(); dag.validate().unwrap(); let roles: Vec<_> = dag .edges @@ -318,13 +216,7 @@ fn swapped_edge_roles_are_rejected() { right: scan("b"), })) .unwrap(); - let timed = apply_materialization_timings( - &join, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); + let mut dag = asap_types::test_support::physical_asap_dag_at_query_time(&join).unwrap(); dag.validate().unwrap(); for edge in &mut dag.edges { edge.role = match edge.role { @@ -345,141 +237,12 @@ fn swapped_edge_roles_are_rejected() { #[test] fn an_empty_batch_is_an_error() { assert!(matches!( - compile_physical_asap_workload(&[]), - Err(ExecutionDataStateError::InvalidPhysicalDAG( - PhysicalASAPDAGValidationError::NoRoots - )) + asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&[]) + .map(|(dag, _)| dag), + Err(PhysicalASAPDAGValidationError::NoRoots) )); } -/// A summary over another summary exports a typed maintenance edge between -/// them, and the document round-trips and rejects unknown fields and versions. -#[test] -fn exports_summary_over_summary_as_typed_precompute_edges() { - use asap_types::ir::physical_export::{GroupingEdgeCompatibility, WindowEdgeCompatibility}; - use asap_types::ir::properties::ExecutionDataState; - use asap_types::test_support::compile_physical_asap_dag_with_node_ids; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap(); - let sum = |child, column| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(column), - reduction: Reduction::by(vec![]), - grouping: Default::default(), - filter: None, - })) - .unwrap() - }; - let inner = sum(scan, ColumnRef::SampleValue); - let outer = sum(Rc::clone(&inner), ColumnRef::Named("state".into())); - let root = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: outer, - })) - .unwrap(); - let mut cache = NodeToTimedNodeCache::new(); - let at_ingestion = - MaterializationAssignment::with_default(Materialization::MatAtIngestion(Medium::Memory)); - let timed = apply_materialization_timings(&root, &at_ingestion, &mut cache).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - assert_eq!(compiled.node_ids.node_id(&timed), Some(3)); - assert_eq!( - compiled - .node_ids - .node_id(asap_types::test_support::timed(&cache, &inner).unwrap()), - Some(1) - ); - let dag = compiled.dag; - assert_eq!(dag.roots, vec![3]); - assert_eq!( - dag.nodes[1].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!( - dag.nodes[2].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - let dependency = dag - .edges - .iter() - .find(|e| e.producer == 1 && e.consumer == 2) - .unwrap(); - assert_eq!( - dependency.data_state(), - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!(dependency.grouping(), GroupingEdgeCompatibility::Identical); - assert_eq!( - dependency.window(), - WindowEdgeCompatibility::RequiresAlignedTumblingWindowPhaseOrExactWindowEdgeResidual - ); - assert!(matches!( - dependency.intermediate_schema().fields[0].dtype, - FieldDataType::ExactAggregate(ExactKind::Sum, _) - )); - assert!(matches!( - dag.nodes[2].payload, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - reduction: Reduction::Reduce(_), - .. - }) - )); - - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); - decoded.validate().unwrap(); - let mut unknown = serde_json::to_value(&dag).unwrap(); - unknown["unexpected"] = serde_json::json!(true); - assert!(serde_json::from_value::(unknown).is_err()); -} - -/// `retention_ms` is accepted on a materialized view (a node kept across -/// evaluations), survives a round trip, and is rejected on a node that is -/// neither a view nor a query-time scan (here an ingestion-time node). -#[test] -fn retention_is_only_on_materialized_views_and_query_time_scans() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); - let view = dag - .nodes - .iter() - .position(|node| asap_types::test_support::is_materialized_view(node.materialization)) - .expect("a node built at ingestion"); - let other = dag - .nodes - .iter() - .position(|node| { - !asap_types::test_support::is_materialized_view(node.materialization) - && node.output_state.timing == ExecutionTiming::IngestionTime - }) - .expect("an ingestion-time node that is not a view"); - dag.nodes[view].retention_ms = Some(60_000); - dag.validate().unwrap(); - let decoded: PhysicalASAPDAG = - serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); - assert_eq!(decoded, dag); - dag.nodes[other].retention_ms = Some(60_000); - assert_eq!( - dag.validate(), - Err(PhysicalASAPDAGValidationError::UnexpectedRetention( - dag.nodes[other].id - )) - ); -} - /// Node-id-keyed bindings serialize in numeric id order. #[test] fn physical_node_ids_serialize_in_deterministic_binding_order() { @@ -493,41 +256,6 @@ fn physical_node_ids_serialize_in_deterministic_binding_order() { ); } -/// The plan with every node at query time is the same operators with -/// nothing materialized: what the default materialization exports. The -/// shared summary stays one node, and the scalar root that reads it keeps -/// its batch position. -#[test] -fn every_node_at_query_time_keeps_the_operators_and_stores_nothing() { - use asap_types::ir::scalar::ArithmeticOpKind; - use asap_types::ir::{ExprSemantics, ScalarExpr}; - let workload = |root: Rc| { - let scalar = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&root))), - right: Box::new(ScalarExpr::literal_f64(1.0)), - semantics: ExprSemantics::Promql, - }; - compile_physical_asap_workload(&[ - QueryRoot::Operator(Rc::clone(&root)), - QueryRoot::Scalar(scalar), - QueryRoot::Operator(root), - ]) - .unwrap() - }; - let materialized = workload(timed_at_ingestion()); - let at_query_time = apply_materialization_timings( - &plan(), - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!( - materialized.with_every_node_at_query_time().unwrap(), - workload(at_query_time) - ); -} - /// A window 600,000 ms wide ending 300,000 ms before T = 3,600,000 covers /// `(2,700,000, 3,300,000]`, and reaches 900,000 ms back. #[test] @@ -572,17 +300,10 @@ fn exported_with_scan( asap_types::ir::physical_export::PhysicalASAPDAG, asap_types::ir::physical_export::PhysicalASAPNodeId, ) { - // One timing cache, so the roots keep sharing their `Scan`. - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = roots - .into_iter() - .map(|root| { - apply_materialization_timings(&root, &MaterializationAssignment::default(), &mut cache) - .unwrap() - }) - .map(QueryRoot::Operator) - .collect(); - let dag = compile_physical_asap_workload(&timed).unwrap(); + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) + .map(|(dag, _)| dag) + .unwrap(); let scans: Vec<_> = dag .nodes .iter() @@ -712,7 +433,8 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { schema: schema.clone(), }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -725,7 +447,8 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { children: inputs.clone(), }), output_state: ExecutionDataState::QUERY_ROWS, - materialization: Materialization::NotMaterialized, + lifetime: Lifetime::None, + parameters: None, output_schema: schema.clone(), guarantee: None, retention_ms: None, @@ -764,16 +487,3 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { Err(PhysicalASAPDAGValidationError::EdgeRoleMismatch { .. }) )); } - -/// A workload export of one timed root is that root's export, and an -/// untimed root is invalid. -#[test] -fn workload_export_matches_the_single_root_export() { - use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let timed = timed_at_ingestion(); - let roots = [QueryRoot::Operator(Rc::clone(&timed))]; - let exported = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); - assert_eq!(exported.dag, compile_physical_asap_dag(&timed).unwrap()); - let untimed = [QueryRoot::Operator(plan())]; - assert!(compile_physical_asap_workload_with_node_ids(&untimed).is_err()); -} diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 6e61144a8..611b0b316 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,5 +1,5 @@ use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; +use asap_types::ir::properties::AccuracyGuarantee; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; @@ -67,8 +67,7 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { schema.fields[0].name = "alias".into(); schema.fields[0].table = Some("result".into()); let renamed = OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))) - .with_timing(Some(ExecutionTiming::QueryTime)); + .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); @@ -80,7 +79,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { Some(&DataType::Utf8) ); assert!(rebuilt.guarantee.is_none()); - assert!(rebuilt.timing.is_none()); rebuilt.validate_structure().unwrap(); } } @@ -229,50 +227,6 @@ fn series_selector() -> Rc { .unwrap() } -/// `topk(5, a)` read from a current-series population of `a`, built by hand. -fn current_series_topk() -> Rc { - use asap_types::ir::operator::maintained_population::*; - let selector = series_selector(); - let population = OperatorNode::new_shared(Operator::ASAP(ASAPOp::MaintainPopulation { - child: selector, - population: MaintainedPopulation { - input: PopulationInput::CurrentSeries(CurrentSeriesInput { - metric: "a".into(), - matchers: vec![], - grouping: vec![], - without: false, - lookback_ms: 1_000, - }), - max_k: 5, - quantiles: false, - }, - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::EvaluatePopulation { - child: population, - evaluation: PopulationStatistic::TopK { k: 5 }, - })) - .unwrap() -} - -/// Time `root` under the default materialization assignment, which runs the -/// data-state and population-contract validation, and export it. -fn compile_physical_asap_dag(root: &Rc) -> Result<(), String> { - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - root.validate_structure().map_err(|e| e.to_string())?; - let timed = apply_materialization_timings( - root, - &MaterializationAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .map_err(|e| format!("{e:?}"))?; - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) - .map_err(|e| format!("{e:?}"))?; - Ok(()) -} - /// A table-row population maintains a numeric value column: one that names /// a label column fails validation. #[test] @@ -331,80 +285,3 @@ fn a_current_series_population_matches_only_its_selectors_lookback() { }; assert!(!asap_types::test_support::matches_node(&wrong, &selector)); } - -/// Population timing is a materialization choice: the population follows its -/// assignment (`NotMaterialized` by default), while its evaluation always -/// runs at query time. -#[test] -fn population_timing_follows_its_materialization() { - use asap_types::ir::physical_export::compile_physical_asap_dag as export_timed; - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - use asap_types::physical::{Materialization, Medium}; - let candidate = current_series_topk(); - let Some(ASAPOp::EvaluatePopulation { - child: population, .. - }) = candidate.asap() - else { - unreachable!() - }; - let on_disk = Materialization::MatAtIngestion(Medium::Disk); - let mut at_ingestion = MaterializationAssignment::default(); - at_ingestion.set(population, on_disk); - for (assignment, timing, materialization) in [ - ( - MaterializationAssignment::default(), - QueryTime, - Materialization::NotMaterialized, - ), - (at_ingestion, IngestionTime, on_disk), - ] { - let timed = apply_materialization_timings( - &candidate, - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = export_timed(&timed).unwrap(); - let [population, evaluation] = &dag.nodes[dag.nodes.len() - 2..] else { - unreachable!() - }; - assert_eq!(population.output_state.timing, timing); - assert_eq!(population.materialization, materialization); - assert_eq!(evaluation.output_state.timing, QueryTime); - } - let mut evaluation_at_ingestion = (*candidate).clone(); - evaluation_at_ingestion.timing = Some(IngestionTime); - assert!(compile_physical_asap_dag(&Rc::new(evaluation_at_ingestion)).is_err()); -} - -/// An evaluation cannot exceed its population's contract (`max_k`), and a -/// population cannot claim a source it does not read. -#[test] -fn malformed_population_dags_fail_closed() { - use asap_types::ir::operator::maintained_population::{PopulationInput, PopulationStatistic}; - let candidate = current_series_topk(); - compile_physical_asap_dag(&candidate).expect("the unmodified candidate is legal"); - let mut bad = (*candidate).clone(); - let Operator::ASAP(ASAPOp::EvaluatePopulation { evaluation, .. }) = &mut bad.operator else { - unreachable!() - }; - *evaluation = PopulationStatistic::TopK { k: 6 }; - assert!(compile_physical_asap_dag(&Rc::new(bad.clone())).is_err()); - let Operator::ASAP(ASAPOp::EvaluatePopulation { child, evaluation }) = &mut bad.operator else { - unreachable!() - }; - *evaluation = PopulationStatistic::TopK { k: 5 }; - let producer = Rc::make_mut(child); - let Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) = &mut producer.operator - else { - unreachable!() - }; - let PopulationInput::CurrentSeries(spec) = &mut population.input else { - unreachable!() - }; - spec.metric = "b".into(); - assert!(compile_physical_asap_dag(&Rc::new(bad)).is_err()); -} diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 0c16614cf..6382be875 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -90,20 +90,10 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, + FieldDataType, GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; - let family = FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 100, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()); let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(), @@ -141,36 +131,6 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { assert!(validate_asap_inputs(&ASAPOp::FinalizeExactAccumulator { child: state }).is_err()); } -/// Phase validation checks dependencies, without declaring a computation query-only. -#[test] -fn execution_timing_checks_edges_not_function_names() { - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::ProjectItem; - let input = Rc::new((*scan()).clone().with_timing(Some(IngestionTime))); - let mut project = OperatorNode::new(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - child: input, - cols: vec![ProjectItem { - expr: ScalarExpr::FunctionCall { - name: "promql_abs".into(), - args: vec![ScalarExpr::Column(0)], - }, - alias: None, - }], - qualifier: None, - })) - .unwrap() - .with_timing(Some(IngestionTime)); - Rc::new(project.clone()) - .validate_execution_timing() - .unwrap(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Project { child, .. }) = - &mut project.operator - { - *child = Rc::new(child.as_ref().clone().with_timing(Some(QueryTime))); - } - assert!(Rc::new(project).validate_execution_timing().is_err()); -} - /// Both operator categories use the same fallible schema-deriving constructor. #[test] fn shared_construction_derives_both_operator_categories() { @@ -195,7 +155,6 @@ fn shared_construction_derives_both_operator_categories() { assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); assert!(state.guarantee.is_none()); - assert!(state.timing.is_none()); state.validate_structure().unwrap(); assert!( diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs index 07357264b..52b414922 100644 --- a/crates/types/tests/summary_coverage.rs +++ b/crates/types/tests/summary_coverage.rs @@ -10,13 +10,11 @@ use asap_types::ir::operator::TimeShift; use asap_types::ir::properties::summary_coverage::{ ColumnIdentity, Constraint, CoverageError, SelectionBox, }; -use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ - ExactKind, ExactParams, GroupingStrategy, MergeInputRule, SketchAlgorithm, SketchKind, - SketchParams, SummaryUpdate, + ExactKind, ExactParams, GroupingStrategy, MergeInputRule, SketchAlgorithm, SummaryUpdate, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, @@ -91,16 +89,7 @@ fn kll(child: Rc, column: &str) -> Rc { summary( child, column, - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), ) } @@ -554,22 +543,6 @@ fn merges_nest() { ); } -/// When a state is computed is not what it computes. -#[test] -fn timing_does_not_block_merges() { - let ingested = Rc::new( - (*tumbling_window(60_000, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ); - let queried = Rc::new( - (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - merge(vec![ingested, queried]).unwrap(); -} - /// A merge built around `new` is still checked by `validate_structure`. #[test] fn forged_merges_fail_validation() { @@ -741,16 +714,7 @@ fn design_doc_worked_example() { let below_100 = compare(2, CompareOpKind::Lt, ScalarValue::Int64(100)); let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: project, - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - Default::default(), - ), + family: FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), input: SummaryUpdate::column(ColumnRef::Named("latency".into())), reduction: Reduction::by(vec![0]), grouping: GroupingStrategy::default(), @@ -805,11 +769,9 @@ fn equal_definitions_get_equal_ids() { let us = latency_where(eq(REGION, "us")); let eu = latency_where(eq(REGION, "eu")); assert_eq!(id(&us), id(&eu)); - let ingested = (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)); - assert_eq!(id(&ingested), id(&tumbling_window(60_000, vec![]))); - assert_eq!(id(&ingested).len(), 32); + let newest = tumbling_window(0, vec![]); + assert_eq!(id(&newest), id(&tumbling_window(60_000, vec![]))); + assert_eq!(id(&newest).len(), 32); let size = kll(filter(table(), eq(REGION, "eu")), "size"); assert_ne!(id(&size), id(&eu)); assert_ne!( @@ -905,16 +867,7 @@ fn grouped( } fn kll_family() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) } fn rollup(state: Rc, keys: Vec) -> Result, String> { diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index b76ad5cc7..b9c73d9ee 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -1,34 +1,22 @@ //! Window composition merges compatible summary states without consuming raw rows. use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ DataType, ExactKind, ExactParams, FamilyCapability, Field, FieldDataType, GroupingStrategy, HydraKind, HydraParams, MergeInputRule, MergedError, SamplingKind, SamplingParams, Schema, - SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, StatModelKind, StatModelParams, - StatisticKind, SummaryUpdate, WaveletKind, WaveletParams, + SketchAlgorithm, SketchParams, SketchStatistic, StatModelKind, StatModelParams, StatisticKind, + SummaryUpdate, WaveletKind, WaveletParams, }; use asap_types::ir::{ - apply_materialization_timings, ASAPOp, ExprSemantics, MaterializationAssignment, - NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, }; -use asap_types::physical::{Materialization, Medium}; use std::rc::Rc; /// KLL over `value` for one `region`, so states of different regions are /// disjoint. -fn state(k: u32, region: &str) -> Rc { +fn state(_k: u32, region: &str) -> Rc { family_state( - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k, - failure_probability: None, - }, - ), - Default::default(), - ), + FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), region, ) } @@ -77,14 +65,12 @@ fn compatible_tumbling_windows_merge_structurally() { root.validate_structure().unwrap(); assert_eq!(root.schema.fields.len(), 1); } -/// An empty merge, raw rows and differently sized state cannot masquerade as compatible tumbling windows. +/// An empty merge and raw rows cannot masquerade as compatible tumbling +/// windows. (State sizes are no longer part of the state's type: θ is a +/// candidate's `LogicalProperties::parameters`.) #[test] fn incompatible_merge_inputs_fail() { - for children in [ - vec![], - vec![state(200, "us"), state(300, "eu")], - vec![state(200, "us").children()[0].clone()], - ] { + for children in [vec![], vec![state(200, "us").children()[0].clone()]] { assert!( OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children, @@ -96,21 +82,18 @@ fn incompatible_merge_inputs_fail() { } } -fn sketch(algorithm: SketchAlgorithm, params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(SketchKind::new(algorithm, params), Default::default()) +fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { + FieldDataType::Sketch(algorithm, Default::default()) } fn hydra( algorithm: SketchAlgorithm, - params: SketchParams, + _params: SketchParams, kind: HydraKind, - hydra_params: HydraParams, + _hydra_params: HydraParams, ) -> FieldDataType { FieldDataType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::SharedMultiSubpopulation { - kind, - params: hydra_params, - }, + algorithm, + GroupingStrategy::SharedMultiSubpopulation { kind }, ) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { @@ -577,6 +560,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { SketchParams::CmsWithHeap { .. } => A::CmsWithHeap, SketchParams::CountSketchWithHeap { .. } => A::CountSketchWithHeap, SketchParams::UnivMon { .. } => A::UnivMon, + SketchParams::Hydra(_) => unreachable!("no Hydra parameters in this list"), }; sketch(algorithm, params) }) @@ -638,7 +622,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { }; for query in &statistics { let fixed = matches!( - (kind.algorithm(), query.kind()), + (kind, query.kind()), ( SketchAlgorithm::UnivMon, StatisticKind::ItemFrequency | StatisticKind::TopK @@ -648,7 +632,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { StatisticKind::ItemFrequency, ) ); - let before = accepted_by_category(kind.algorithm(), query); + let before = accepted_by_category(kind, query); let after = estimate(family.clone(), query.clone()).is_ok(); assert_eq!(after, before != fixed, "{family:?} {query:?}"); } @@ -694,53 +678,3 @@ fn idempotent_families_merge_overlapping_states() { assert!(error.contains("not proven disjoint"), "{error}"); } } - -/// `SummaryMerge` takes the timing of its consumer, and reads tumbling-window states -/// stored at ingestion time (#509 Example 3/4 Pattern B1) or built at query -/// time (B2). Merging at ingestion time needs every tumbling window at ingestion time. -#[test] -fn summary_merge_timing() { - let tumbling_windows = [state(200, "us"), state(200, "eu")]; - let root = merge(tumbling_windows.to_vec()).unwrap(); - let timings = |assignment: &MaterializationAssignment| { - let timed = - apply_materialization_timings(&root, assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let children = timed.children(); - ( - timed.timing.unwrap(), - children - .iter() - .map(|c| c.timing.unwrap()) - .collect::>(), - ) - }; - use ExecutionTiming::{IngestionTime, QueryTime}; - assert_eq!( - timings(&MaterializationAssignment::default()), - (QueryTime, vec![QueryTime, QueryTime]) - ); - let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); - let mut stored_tumbling_windows = MaterializationAssignment::default(); - for tumbling_window in &tumbling_windows { - stored_tumbling_windows.set(tumbling_window, at_ingestion); - } - assert_eq!( - timings(&stored_tumbling_windows), - (QueryTime, vec![IngestionTime, IngestionTime]) - ); - // A `SummaryAgg` without an assignment runs at query time, so a merge - // stored at ingestion time needs its tumbling windows there too. - let mut only_merge = MaterializationAssignment::default(); - only_merge.set(&root, at_ingestion); - assert!( - apply_materialization_timings(&root, &only_merge, &mut NodeToTimedNodeCache::new()) - .is_err() - ); - let mut stored_merge = MaterializationAssignment::with_default(at_ingestion); - stored_merge.set(&root, at_ingestion); - assert_eq!( - timings(&stored_merge), - (IngestionTime, vec![IngestionTime, IngestionTime]) - ); -} diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index b37d77385..d5242da20 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -11,9 +11,7 @@ use asap_types::ir::operator::tumbling_window_chain::{ }; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ - GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, -}; +use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; use asap_types::test_support::{tumbling_window, whole_window_build}; @@ -47,16 +45,7 @@ fn shifted(child: Rc, offset_ms: i64) -> Rc { } fn kll() -> FieldDataType { - FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ), - GroupingStrategy::default(), - ) + FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) } /// Five 1-min KLL tumbling windows per series over `input`. @@ -174,49 +163,6 @@ fn expansion_merges_the_tumbling_windows() { assert!(expand_tumbling_window_chains(&chain).is_err()); } -/// Expanding a timed chain carries its timing and materialization onto its -/// tumbling windows: a chain materialized at ingestion time expands into tumbling windows each -/// built and stored at ingestion time, over ranges and shifts that run at -/// ingestion time too, while the merge reading them stays at query time. -/// An unmaterialized chain expands into query-time tumbling windows. -#[test] -fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { - use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::{ - apply_materialization_timings, MaterializationAssignment, NodeToTimedNodeCache, - }; - use asap_types::physical::{Materialization, Medium}; - let at_ingestion = Materialization::MatAtIngestion(Medium::Memory); - for (materialization, timing) in [ - (at_ingestion, ExecutionTiming::IngestionTime), - (Materialization::NotMaterialized, ExecutionTiming::QueryTime), - ] { - let chain = chain(shifted(scan(), MINUTE as i64)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: merge(vec![Rc::clone(&chain)]), - query: SketchStatistic::Quantile { q: 0.99 }, - })) - .unwrap(); - let mut assignment = MaterializationAssignment::default(); - assignment.set(&chain, materialization); - let timed = - apply_materialization_timings(&estimate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - let merged = expanded.children()[0]; - assert_eq!(merged.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(merged.children().len(), 5); - for tumbling_window in merged.children() { - assert_eq!(tumbling_window.timing, Some(timing), "{materialization:?}"); - assert_eq!(tumbling_window.materialization, materialization); - for below in OperatorNode::reachable(tumbling_window).iter().skip(1) { - assert_eq!(below.timing, Some(timing), "{}", below.operator.kind_name()); - assert_eq!(below.materialization, Materialization::NotMaterialized); - } - } - } -} - /// A chain needs a positive width and count, and reads the instant vector /// a range selector would. #[test] @@ -245,33 +191,6 @@ fn a_chain_reads_an_instant_vector() { assert!(empty.validate_structure().is_err()); } -/// Two merges over one chain (two queries reading the same tumbling windows) expand -/// into one set of tumbling windows: the same five `SummaryAgg` nodes, not ten. -#[test] -fn merges_over_one_chain_share_its_tumbling_windows() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(merge(vec![Rc::clone(&shared)])), - QueryRoot::Operator(merge(vec![shared])), - ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(expansion.chains.len(), 1); - assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); - let expanded = expansion.roots; - let tumbling_windows = |root: &QueryRoot| match root { - QueryRoot::Operator(node) => node.children().into_iter().cloned().collect::>(), - QueryRoot::Scalar(_) => unreachable!(), - }; - let (first, second) = ( - tumbling_windows(&expanded[0]), - tumbling_windows(&expanded[1]), - ); - assert_eq!(first.len(), 5); - assert!(first.iter().zip(&second).all(|(a, b)| Rc::ptr_eq(a, b))); -} - fn newest(chain: Rc, newest_tumbling_windows: u32) -> Rc { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![chain], @@ -324,42 +243,6 @@ fn a_merge_of_the_newest_tumbling_windows_reads_one_chain_and_fewer_than_all_tum } } -/// A merge of the newest 2 tumbling windows and a merge of all 5 over one chain -/// expand into one set of tumbling windows: the partial merge reads the first 2 of -/// the full merge's 5 nodes, newest first, and the chain lists all 5. -#[test] -fn a_merge_of_the_newest_tumbling_windows_reads_the_first_tumbling_windows_of_the_chain() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(newest(Rc::clone(&shared), 2)), - QueryRoot::Operator(merge(vec![shared])), - ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(expansion.chains.len(), 1); - assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); - let tumbling_windows = |root: &QueryRoot| match root { - QueryRoot::Operator(node) => node.children().into_iter().cloned().collect::>(), - QueryRoot::Scalar(_) => unreachable!(), - }; - let (partial, full) = ( - tumbling_windows(&expansion.roots[0]), - tumbling_windows(&expansion.roots[1]), - ); - assert_eq!((partial.len(), full.len()), (2, 5)); - assert!(partial.iter().zip(&full).all(|(a, b)| Rc::ptr_eq(a, b))); - for (root, window) in expansion.roots.iter().zip([2, 5]) { - let QueryRoot::Operator(node) = root else { - unreachable!() - }; - assert_eq!( - relative_time(node), - (Bound::Excluded(-window * MINUTE as i64), Bound::Included(0)) - ); - } -} - /// `newest_tumbling_windows` is left out of the JSON when `None`, and JSON without it /// reads as `None`; `Some(n)` round-trips. #[test] @@ -443,33 +326,6 @@ fn a_rollup_of_the_newest_tumbling_windows_covers_their_window() { ); } -/// Expansion keeps every tumbling window and the identity of a chain two -/// readers share. -#[test] -fn expansion_preserves_every_tumbling_window_and_shared_reader() { - use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; - use asap_types::ir::QueryRoot; - let shared = chain(scan()); - let roots = [ - QueryRoot::Operator(merge(vec![shared.clone()])), - QueryRoot::Operator(merge(vec![shared])), - ]; - let result = expand_tumbling_window_chains_in_workload(&roots).unwrap(); - assert_eq!(result.chains.len(), 1); - assert_eq!(result.chains[0].tumbling_windows.len(), 5); - let QueryRoot::Operator(first) = &result.roots[0] else { - unreachable!() - }; - let QueryRoot::Operator(second) = &result.roots[1] else { - unreachable!() - }; - assert!(first - .children() - .iter() - .zip(second.children()) - .all(|(a, b)| Rc::ptr_eq(a, b))); -} - /// JSON written before #1163 names the chain `PaneChain`, its fields /// `tumbling_window_width_ms` and `tumbling_window_count`, and a merge's `newest_tumbling_windows`; it reads /// as today's operators. diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 400b73675..a3fc422c1 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -305,7 +305,7 @@ alternative stays visible in `PlanningRun`. PlanningWorkload + frontend dependencies + PlanningModels -> Stage 0: frontend lowering, one root per normalized query entry -> Stage 1: Pass 1 per-computation candidates, Pass 2 sharing, tumbling windows, reuse - -> Stage 2: one physical candidate per materialization assignment + -> Stage 2: one physical candidate per physical properties assignment -> Stage 3: check, measure and select -> PlanOutput ``` diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index a9961b710..4d42aaab9 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -461,9 +461,9 @@ a numeric entity key is not a score. Exact accumulator inputs are explicitly finalized before row operators consume them. None of these operations proves candidate completeness; that evidence belongs to the semi-join's pruning step. -The logical DAG carries no execution layout: `apply_materialization_timings` -writes each node's timing and materialization from a -`MaterializationAssignment` before export (every sub-DAG `NotMaterialized` by -default, so all query time). The exported DAG carries both, and the deployment +The logical DAG carries no execution layout: `apply_physical_properties` +writes each node's timing and lifetime from a +`PhysicalPropertiesAssignment` before export (no sub-DAG materialized by default, so all +query time). The exported DAG carries both, and the deployment executes it as given; this is not a claim that every deployment has implemented every placement. diff --git a/docs/design_docs/architecture/planner-runtime-contract.md b/docs/design_docs/architecture/planner-runtime-contract.md index 69813bc5f..dbab5b3cc 100644 --- a/docs/design_docs/architecture/planner-runtime-contract.md +++ b/docs/design_docs/architecture/planner-runtime-contract.md @@ -14,7 +14,7 @@ execution remain downstream. The | Layer | Owner | Examples | |---|---|---| | Logical candidate semantics | ASAPPlanner | Query rewrite; summary family and parameters; grouping; accuracy guarantees when established. | -| Summary materialization and realization selection | ASAPPlanner Stage 2 materialization (#509) | `NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)` per sub-DAG, and how long a stored output is kept; direct build or incremental update; window implementations compared using provider evidence. | +| Summary materialization and realization selection | ASAPPlanner Stage 2 materialization (#509) | a timing and a lifetime per sub-DAG (`None`, `OneEvaluation(medium)` or `AcrossEvaluations(medium)`), and how much a materialized output retains; direct build or incremental update; window implementations compared using provider evidence. | | Concrete implementation and deployment | ASAPQuery-backend and its workload optimizer | Library and data-structure implementation, exact tumbling-window layout, placement, sharding, storage, transmission, materialization IDs, executor configuration, and workload-wide assignment. | ASAPCollector and the ASAPQuery data plane execute the compiled downstream diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index 3011064f3..d8756480f 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -101,7 +101,7 @@ pub struct QueryPlan { ``` `Stage1ToStage3Pass` returns the selected plan's roots, as Stage 2's realizer -built them with each node's timing and materialization, and with the weight +built them with each node's timing and lifetime, and with the weight proofs Stage 3 made applied; `PlanOutput::physical_asap_dag()` exports them. Physical ASAP-aware optimization decides per sub-DAG whether to materialize, and whether at ingestion or query time, from #604 on. @@ -125,7 +125,7 @@ tracked in #580. | Step | Call | |---|---| | Prepare roots | Entries in `entries()` order, the accuracy target from each entry's `requirements`; PromQL operator roots carry the series identity (`with_promql_series_identity`) where it has a realization | -| Stage 1: logical ASAP-aware optimization | `generate_per_computation_logical_memo_candidates` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | +| Stage 1: logical ASAP-aware optimization | `pass1_memo` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | | Stage 2: physical ASAP-aware optimization | `generate_physical_asap_memo_candidates` | | Stage 3: plan selection | `select_plan`: check accuracy and latency, price under the cost model, select one plan; the facade builds it with Stage 2's `AssignmentRealizer::realize` | diff --git a/docs/design_docs/concepts/glossary.md b/docs/design_docs/concepts/glossary.md index c27400a31..8b52e6ef6 100644 --- a/docs/design_docs/concepts/glossary.md +++ b/docs/design_docs/concepts/glossary.md @@ -38,7 +38,7 @@ A concrete execution and deployment choice: runtime topology of data lifecycle s ## Materialized view -A node of the physical DAG whose output is kept across evaluations (`MatAtIngestion` or `MatAtQueryKeptAcrossEvaluations`), with how long it is kept (`retention_ms` on the node). A `MatAtQuery` node is not a view: its output is discarded after one evaluation. Code: `MaterializedView`, `PhysicalASAPDAG::materialized_views`. +A node of the physical DAG whose output is materialized across evaluations (lifetime `AcrossEvaluations`, at ingestion time or at query time), with how long it is materialized (`retention_ms` on the node). A node materialized for `OneEvaluation` is not a view: its output is discarded after one evaluation. Code: `MaterializedView`, `PhysicalASAPDAG::materialized_views`. ## Raw data sample diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index ea9d9e57b..9377eee5d 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -81,7 +81,7 @@ Exact work is represented by the ordinary operators, unchanged: - `BinaryOp` combines independently planned operands. Summary planning may set its typed division guards (`checked_finite_division`, `checked_relative_division`); the operator's timing comes from the - materialization assignment, not from the operator. + physical properties assignment, not from the operator. - Aggregate, projection, filter, sort and limit over an evaluation are the ordinary `Aggregate`, `Project`, `Filter`, `Sort` and `Limit` operators reading an ASAP node. Exact-accumulator state may pass through the projection-like @@ -108,60 +108,63 @@ happens: **ingestion time** or **query time**. Operator identity must not imply one of these phases. Backend capability restrictions are implementation gaps, not definitions of the operator. -The logical DAG carries no timing: `OperatorNode::timing` is `None` on every -front-end node and every candidate, and `map_children` clears it. +Operator nodes carry no physical properties: the logical DAG has no timing +or lifetime. Materialization ([#509 §Materialization](../proposals/planner-layering.md#materialization)) -gives each sub-DAG one of three options: `NotMaterialized`, -`MatAtIngestion(medium)` or `MatAtQuery(medium)`, with `Medium::Memory` or -`Medium::Disk`. A -[`MaterializationAssignment`](../../../crates/types/src/ir/properties/timing.rs) records -the option per sub-DAG, named by the node whose output would be stored. Any -node may be named: a summary state, a population, or, as in #509 Example 4, -the tumbling windows a query reads. `set(node, materialization)` assigns -one, and every other sub-DAG is `NotMaterialized`. Choosing the options is -Stage 2's job (#604); until then the planner materializes nothing. -`apply_materialization_timings(root, &assignment, &mut NodeToTimedNodeCache)` then writes a -timing and a materialization into every node, top-down: +decides, for each sub-DAG, whether its output is materialized, and if so with which +[`PhysicalProperties`](../../../crates/types/src/physical/properties.rs): a +timing and a `Lifetime` (#1200). The lifetime is `None` (discarded after its +reader), `OneEvaluation(medium)` (materialized until every reader of one evaluation +finishes) or `AcrossEvaluations(medium)` (materialized for later evaluations), with +`Medium::Memory` or `Medium::Disk`; an ingestion-time output cannot be materialized +for `OneEvaluation`. A +[`PhysicalPropertiesAssignment`](../../../crates/types/src/ir/properties/timing.rs) records, +per sub-DAG, the properties of its materialized output, named by the node whose output +would be materialized. Any node may be named: a summary state, a population, or, as in +#509 Example 4, the tumbling windows a query reads. `set(node, properties)` +lists one; an entry whose lifetime is `None` fixes only the node's timing, and every +sub-DAG the assignment does not list is not materialized. Stage 2 chooses the +assignment. Operator nodes carry no timing or lifetime (#1200): physical +stages read them from the assignment. +`apply_physical_properties(root, &assignment)` then completes it into a timing and a +lifetime for every node, top-down, and returns the completed assignment: - `SummaryEstimate` and `EvaluatePopulation` run at query time; -- a node assigned `MatAtIngestion` runs at ingestion time, and one assigned - `MatAtQuery` at query time; a `SummaryAgg` or `MaintainPopulation` that is - `NotMaterialized` runs at query time. Every node upstream of an - ingestion-time node also runs at ingestion time, so `MatAtIngestion` over - something that can only exist at query time (an evaluation, or a state - that is not `MatAtIngestion`) fails with +- a node the assignment lists runs at its entry's timing; a `SummaryAgg` + or `MaintainPopulation` that is not materialized runs at query time. Every node + upstream of an ingestion-time node also runs at ingestion time, so a materialized + ingestion-time output over something that can only exist at query time (an + evaluation, or a state not materialized at ingestion time) fails with `ExecutionDataStateError::UpstreamNotAtIngestion` ("upstream of ingestion-time node not at ingestion time"); -- every other node is not stored and runs when its consumer runs: everything +- every other node is not materialized and runs when its consumer runs: everything that feeds an ingestion-time node runs at ingestion time, everything above - an evaluation at query time. A query-time consumer reads a `MatAtIngestion` - node's stored output. - -A node whose timing a candidate builder fixed (the query-time -`FinalizeExactAccumulator` read boundary of `finalize_query_candidate`) keeps -it: an explicit assignment that agrees with that timing is kept, and one that -needs the other timing fails with -`ExecutionDataStateError::MaterializationConflictsWithFixedTiming`; an -explicit `MatAtIngestion` is never silently moved or dropped. A root may be -`MatAtIngestion`, and the query reads its stored output; a root of rows that + an evaluation at query time. A query-time consumer reads the stored output + of a node materialized at ingestion time (`AcrossEvaluations`). + +Applying a completed assignment again gives every node the same properties: the +facade re-applies Stage 3's selection this way for export +(`PlanOutput::physical_properties`). A root may be materialized +at ingestion time, and the query reads its stored output; a root of rows that runs at ingestion time without being stored there fails with `ExecutionDataStateError::UnstoredIngestionRoot`. The pass then validates every edge (rows or exact-accumulator state into a -`SummaryAgg`, state into an evaluation, a timed `MaintainPopulation` under +`SummaryAgg`, state into an evaluation, a `MaintainPopulation` under an `EvaluatePopulation`, no ingestion work reading a query-time value) and rejects a node reached from two consumers that need different timings; `split_shared_by_timing` copies such a sub-DAG for one side before the -assignment is applied. How long a materialized output is kept (retention) is -also Stage 2's decision and is not in the assignment. - -`MaterializationAssignment::with_default(materialization)` gives every sub-DAG -that is not set explicitly `materialization` where the DAG allows it; a -`MatAtIngestion` default over a sub-DAG that can only run at query time leaves -that sub-DAG `NotMaterialized` instead of failing (the default applies to -`SummaryAgg` and `MaintainPopulation` nodes). The test helpers `validate_at_ingestion` and -`planned_data_state` (feature `test-support`) use it to check, at planning time and without keeping -anything, that a candidate stays valid when every sub-DAG is `MatAtIngestion`. +assignment is applied. How much a materialized output retains (retention) is decided +later and is not in the assignment. + +A node the assignment does not set is not materialized. The test helper +`PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed(roots)` +(feature `test-support`) sets every `SummaryAgg`, `SummaryTumblingWindowChain` +and `MaintainPopulation` materialized at ingestion time where the DAG allows it (not +filtered, nothing below it that can only run at query time). The test +helpers `validate_at_ingestion` and `planned_data_state` use it to check, at +planning time and without materializing anything, that a candidate stays valid +when every sub-DAG is materialized at ingestion time. ## Exported DAG @@ -188,17 +191,20 @@ children as edges and no embedded sub-DAGs: expressions (`scalar(v)`). Every edge records the intermediate schema, the producer's data state and grouping/window compatibility. - Each node records `output_state` (timing plus `Raw` or `SummaryState`), - `output_schema` and `guarantee`. Export reads the timing written by - `apply_materialization_timings` and rejects an untimed node + `output_schema` and `guarantee`. Export reads each node's timing and lifetime + from the assignment `apply_physical_properties` completed and rejects a node + it does not list (`ExecutionDataStateError::UntimedNode`); it does not re-run data-state validation. -Each node also records its `materialization`, including the medium, so the -deployment knows which outputs to store and where; validation rejects a -`SummaryAgg` or `MaintainPopulation` whose timing differs from the one its -materialization implies, and any `MatAtIngestion` or `MatAtQuery` node at the -other timing. Ingestion work cannot depend on a future query result. The -deployment executes the plan as given and does not change its materialization +Each node also records its `lifetime`, including the medium, so the +deployment knows which outputs to materialize, for how long and where; validation +rejects a timing and lifetime that cannot go together (an ingestion-time +output materialized for `OneEvaluation`) and a `SummaryAgg` or `MaintainPopulation` +that is not materialized outside query time +(`PhysicalASAPDAGValidationError::InvalidPhysicalProperties`). Ingestion +work cannot depend on a future query result. The +deployment executes the plan as given and does not change its lifetime or timing ([#509 Assumption 8](../proposals/planner-layering.md#assumptions)); it must separately check that it has an implementation and a valid data source for each placement. @@ -231,7 +237,7 @@ failure probabilities are combined, and the score guarantee remains in the membership guarantee's child provenance. An exact request does not accept this approximate output path merely because its selected identities are certified. -The materialization assignment decides whether these summaries run at +The physical properties assignment decides whether these summaries run at ingestion time or at query time, and the exported DAG carries that placement. Either placement must give each evaluation a complete rate window and an isolated summary state, or maintain an equivalent replacement diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index cf597bf16..d0da2d084 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -39,22 +39,21 @@ The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG, the language-independent query semantics before summary selection. Both are logical `OperatorNode` DAGs. `compile_physical_asap_dag` exports the selected DAG as a `PhysicalASAPDAG`, which is the Physical Plan Compiler's input. Its per-node execution phase (ingestion or query time) is -decided by a `MaterializationAssignment`, as the layer contract below states. +decided by a `PhysicalPropertiesAssignment`, as the layer contract below states. ### Layer contract 1. **Logical Post-ASAP** (`CandidateLogicalASAPDAGs`) decides what to compute: summary families, readouts and sharing. It does not decide placement; timing that a realization strategy writes while building a candidate is provisional. -2. **Materialization** (Stage 2, #509) chooses `NotMaterialized`, - `MatAtIngestion(medium)`, `MatAtQuery(medium)` or - `MatAtQueryKeptAcrossEvaluations(medium)` for each sub-DAG whose - output may be stored (a `SummaryAgg` or a `MaintainPopulation`) and records - it in a `MaterializationAssignment`. `apply_materialization_timings` writes - every node's `ExecutionTiming` and `Materialization`: a `MatAtIngestion` - state and all of its inputs run at ingestion time; readouts, other - consumers, and the other states run at query time. The default assignment - is `NotMaterialized` everywhere; `PlanOutput::physical_asap_dag` applies it +2. **Materialization** (Stage 2, #509) chooses, for each sub-DAG whose + output may be stored (a `SummaryAgg` or a `MaintainPopulation`), whether + it is materialized and with which timing and `Lifetime` (`None`, + `OneEvaluation(medium)` or `AcrossEvaluations(medium)`), and records it in + a `PhysicalPropertiesAssignment`. `apply_physical_properties` writes every + node's `ExecutionTiming` and `Lifetime`: a state materialized at ingestion time and + all of its inputs run at ingestion time; readouts, other consumers, and the + other states run at query time. The default assignment materializes nothing; `PlanOutput::physical_asap_dag` applies it only to nodes the optimization pass left untimed (a pass's own timing, such as Stage 3's ingestion-time tumbling windows, is kept). 3. **Physical compile** (executor) reads the plan's materialized views: each @@ -97,7 +96,7 @@ and a feasible candidate that loses on cost. Absence is not a cost comparison. For `sum by(job)(rate(m[1m]))`, Rate remains per series before grouped Sum. `CandidateLogicalASAPDAGs` offers one such candidate, with a per-series Rate state and a grouped -Sum state. Its materialization assignment places it: an ingestion-time Sum state +Sum state. Its physical properties assignment places it: an ingestion-time Sum state finalizes Rate and builds Sum within a bounded precompute run; a query-time Sum over an ingestion-time Rate state leaves the Rate readout and Sum in the query DAG. Storing a value requires its exact evaluation window, revision, readiness and serving @@ -113,7 +112,7 @@ Planner's candidate space decides what to compute, not placement. For an instant-vector PromQL TopK, Planner resolves rows that carry the complete series identity and lists the current-series heap realizations per root with the other candidates, unranked. Precompute or query-time placement of Rate and grouped Sum -is not a separate Planner candidate: the materialization assignment sets +is not a separate Planner candidate: the physical properties assignment sets each node's timing, and the physical compiler reads it. This is the target ownership contract. A backend path that still reconstructs @@ -473,8 +472,8 @@ These derivations are methods on `PhysicalASAPDAG` in `asap-types`, so a runtime needs no dependency on the planner crate. A **materialized view** is a node whose output is kept across evaluations -(`MatAtIngestion` or `MatAtQueryKeptAcrossEvaluations`). A `MatAtQuery` node is -not a view: its output is discarded after one evaluation. Everything a runtime +(lifetime `AcrossEvaluations`, at ingestion or at query time). A node materialized for +`OneEvaluation` is not a view: its output is discarded after one evaluation. Everything a runtime needs that the DAG determines is derived from the DAG, so the plan file has no second copy to keep consistent. Stage 3's selection report is returned beside the plan, not inside it. diff --git a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md index 0975ba578..ffb87c859 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md +++ b/docs/design_docs/proposals/asap-aware-mapping/maintained-populations.md @@ -79,9 +79,9 @@ input `EvaluatePopulation` runs at query time. `MaintainPopulation` has no fixed timing: like a `SummaryAgg`, it follows its materialization -(`NotMaterialized`, `MatAtIngestion(medium)` or `MatAtQuery(medium)`, #509 +(`Lifetime::None`, `(IngestionTime, AcrossEvaluations(medium))` or `(QueryTime, OneEvaluation(medium))`, #509 §Materialization), which Stage 2 chooses; it runs at ingestion time only when -it is `MatAtIngestion`. +it is ingestion-time `AcrossEvaluations`. The rule examines compatible workload roots, sets `max_k` to the largest requested k and enables quantile readout if any consumer needs it. It emits a candidate for diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index ee15ead49..4bf59e9ea 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -296,7 +296,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **When two definitions are equal.** Merging (§5.6) requires equal definitions. -- They must have the same structure. Planning details are ignored: `timing`, `guarantee` and `coverage_cache`. So a tumbling window built at ingestion time can merge with one built at query time. +- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. - `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. - States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. @@ -921,9 +921,6 @@ pub struct OperatorNode { /// The accuracy statement of the output, once known. `None` does not /// mean exact. pub guarantee: Option, - /// When the node runs: `IngestionTime` or `QueryTime`. `None` until - /// planning assigns it. - pub timing: Option, /// Cache for `coverage()`. Filled on first use, never serialized, /// ignored by equality, emptied on clone. Not a source of truth: /// coverage can always be derived again. diff --git a/docs/design_docs/proposals/joint-planning-implementation-plan.md b/docs/design_docs/proposals/joint-planning-implementation-plan.md index e561406ac..89b8dde7b 100644 --- a/docs/design_docs/proposals/joint-planning-implementation-plan.md +++ b/docs/design_docs/proposals/joint-planning-implementation-plan.md @@ -12,7 +12,7 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc | Scope | Current implementation | Responsibility | | --- | --- | --- | -| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `generate_dag_of_logical_memo_groups`), stable initial query/group references. | +| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`), stable initial query/group references. | | Computation and sharing rules | `logical-asap-optimization/src/pass2/memo/{candidates,rules,state_groups,shared_configurations,window_candidates}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | | Physical legality | `physical-asap-optimization` | Executable operators, timing, materialization eligibility, and `PhysicalWorkloadPlan`. | | Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning, the selected DAG (`DagOfSelectedCandidates`, one node per run of a candidate), and reconstruction. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | @@ -23,7 +23,7 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc `PhysicalCandidate` is a local selectable computation. `PhysicalWorkloadPlan` is a complete Stage 2 workload DAG; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. -Pass 1 has one entry, `generate_per_computation_logical_memo_candidates`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: +Pass 1 has one entry, `pass1_memo`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: ```text G95 = { ExactP95, LocalKLLQuantileP95, EstimateP95(G_KLL) } diff --git a/docs/design_docs/proposals/joint-planning-search.md b/docs/design_docs/proposals/joint-planning-search.md index 713a7e356..2558a5341 100644 --- a/docs/design_docs/proposals/joint-planning-search.md +++ b/docs/design_docs/proposals/joint-planning-search.md @@ -125,17 +125,17 @@ G_KLL += MergeLatest15(G_tumbling_windows) Non-summary query-time materialization must be a query output or have a selected immediate consumer used by only one query; the materialized node itself must have multiple query readers. These connection-dependent constraints match Stage 2's highest-shared-node rule and are enforced in MILP. Summary-state eligibility remains separate. -Physical rules provide executable operators, configurations, and materialization assignments for the generated computations. For example, a supported tumbling window implementation can maintain state at ingestion and merge it at evaluation. This introduces two timing decisions within the same query, rather than one materialization choice for the whole query. Eligibility that depends on consumers is checked against the selected connections; considering every potential consumer at once could incorrectly reject a useful candidate. +Physical rules provide executable operators, configurations, and physical properties assignments for the generated computations. For example, a supported tumbling window implementation can maintain state at ingestion and merge it at evaluation. This introduces two timing decisions within the same query, rather than one materialization choice for the whole query. Eligibility that depends on consumers is checked against the selected connections; considering every potential consumer at once could incorrectly reject a useful candidate. For the running example, assume evaluations occur at aligned minute boundaries and the deployment supports ingestion-time tumbling window maintenance. Stage 2 can expand the state groups as follows; `s` is a generated KLL capacity: ```text -Group Physical candidate Computation Materialization -G_tumbling_windows P_ingest(s) Maintain aligned 1m tumbling windows MatAtIngestion(Memory) -G_KLL K_merge(s) MergeLatest15(G_tumbling_windows) MatAtQuery(Memory) -G_KLL K_build(s) BuildKLL(G_input, 15m) MatAtQuery(Memory) -G95 E95 EstimateQuantile(.95, G_KLL) NotMaterialized -G99 E99 EstimateQuantile(.99, G_KLL) NotMaterialized +Group Physical candidate Computation (timing, lifetime) +G_tumbling_windows P_ingest(s) Maintain aligned 1m tumbling windows (IngestionTime, AcrossEvaluations(Memory)) +G_KLL K_merge(s) MergeLatest15(G_tumbling_windows) (QueryTime, OneEvaluation(Memory)) +G_KLL K_build(s) BuildKLL(G_input, 15m) (QueryTime, OneEvaluation(Memory)) +G95 E95 EstimateQuantile(.95, G_KLL) (QueryTime, None) +G99 E99 EstimateQuantile(.99, G_KLL) (QueryTime, None) ``` `P_ingest(s)` retains the tumbling windows needed to cover the 15m window. `K_merge(s)` constructs window state from those tumbling windows at evaluation; `K_build(s)` instead builds window state from available raw input. Each candidate records its input requirements and costs. The estimators can use either compatible state candidate. Exact and local KLL candidates from Pass 1 also receive supported physical implementations and remain available. Stage 2 has generated choices; it has not yet selected a build or connected an estimator to it. @@ -189,7 +189,7 @@ The following cases make the accounting concrete. Let `B` be the cost of one who | Repeated common evaluations | Both queries evaluate together `n` times; state is discarded after each evaluation. | `n × (B + E95 + E99)`. Selecting one candidate does not mean building only once over the entire horizon. | | Different evaluation timestamps | Q1 evaluates every minute; Q2 every five minutes on the same grid. Both use a 15m window. | Over 15 evaluation ticks, a shareable query-local build executes 15 times, with 15 P95 extractions and 3 P99 extractions: `15B + 15E95 + 3E99`. This assumes sharing at the three common timestamps; an earlier window state cannot automatically answer a later window. | | Ingestion-maintained tumbling windows | One tumbling-window chain feeds a shared query-time merge and both estimators. | Ingestion updates and tumbling-window retention are charged once for the chain. Add `n × (M + E95 + E99)` for `n` common evaluations, plus query-local merge memory. | -| State retained across evaluations | An eligible tumbling-window-chain candidate uses `MatAtQueryKeptAcrossEvaluations(Memory)`. | Charge initial construction, required refresh/update work, retained memory/storage over its lifetime, and evaluation work. Reuse does not eliminate freshness maintenance; generate this case only when supported. | +| State retained across evaluations | An eligible tumbling-window-chain candidate uses `(QueryTime, AcrossEvaluations(Memory))`. | Charge initial construction, required refresh/update work, retained memory/storage over its lifetime, and evaluation work. Reuse does not eliminate freshness maintenance; generate this case only when supported. | | Multiple selected candidates in one group | Some consumers use a small KLL state and others a separate large state. | Charge each selected state's build/update and retention, then each consumer's local work. Equal group identity does not collapse two physical candidates into one charge. | | Shared raw-data retention | Two consumers need 5m and 15m of the same raw source and compatible retention scope. | Both consumers connect to `Retain15m` in `G_raw[d]`. Charge that candidate once; reads and intermediate KLL state have separate owners. | | Per-request overhead | Two requests share computation but still require separate response handling. | Report fixed overhead as `Σ_q N_q × h_q`; omit this constant from the optimization objective but retain its applicable resource usage. Decision-dependent overhead stays in the optimization model. | diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 5f9deb85f..cb822d7a6 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -1,6 +1,9 @@ # Sharing Operators Between Pre-ASAP IR and Post-ASAP IR > Status: proposal, not implemented. Audience: planner designers and architects. +> Note: since #1200, operator nodes carry no `timing`; physical stages keep each +> node's timing and lifetime in a `PhysicalPropertiesAssignment`, and +> `validate_execution_timing` no longer exists. The `timing` text below predates that. > Addresses [#468](https://github.com/ProjectASAP/ASAPPlanner/issues/468). > Companion: [Decoupling operators from scalar expressions](decoupling_op_and_expr.md). diff --git a/docs/design_docs/proposals/planner-layering-example1-acceptance.md b/docs/design_docs/proposals/planner-layering-example1-acceptance.md index c66cd3aa6..0dd10ff90 100644 --- a/docs/design_docs/proposals/planner-layering-example1-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example1-acceptance.md @@ -19,7 +19,7 @@ whose expectation needs a feature that is not in the stack yet is |---|---|---| | Query-language frontends | 1 workload `LogicalDAG` | same | | Logical ASAP-aware optimization | Pass 1 (3) × identical-expression rule (2) × window-composition rule (3 × 3) = **54** | Pass 1 (3), each with 2 identical-expression candidates and 6 window-composition candidates (Q1's rate and Q2's option in six 10-s tumbling windows, alone or both, each also with a shared scan) = **27**. Sliding windows are not implemented. | -| Physical ASAP-aware optimization | Materialization options per window form = **156** | Physical operator implementation, plus materialization: each logical candidate with every sub-DAG `NotMaterialized` (27: the doc's Raw plans, with tumbling windows rebuilt at every refresh), each tumbling-window chain also `MatAtIngestion(Memory)` (30 more), and each shared input built once per evaluation, `MatAtQuery(Memory)`, with every tumbling-window chain at query time (15 more) = **72** | +| Physical ASAP-aware optimization | Physical properties options per window form = **156** | Physical operator implementation, plus materialization: each logical candidate with every sub-DAG `Lifetime::None` (27: the doc's Raw plans, with tumbling windows rebuilt at every refresh), each tumbling-window chain also `(IngestionTime, AcrossEvaluations(Memory))` (30 more), and each shared input built once per evaluation, `(QueryTime, OneEvaluation(Memory))`, with every tumbling-window chain at query time (15 more) = **72** | | Plan selection | 1 plan | 1 plan | Every MVP candidate is one of the doc's candidates: in logical ASAP-aware @@ -100,26 +100,26 @@ The three whole-window candidates without a shared input: | Id | From | Q2 physical operators | Materialization | |---|---|---|---| -| P1 | L1 | sort (partition by job) → limit 10 | all `NotMaterialized` | -| P10 | L10 | CMS + heap build → top-10 estimate | all `NotMaterialized` | -| P19 | L19 | CountSketch + heap build → top-10 estimate | all `NotMaterialized` | +| P1 | L1 | sort (partition by job) → limit 10 | all `Lifetime::None` | +| P10 | L10 | CMS + heap build → top-10 estimate | all `Lifetime::None` | +| P19 | L19 | CountSketch + heap build → top-10 estimate | all `Lifetime::None` | Q1 is the same in all three: scan → range → rate accumulator → sum accumulator, each finalized. Invariants: -* Every logical candidate keeps its all-`NotMaterialized` physical candidate, +* Every logical candidate keeps its all-`Lifetime::None` physical candidate, and none is pruned, so no candidate is dropped before plan selection. A logical candidate with `n` tumbling-window chains has 2ⁿ physical candidates: each - chain `NotMaterialized` or `MatAtIngestion(Memory)` (ids `P-m`). + chain `Lifetime::None` or `(IngestionTime, AcrossEvaluations(Memory))` (ids `P-m`). Only tumbling-window chains are placed at ingestion time; Q1's rate tumbling windows may be, because continuously ingesting PromQL data is taken to arrive in time order per series. A logical candidate with a shared input has one more: the highest node both queries read (the scan, or the time range above - it) `MatAtQuery(Memory)`, with every chain at query time. A chain at + it) `(QueryTime, OneEvaluation(Memory))`, with every chain at query time. A chain at ingestion time leaves one query reading the input at query time, so - the input stays `NotMaterialized` there. + the input stays `Lifetime::None` there. * Each physical candidate keeps its logical Q2 option and input sharing. * Exact top-*k* is implemented as a sort followed by a limit: each job's series sorted by the ranked value, descending, then a `Project` that @@ -128,7 +128,7 @@ Invariants: returns each job's `k` heaviest series, and a `Project` to `[job, topk_10]` followed by each ranked series' identity. There is no merge node, because there are no window summaries. -* In the whole-window candidates, every node is `NotMaterialized` and runs +* In the whole-window candidates, every node is `Lifetime::None` and runs at query time. ## Plan selection: 1 plan @@ -137,7 +137,7 @@ Invariants: |---|---|---| | P3-m1 | selected | all exact over one time range built once per evaluation for both queries, at 4.38 per second (cost model 1) | | P2-m1 | valid · costlier | all exact over one scan built once per evaluation, at 4.78 per second: each query still reads its own 1 m time range | -| P1, P2, P3 | valid · costlier | all exact at 7.01 per second, with separate inputs (P1) or a `NotMaterialized` shared scan (P2) or scan and range (P3), charged once per query; they tie with each other up to rounding | +| P1, P2, P3 | valid · costlier | all exact at 7.01 per second, with separate inputs (P1) or a `Lifetime::None` shared scan (P2) or scan and range (P3), charged once per query; they tie with each other up to rounding | | P10 | invalid | Count-Min + heap: a top-*k* output has no guarantee (a heap's bound is per item frequency, not top-*k* membership), and the deployment has no evidence that it meets the target | | P19 | invalid | CountSketch + heap: the same reason | @@ -155,7 +155,7 @@ Invariants: reason, and marked invalid or valid but costlier. * Only valid candidates are priced. Each valid candidate's cost has one entry per DAG node, and `cost_per_second` is their sum. A materialized node is charged - once. A `NotMaterialized` node is charged once per consumer that reads it + once. A `Lifetime::None` node is charged once per consumer that reads it (its multiplier), where a consumer is a query or a materialized node and a read stops at materialized nodes. `stage3_plan_selection_charging_helper_*` check the helper on a hand-built candidate with a @@ -185,13 +185,13 @@ per evaluation, within its 100 ms. deployment evidence every heap candidate is invalid and the exact plan is selected. 4. **Physical ASAP-aware optimization is materialization in the doc's - example.** Example 1 lists only window-form and materialization options. + example.** Example 1 lists only window-form and physical properties options. The planner also reads "physical operator implementation" from the physical ASAP-aware optimization section ("TopK as a sort followed by a limit"), and enumerates materialization for summary builds and shared - inputs: each tumbling-window chain is `NotMaterialized` or `MatAtIngestion(Memory)`, - and the highest node both queries read is `NotMaterialized` or - `MatAtQuery(Memory)`. + inputs: each tumbling-window chain is `Lifetime::None` or `(IngestionTime, AcrossEvaluations(Memory))`, + and the highest node both queries read is `Lifetime::None` or + `(QueryTime, OneEvaluation(Memory))`. 5. **Where exact top-*k* becomes sort + limit.** The Pass 1 diagram draws exact Q2 as "sort + limit 10 per job", but the frontend emits `Aggregate{[TopK]}` and the physical ASAP-aware optimization section calls @@ -201,9 +201,9 @@ per evaluation, within its 100 ms. column" without naming the inner sketch. The test assumes `HydraCms`. 7. **"Shared summary charged once"** does not apply literally: the doc says no summary is shared in Example 1. Plan selection charges a - `NotMaterialized` node once per consuming query and a materialized one - once; the tests check that rule. A `NotMaterialized` shared input - therefore saves nothing here; built once per evaluation (`MatAtQuery`), + `Lifetime::None` node once per consuming query and a materialized one + once; the tests check that rule. A `Lifetime::None` shared input + therefore saves nothing here; built once per evaluation (`OneEvaluation`), it is read once, and that candidate (the doc's "Raw with a shared input") is selected. 8. **Latency.** The doc says an exact top 10 rebuilt at every refresh "may diff --git a/docs/design_docs/proposals/planner-layering-example2-acceptance.md b/docs/design_docs/proposals/planner-layering-example2-acceptance.md index cbb2214e7..0ec29bd09 100644 --- a/docs/design_docs/proposals/planner-layering-example2-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example2-acceptance.md @@ -48,8 +48,8 @@ workload itself. | Query-language frontends | One summary-free DAG: `Distinct`, `Entropy`, `L2` over `flows.src_ip`, last 1 m | Same. Q1 is a distinct count; Q2 is `Entropy` and Q3 is `L2` over the per-source counts, each with an exact population count for the empty-input NULL. | | Logical ASAP-aware optimization, Pass 1 | Each computation: exact, a specialized summary, UnivMon. 3 × 3 × 3 = **27** | Q1: exact, HLL, KMV, UnivMon (no Theta: `asap_sketchlib` has no Theta sketch). Q2, Q3: exact, UnivMon (no specialized entropy or norm summary). 4 × 2 × 2 = **16** | | Logical ASAP-aware optimization, Pass 2 | Summary-capability rule: + 1 (one UnivMon for all three, sized for ε = 0.01) + 9 (3 pairs share a UnivMon × 3 options of the third). Independent kept. **37**. Window composition left out. | Identical-expression rule (Q2 and Q3's shared per-source count, and the shared scan) and summary-capability rule (one UnivMon for two of the queries or all three, sized with each parameter's largest value among its readers): **82** | -| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | Each logical candidate with every sub-DAG `NotMaterialized`, plus `MatAtQuery(Memory)` for each summary build two or more queries read: **184** | -| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | 164 invalid: HLL or UnivMon for Q1 (120), whose bound has failure probability None without a distinct bound for `flows`, and UnivMon for Q2 (44), whose entropy has no guarantee. 20 valid. Under cost model 1, KMV for Q1 with the other two exact (P33) costs 6.097 per second and P1, all exact, 6.104, the exact distinct count's values held while Q1 runs; the shared per-source count costs 7.46 rebuilt for each query and 7.06 kept `MatAtQuery`; with the shared scan also built once per evaluation, KMV for Q1 (P35-m3) is selected at 5.745, and all exact costs 5.752; UnivMon for Q3 costs 8.09. | +| Physical ASAP-aware optimization | Same options as Example 1: no window form → rebuilt from raw samples each refresh. | Each logical candidate with every sub-DAG `Lifetime::None`, plus `(QueryTime, OneEvaluation(Memory))` for each summary build two or more queries read: **184** | +| Plan selection | One UnivMon sized for ε = 0.01 vs three separate summaries; the shared one usually wins (one update per flow record instead of three). | 164 invalid: HLL or UnivMon for Q1 (120), whose bound has failure probability None without a distinct bound for `flows`, and UnivMon for Q2 (44), whose entropy has no guarantee. 20 valid. Under cost model 1, KMV for Q1 with the other two exact (P33) costs 6.097 per second and P1, all exact, 6.104, the exact distinct count's values held while Q1 runs; the shared per-source count costs 7.46 rebuilt for each query and 7.06 materialized `OneEvaluation`; with the shared scan also built once per evaluation, KMV for Q1 (P35-m3) is selected at 5.745, and all exact costs 5.752; UnivMon for Q3 costs 8.09. | ## Invariants and tests @@ -84,12 +84,12 @@ workload itself. | Invariant | Test | Status | |---|---|---| -| Each logical candidate keeps one physical candidate with nothing materialized, plus one per other materialization assignment; none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | +| Each logical candidate keeps one physical candidate with nothing materialized, plus one per other physical properties assignment; none pruned | `stage2_physical_asap_optimization_keeps_every_logical_candidate` | passes | | The shared UnivMon stays one build read by all three queries | `stage2_physical_asap_optimization_keeps_the_shared_univmon_as_one_build` | passes | Window-composition candidates are out of scope: the doc leaves them out for this example, and a summary with no window form is rebuilt at every refresh -(Example 1's table) unless a shared build is kept `MatAtQuery` within one +(Example 1's table) unless a shared build is materialized `OneEvaluation` within one evaluation. ### Plan selection @@ -98,7 +98,7 @@ evaluation. |---|---|---| | One selected id; every other candidate rejected once with a reason | `stage3_plan_selection_selects_one_and_explains_the_rest` | passes | | The selected plan is the cheapest valid one | `stage3_plan_selection_selects_cheapest_valid` | passes | -| Each node of a valid candidate is charged once per consuming query while `NotMaterialized`, once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | +| Each node of a valid candidate is charged once per consuming query while `Lifetime::None`, once when materialized | `stage3_plan_selection_charges_each_node_per_consuming_query` | passes | | No candidate builds a Theta sketch | `stage3_plan_selection_has_no_theta_candidate` | passes | | A UnivMon estimate has a guarantee, so it is judged against its target instead of rejected for having none | `stage3_plan_selection_judges_univmon_with_an_accuracy_model` | ignored: needs a UnivMon entropy accuracy model (#788); L2 and the distinct count have one | | Given a distinct bound for the source, classic HLL's failure probability is bound and Q1's HLL is accepted; a larger bound enlarges its precision, also in a tumbling-window chain | `stage3_plan_selection_accepts_hll_given_a_distinct_bound`, `…_enlarges_hll_for_a_larger_distinct_bound`, `…_enlarges_tumbling_window_hll` | passes | @@ -112,8 +112,8 @@ The doc's "the shared candidate usually wins" depends on the cost model, so it is tested only as the dominance above: one build sized for the strictest consumer costs no more than three builds that include one of that size. Whether it also beats the exact plans is the model's call. The dominance needs the -shared UnivMon materialized: plan selection charges a `NotMaterialized` -node once per consuming query, so unmaterialized it costs three builds of the +shared UnivMon materialized: plan selection charges a `Lifetime::None` +node once per consuming query, so not_materialized it costs three builds of the strictest size. ## Ambiguities and conflicts with the planner diff --git a/docs/design_docs/proposals/planner-layering-example3-acceptance.md b/docs/design_docs/proposals/planner-layering-example3-acceptance.md index f6f253b9d..8966edd70 100644 --- a/docs/design_docs/proposals/planner-layering-example3-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example3-acceptance.md @@ -47,7 +47,7 @@ Shared data workload, except Pattern A's `arrival` is `Mixed`. | Query-language frontends | A: five scan → range → quantile chains. B: one. | Same; q3–q5 also have a `time_shift`. | | Logical ASAP-aware optimization, Pass 1 | A: exact or KLL over its own interval per query, 2⁵ = 32. B: exact, KLL. | A: exact, KLL (k = 547), DDSketch per query, 3⁵ = **243**. B: exact, KLL (k = 269), DDSketch, **3**. | | Logical ASAP-aware optimization, Pass 2 | A: window composition adds one EH of KLLs over [T − 5 y, T], one merge + p99 estimate per query, plus a candidate for every grouping of two or more queries onto shared window summaries ("hundreds"). B: each Pass 1 option × {none, sliding L = 5 min s = 1 min (5 active windows), 1-min tumbling (merge 5)} = 6. Independent kept. | Identical-expression rule: each Pass 1 candidate also gets its shared-input candidates (A: 3 per Pass 1 candidate; B: none). Window-composition rule, tumbling windows only: B's KLL and DDSketch each also come as 1-min tumbling windows (a `SummaryTumblingWindowChain` of 5, merged before the p99 estimate), **5** in all; A is run once, so it gets none. No sliding windows or EHs. | -| Physical ASAP-aware optimization | See Example 4. | Each logical candidate with every sub-DAG `NotMaterialized` (A: 972; B: 5), plus, for A, each highest shared input (a scan or a shift) also built once per evaluation, `MatAtQuery(Memory)` (A: 2,187 in all), and, for B, each tumbling-window chain maintained `MatAtIngestion(Memory)` (B: 7 in all). | +| Physical ASAP-aware optimization | See Example 4. | Each logical candidate with every sub-DAG `Lifetime::None` (A: 972; B: 5), plus, for A, each highest shared input (a scan or a shift) also built once per evaluation, `(QueryTime, OneEvaluation(Memory))` (A: 2,187 in all), and, for B, each tumbling-window chain maintained `(IngestionTime, AcrossEvaluations(Memory))` (B: 7 in all). | | Plan selection | Not stated for Example 3. | A: 288 valid, 1,899 invalid: every candidate with a DDSketch misses the rank target (its bound is relative). KLL for all five queries (P485) is selected at 207,189.43 per second (cost model 1); all exact costs 327,132,784.05, about 1,600 times as much, its values held for the whole of each long query. B: KLL (P2, 2.363 per second); the 1-min KLL tumbling windows rebuilt at every evaluation cost 3.611 (the same rows and updates plus merging 5 tumbling windows per series, and the tumbling windows held while the query runs); maintained at ingestion time they keep six tumbling windows of 1,000,000 per-series KLLs for 654.54; exact costs 2.458; DDSketch, whole or in tumbling windows, misses the rank target. Every valid B plan meets 200 ms: 126 ms for the whole-window KLL, 131 ms for the rebuilt tumbling windows, 7 ms for the maintained tumbling windows, 144 ms exact. | ## Invariants and tests @@ -66,7 +66,7 @@ Shared data workload, except Pattern A's `arrival` is `Mixed`. | The EH candidate coexists with the five independent KLLs | `stage1_logical_asap_optimization_a_keeps_independent_and_shared_window_summaries` | ignored: #628, #634 | | Every pair of queries shares a window summary in some candidate | `stage1_logical_asap_optimization_a_window_composition_groups_every_pair` | ignored: #628, #634 | | One selected id, the rest explained; the selected plan is the cheapest valid one | `stage3_plan_selection_a_selects_cheapest_valid` | passes | -| Each node of a valid candidate is charged once per consuming query while `NotMaterialized`, once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | +| Each node of a valid candidate is charged once per consuming query while `Lifetime::None`, once when materialized | `stage3_plan_selection_a_charges_each_node_per_consuming_query` | passes | | For the same per-computation choices, sharing the scan costs no more than separate scans | `stage3_plan_selection_a_shared_scan_is_not_costlier` | ignored: #632 (see ambiguity 7) | ### Pattern B diff --git a/docs/design_docs/proposals/planner-layering-example4-acceptance.md b/docs/design_docs/proposals/planner-layering-example4-acceptance.md index ae9371317..3565c9d27 100644 --- a/docs/design_docs/proposals/planner-layering-example4-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example4-acceptance.md @@ -50,12 +50,12 @@ time, both kept. Not materializing it is the no-window plan, so it is not a separate option. **Today:** Pass 2 builds tumbling windows (`SummaryTumblingWindowChain`), and physical -ASAP-aware optimization enumerates `NotMaterialized`, `MatAtIngestion` and -`MatAtQuery` (state reused within one evaluation) for every summary build. +ASAP-aware optimization enumerates `Lifetime::None`, ingestion-time `AcrossEvaluations` and +`OneEvaluation` (state reused within one evaluation) for every summary build. B1 and B2 exist. B3 (tumbling windows built at query time and kept across evaluations) is offered when the deployment declares `query_time_results_retained` in its `deployment_capabilities`: Stage 2 then -adds `MatAtQueryKeptAcrossEvaluations` for a tumbling-window chain that builds +adds query-time `AcrossEvaluations` for a tumbling-window chain that builds one new tumbling window per evaluation. Pattern A's shared EH needs window segments shared across queries (#628, #634), so A1–A3 do not exist yet. B3's 4 kept tumbling windows plus the newest cost 436.59 per second under cost model 1, @@ -63,12 +63,12 @@ between B2 (3.611) and B1 (654.54). ## Invariants and tests -**Reading materialization.** The physical DAG records each node's -`Materialization` (#667). Tests read it through -`planner_layering_common::materialization(p, node)` without the storage -medium (`MatAtIngestion`, `MatAtQuery`, `NotMaterialized`), and the kept +**Reading physical properties.** The physical DAG records each node's +timing and `Lifetime` (#667, #1200). Tests read them through +`planner_layering_common::materializing(p, node)` without the lifetime length or +storage medium (`MaterializedAtIngestion`, `MaterializedAtQueryTime`, `NotMaterialized`), and the materialized event-time span through `retention_ms(p, node)`. The IR does not record -retention, so the adapter derives it the way plan selection prices kept +retention, so the adapter derives it the way plan selection prices materialized state: a chain of `N` tumbling windows of width `w` keeps `N·w`, plus one tumbling window while maintained at ingestion time; a whole-window build keeps its window, plus the next one at ingestion time; each plus the offset of the shifts below @@ -89,7 +89,7 @@ it. | The shared EH yields exactly A1, A2, A3 | `stage2_physical_asap_optimization_a_shared_eh_has_three_materialization_options` | ignored: the shared EH (#628, #634) | | At rest, A2 is not generated; A1 and A3 remain | `stage2_physical_asap_optimization_a_at_rest_drops_the_ingestion_time_option` | ignored: the shared EH (#628, #634) | | A materialized EH is one build node read by all five queries, priced once | `stage2_physical_asap_optimization_a_materialized_eh_is_built_once_for_all_consumers` | ignored: the shared EH (#628, #634) | -| A3 costs strictly more than A1 (five builds instead of one: a `NotMaterialized` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: the shared EH (#628, #634) | +| A3 costs strictly more than A1 (five builds instead of one: a `Lifetime::None` node is charged per consuming query) | `stage3_plan_selection_a_rebuilding_per_query_costs_more_than_building_once` | ignored: the shared EH (#628, #634) | | As given, A1 is the cheapest of the three | `stage3_plan_selection_a_once_adhoc_prefers_the_query_time_eh` | ignored: the shared EH (#628, #634) | | Monthly, A2's cost relative to A1 drops: cost(A2)/cost(A1) monthly < as given | `stage3_plan_selection_a_monthly_amortizes_ingestion_time_maintenance` | ignored: the shared EH (#628, #634) | @@ -127,11 +127,11 @@ it. 1. **Charging A3 (decided).** A3 is one logical EH node that is not materialized, so it is rebuilt by each of its five consumers. Plan - selection charges a `NotMaterialized` node once per consuming query + selection charges a `Lifetime::None` node once per consuming query (`NodeCost::multiplier`) and a materialized node once; physical ASAP-aware optimization does not duplicate the node. So A3 costs five EH builds and A1 one, as the tests require. The A3 tests stay ignored only because the - shared EH and its materialization options do not exist yet. + shared EH and its physical properties options do not exist yet. 2. **Recurrence input (resolved).** Materialization depends on `recurrence`, `predictability` and `arrival`: `optimize_physical_asap` takes the workload entries (`QueryWorkloadEntry`: recurrence, diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index cb93a3acb..3c0678a33 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -281,7 +281,7 @@ lists each stage's input and output types. one. It makes no implementation choice, so this section counts it as Stage 0 (its output); the code runs it as Stage 1 starts, which is why [Stage inputs and outputs](#stage-inputs-and-outputs-whole-workload-candidates-and-memo) -lists `generate_dag_of_logical_memo_groups` with Pass 1. `generate_dag_of_logical_memo_groups` runs +lists `memo_of_queries` with Pass 1. `memo_of_queries` runs `share_common_sub_dags` (`crates/logical-asap-optimization/src/stage0/cse.rs`) over the frontends' roots and numbers the shared nodes as groups. Two structurally equal nodes become one group only when sharing them is legal, that is, when every reader @@ -345,7 +345,7 @@ One query, evaluated every minute, with `max_error 0.01` and avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m])) ``` -**Step 1, Stage 0.** `generate_dag_of_logical_memo_groups` makes 7 groups: +**Step 1, Stage 0.** `memo_of_queries` makes 7 groups: | Group | Logical sub-expression | Reads | |---|---|---| @@ -523,12 +523,12 @@ candidates is in `crates/plan-selection/src/` (`select.rs`, `groups/`, `milp/`). | Step | Entry point | Path | |---|---|---| -| Stage 0 | `generate_dag_of_logical_memo_groups` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | -| Pass 1 | `generate_per_computation_logical_memo_candidates` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | +| Stage 0 | `memo_of_queries` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | +| Pass 1 | `pass1_memo` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | -| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`generate_per_computation_logical_memo_candidates`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | +| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -567,14 +567,14 @@ The sections below describe each stage. | 0. Frontends | nothing: a query that cannot be lowered fails planning with an error | — | — | | 1, Pass 1 | what the measure or the deployment cannot compute: `RequiresExact`, `NullableAvgInput`, `NoRelationalCountSketchRealization`, `UnsupportedByDeployment`, `QuantileOutsideUnitInterval`, `TopKHeapOverOneRowPerItem`; and, only when the deployment's accuracy model and evidence cannot accept an output without its guarantee, a summary no query reading it can use: `NoErrorBoundForQueryTarget` | `PerComputationLogicalMemoCandidates::pruned_candidates`, each on its aggregate's memo group | the way to compute the measure (`HowToComputeAggregateMeasure`) | | 1, Pass 2 | nothing: it only adds candidates | — | — | -| 2 | what cannot be implemented, built or kept: `PerEntityAggregateNotExecutable` (an exact per-entity `Aggregate` the executor has no kernel for), `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders` (shared within one evaluation with fewer than two reading queries) | `PhysicalASAPMemoCandidates::pruned_candidates`, each on its operator node's memo group | `PhysicalCandidateKey { operator_node, timing, materialization }` | +| 2 | what cannot be implemented, built or materialized: `PerEntityAggregateNotExecutable` (an exact per-entity `Aggregate` the executor has no kernel for), `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders` (shared within one evaluation with fewer than two reading queries) | `PhysicalASAPMemoCandidates::pruned_candidates`, each on its operator node's memo group | `PhysicalCandidateKey { operator_node, properties }` | | 3 | valid candidates that lose, and the checks only a whole plan can make (accuracy, latency, cluster capacity) | its own `RejectionCause` | — | Stage 2's key names what a pruned candidate would have been, since kept candidates are renumbered after pruning and a pruned one has no `PhysicalCandidateId`. Stage 2 emits one candidate per operator node, -timing and materialization, and makes no copies of it, so (`operator_node`, -`timing`, `materialization`) is unique. Operator +timing and lifetime, and makes no copies of it, so (`operator_node`, +`properties`) is unique. Operator nodes are kept whole, so `operator_node` indexes the output's `operator_nodes_of_candidates_for_memo_groups`. Pass 1's accuracy pruning (`NoErrorBoundForQueryTarget`) runs inside Pass 1's entry, before the @@ -597,10 +597,10 @@ the memo path's entry points. | Stage | Whole-workload candidates: input → output | Memo: input → output | Entry point (memo path) | |---|---|---|---| | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | -| 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `generate_dag_of_logical_memo_groups`, `generate_per_computation_logical_memo_candidates` | +| 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per materialization assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and materialization (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -611,7 +611,7 @@ What changed with the memo: jointly. - Stage 2 is no longer a pass over whole candidate DAGs. It implements every operator node of every candidate for a memo group once, enumerates each - node's timings and materializations, and prunes what cannot be built, + node's timings and lifetimes, and prunes what cannot be built, used or kept; Stage 3 selects among what remains. - The concrete summary-state groups are expanded by Stage 3's search today, through the Stage 1 functions above. A planned change moves this expansion @@ -844,7 +844,7 @@ tumbling windows keeps the guarantee when the family's merged error is `Exact` o `AsIfBuiltOverUnion`), and a build shared by several readers takes the strictest reader's size, one state group per distinct size. The failure probability is part of Pass 1's size too -(`generate_per_computation_logical_memo_candidates`): each approximate input +(`pass1_memo`): each approximate input of a candidate with `n` budget parts is sized for `δ / (n · V)`, the smallest over the queries reaching its group, where `V` is the number of values the query's accuracy scope covers (1 per value, at most `⌊1 / share⌋` for a record @@ -1198,21 +1198,26 @@ the `DataWorkload`. Typical outcomes: A shared summary is materialized once for all its consumers. See Example 4. -The planner implements a subset of these options today. Stage 2 enumerates +The planner implements a subset of these options today. In code each option +is a timing and a lifetime (`PhysicalProperties`, #1200): materialized at +ingestion time is ingestion time with lifetime `AcrossEvaluations`; +materialized at query time is query time with lifetime `OneEvaluation` +(reused within one evaluation) or `AcrossEvaluations` (materialized for later +evaluations); not materialized is lifetime `None`. Stage 2 enumerates materialization for summary builds (`SummaryAgg` and `SummaryTumblingWindowChain`); each -gets `NotMaterialized`, `MatAtIngestion(Memory)` when it is eligible to run -at ingestion time, and `MatAtQuery(Memory)` when two or more queries read -it. Of every other sub-DAG that two or more queries read (for example a -`Scan` that a 1 m and a 5 m window both read), only the highest node is a -materialization candidate: one that a query's output, or a node only one -query reads, consumes. It gets `NotMaterialized` and `MatAtQuery(Memory)` -only, since raw rows are not maintained at ingestion time. Below an -ingestion-time build it stays `NotMaterialized`: the ingestion side reads -its own copy. A `MatAtQuery` state is reused within one evaluation only. When the -deployment's capabilities declare `query_time_results_retained`, a -tumbling-window chain that builds exactly one new tumbling window per evaluation also gets -`MatAtQueryKeptAcrossEvaluations(Memory)`: the tumbling windows built at query time are -kept for the next evaluations. Nothing is placed on disk yet. +is offered not materialized, materialized at ingestion time `AcrossEvaluations(Memory)` when +it is eligible to run at ingestion time, and materialized at query time +`OneEvaluation(Memory)` when two or more queries read it. Of every other +sub-DAG that two or more queries read (for example a `Scan` that a 1 m and a +5 m window both read), only the highest node is a materialized-output candidate: +one that a query's output, or a node only one query reads, consumes. It is +offered not materialized and `OneEvaluation(Memory)` only, since raw rows are not +maintained at ingestion time. Below an ingestion-time build it stays not +materialized: the ingestion side reads its own copy. When the deployment's +capabilities declare `query_time_results_retained`, a tumbling-window chain +that builds exactly one new tumbling window per evaluation is also offered +query time `AcrossEvaluations(Memory)`: the tumbling windows built at query +time are materialized for the next evaluations. Nothing is placed on disk yet. A runtime builds every ingestion-time state on the grid of its width anchored at the epoch, so an ingestion-time build must end on that grid at every @@ -1243,20 +1248,20 @@ Output: C3 — CandidatePhysicalASAPDAGs 2: for each LogicalASAPDAG d in C2 do 3: // Materialization: options per sub-DAG 4: for each sub-DAG s in d do -5: Opt(s) ← { NotMaterialized } +5: Opt(s) ← { (QueryTime, None) } 6: for each medium in {memory, disk} do -7: Opt(s) ← Opt(s) ∪ { MatAtIngestion(medium), MatAtQuery(medium) } +7: Opt(s) ← Opt(s) ∪ { (IngestionTime, AcrossEvaluations(medium)), (QueryTime, OneEvaluation(medium)) } 8: end for 9: end for 10: 11: for each assignment m in Opt(s_1) × … × Opt(s_n) do 12: // constraint: everything upstream of an ingestion-time node also runs at ingestion time -13: if some s with m(s) = MatAtIngestion has an upstream node u with m(u) ≠ MatAtIngestion then +13: if some s with timing(m(s)) = IngestionTime has an upstream node u with timing(m(u)) ≠ IngestionTime then 14: PRUNE(m, "upstream of ingestion-time node not at ingestion time") 15: continue 16: end if 17: // constraint: keep a materialized output as long as any consumer needs it -18: for each s with m(s) ≠ NotMaterialized do +18: for each s with lifetime(m(s)) ≠ None do 19: retention(s) ← latest time any consumer of s reads it // from recurrence, windows 20: end for 21: // a shared summary is one node, so it is materialized once for all consumers @@ -1299,7 +1304,7 @@ into a node. *Example for Algorithm 3* ([Example 4, Pattern B](#example-4-materialization-of-window-summaries-in-physical-planning)). The logical candidate is the `Tumbling(1m)` KLL option from Algorithm 2. The -diagram shows three of its materialization assignments, each drawn as a copy +diagram shows three of its physical properties assignments, each drawn as a copy of the DAG with every node colored by its materialization. Line 13 enforces one rule: a node can run at ingestion time only if every node it reads from also runs at ingestion time, because otherwise its input does not exist yet when @@ -1328,7 +1333,7 @@ flowchart TB end subgraph LEG["Legend: when a node runs and whether its output is stored"] direction LR - l1["MatAtIngestion"]:::ingest ~~~ l2["MatAtQuery"]:::qtime ~~~ l3["NotMaterialized"]:::notmat + l1["(IngestionTime, AcrossEvaluations)"]:::ingest ~~~ l2["(QueryTime, OneEvaluation)"]:::qtime ~~~ l3["(QueryTime, None)"]:::notmat end M1 --> P1 LEG ~~~ M1 diff --git a/docs/design_docs/proposals/window-materialization-choices.md b/docs/design_docs/proposals/window-materialization-choices.md index 7646956cd..026edabfb 100644 --- a/docs/design_docs/proposals/window-materialization-choices.md +++ b/docs/design_docs/proposals/window-materialization-choices.md @@ -22,5 +22,5 @@ A tumbling window is an aligned, non-overlapping fixed-width interval and its ag This table describes possible strategies. Current tumbling window support does not imply general direct sliding-state or EH support. Offer only executable candidates. Multiple tumbling-window widths can coexist; alignment, coverage, mergeability, retention, and error contracts determine legality. A 15m window evaluated every 1m can use 1m tumbling windows; complete 5m tumbling windows alone generally cannot cover every endpoint. -Use existing `Materialization` options: `NotMaterialized`, `MatAtQuery(Memory)`, `MatAtIngestion(Memory)`, and `MatAtQueryKeptAcrossEvaluations(Memory)`. Assign them to eligible internal nodes, not necessarily the entire query. For example, maintain tumbling windows at ingestion and merge at query time. Generate only supported assignments: retained query-time state currently requires deployment capability and an eligible tumbling-window chain. Ingestion timing and upstream dependencies must satisfy Stage 2 rules. +Use existing physical properties: `Lifetime::None`, `(QueryTime, OneEvaluation(Memory))`, `(IngestionTime, AcrossEvaluations(Memory))`, and `(QueryTime, AcrossEvaluations(Memory))`. Assign them to eligible internal nodes, not necessarily the entire query. For example, maintain tumbling windows at ingestion and merge at query time. Generate only supported assignments: retained query-time state currently requires deployment capability and an eligible tumbling-window chain. Ingestion timing and upstream dependencies must satisfy Stage 2 rules. diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index d638c7a36..a1b24044c 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,7 +17,7 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `Materialization`, tumbling windows) | +| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `PhysicalProperties` (timing and `Lifetime`), tumbling windows) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-logical-asap-optimization` (`crates/logical-asap-optimization`) | Stage 1: Pass 1 and Pass 2 (the memo rules) | | `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-family estimators, composition, budget split, target check) and the deployment's `AccuracyModel` and `AccuracyEvidenceProvider`; depends only on `asap-types` | @@ -42,8 +42,8 @@ asap-types ◄── asap-executor (Stage 4; no stage crate depends on it) ```text QueryRoots (Stage 0) - │ generate_dag_of_logical_memo_groups → DagOfLogicalMemoGroups, then - │ generate_per_computation_logical_memo_candidates + │ memo_of_queries → DagOfLogicalMemoGroups, then + │ pass1_memo ▼ Stage 1, Pass 1 PerComputationLogicalMemoCandidates (Memo + pruned_candidates) │ generate_logical_asap_memo_candidates @@ -71,8 +71,8 @@ same pipeline behind the `OptimizationPass` trait. | Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-logical-asap-optimization`, `pass2/memo/rules/` and `pass2/memo/window_candidates.rs`, over the helpers in `pass2/asap_aware_cse.rs` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | `asap-logical-asap-optimization`, `pass2/memo/enumeration.rs` | | What each summary family can merge and answer (`FamilyCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/family_capability.rs` | -| Stage 2: materialization options per summary build (`NotMaterialized`, `MatAtIngestion`, `MatAtQuery`, `MatAtQueryKeptAcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | -| Timing and data-state edge rules applied from a `MaterializationAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `Materialization` in `physical/materialization.rs` | +| Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | +| Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | | Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | | The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyModel`, `BuiltinAccuracyModel`, `estimators/`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | @@ -98,8 +98,8 @@ The unwired maintained-population rule has been removed as well; no Stage 1 pass For each composed DAG, the Pass 2 rules find what can be shared, split into tumbling windows or reused, and each subset of their candidates becomes another Stage 1 candidate. -3. **Stage 2** gives each summary build its materialization options and emits - one physical candidate per assignment (the all-`NotMaterialized` one +3. **Stage 2** gives each summary build its physical properties options and emits + one physical candidate per assignment (the all-not-materialized one first), skipping assignments that put a build at ingestion time over one that is not. 4. **Stage 3** checks each physical candidate: every query's root guarantee @@ -121,7 +121,7 @@ No stage caps the candidates it plans, and neither does `asap_planner`. Pass 1. - `crates/integration-tests/tests/planner_layering_example{1,2,3,4}.rs` pin the design document's examples through every stage; - `reuse_rules.rs`, `deployment_capabilities.rs`, `materialization_e2e.rs` + `reuse_rules.rs`, `deployment_capabilities.rs`, `physical_properties_e2e.rs` and `stage1_logical_asap_optimization.rs` cover single rules end to end. - `crates/devtools/tests/asap_planner.rs` checks the stage document and the committed viewer examples; `asap_planner_acceptance.rs` checks diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 6086507c2..98c9219ce 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -15,12 +15,12 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | --- | --- | --- | | Stage 0 (frontends) | `NonASAPOp` nodes only; aggregates carry `AggIntent` measures with each query's `AccuracyTarget` as written | none | | Stage 1 (`compose_logical_candidate`, Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | -| Stage 2 (`generate_physical_asap_memo_candidates`; a selected plan built by `Stage2AssignmentRealizer`) | the same operators, with a `Materialization` on each summary build | every node timed from the candidate's `MaterializationAssignment` | +| Stage 2 (`generate_physical_asap_memo_candidates`; a selected plan built by `Stage2AssignmentRealizer`) | the same operators, with a timing and a `Lifetime` on each summary build | every node timed from the candidate's `PhysicalPropertiesAssignment` | - Structurally equal sub-DAGs are interned before Stage 1 (`stage0/cse.rs` in `asap_logical_asap_optimization`), so Stage 1 compares nodes by structure and Pass 2 matches readers structurally, never by pointer. -- Timing is applied with `apply_materialization_timings`, which validates +- Timing is applied with `apply_physical_properties`, which validates every data-state edge. A violation is an `ExecutionDataStateError`: for example `QueryTimeValueUnderIngestionSummary` (a query-time value beneath an ingestion-time build), `ConflictingTiming` (one shared node @@ -78,13 +78,14 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types ## 4. Stage 2 - Only summary builds (`SummaryAgg`, `SummaryTumblingWindowChain`) get materialization - options: `NotMaterialized` always; `MatAtIngestion(Memory)` when the build - is eligible to run as data arrives; `MatAtQuery(Memory)` when two or more - queries read it (reused within one evaluation); - `MatAtQueryKeptAcrossEvaluations(Memory)` for a tumbling-window chain that + options, each a timing and a lifetime (`PhysicalProperties`): not materialized always; + materialized at ingestion time, `AcrossEvaluations(Memory)`, when the build is + eligible to run as data arrives; materialized at query time for `OneEvaluation(Memory)` + when two or more queries read it (reused within one evaluation); materialized at + query time `AcrossEvaluations(Memory)` for a tumbling-window chain that builds one new tumbling window per evaluation, when the deployment declares `query_time_results_retained`. -- One `PhysicalCandidate` per assignment, the all-`NotMaterialized` one first +- One `PhysicalCandidate` per assignment, the all-not-materialized one first (`P`, then `P-m`). An assignment that puts a build at ingestion time over one that is not is not generated. A logical candidate Stage 2 cannot implement becomes a `PrunedCandidate` with its reason. @@ -98,7 +99,7 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types candidate no buildable input, query output or reader count can use is removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders`). Stage 2 emits one candidate per operator node, - timing and materialization; it makes no execution-scope copies. + timing and lifetime; it makes no execution-scope copies. ## 5. Stage 3 diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 9c03a415e..f59566dcd 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -74,20 +74,21 @@ and when it must not; an end-to-end test like `crates/integration-tests/tests/reuse_rules.rs` that the planned workload contains the candidate and that Stage 3 prices it. -## 3. Add a Stage 2 materialization option +## 3. Add a Stage 2 physical properties option -1. Add the `Materialization` variant - (`crates/types/src/physical/materialization.rs`) and its timing in +1. Add the `Lifetime` variant or the valid (timing, lifetime) pair + (`crates/types/src/physical/properties.rs`, `PhysicalProperties::new` and + `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. 2. Offer it in `crates/physical-asap-optimization/src/lib.rs` (the - `MaterializationCandidate` options), after the existing options so default + `MaterializedOutputCandidate` options), after the existing options so default candidate ids stay the same, and gate it on the deployment input that allows it. 3. Price it in `crates/cost/src/usage.rs` (`asap_cost::node_usage`): what it keeps in memory or on disk, and what each evaluation rebuilds. Tests: Stage 2 unit tests for when the option is offered; -`crates/integration-tests/tests/materialization_e2e.rs` and the Example 4 +`crates/integration-tests/tests/physical_properties_e2e.rs` and the Example 4 tests in `planner_layering_example4.rs`. ## 4. Add a Stage 3 check or a deployment input diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 03f3a695c..122eae3b8 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -197,7 +197,7 @@ asap_plan_selection::select_plan( | --- | --- | | `PlanningModels::new(accuracy, evidence)` | Your `&dyn AccuracyModel` and `&dyn AccuracyEvidenceProvider`, with default cost inputs and deployment capabilities; `with_accuracy` and `with_evidence` replace one of them | | `PlanningModels::builtin()` | Built-in accuracy model (`BuiltinAccuracyModel`) and no extra evidence (`NoAccuracyEvidence`). `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | -| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers `MatAtQueryKeptAcrossEvaluations` for tumbling windows) | +| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | | `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::price_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | @@ -206,7 +206,7 @@ asap_plan_selection::select_plan( | `PlanOutput.selection` | `Some` with Stage 3's `Selection` under the default pass: `selected_dag`, `nodes_with_proven_non_negative_weights`, `cost` (the selected plan's `PlanCost` with its `ResourceUsage` and `cost_per_second`, `None` under cost model 3), `accepted_accuracy_per_query`, `pruned_candidates` and `search_report` | | `PlanOutput.tumbling_window_retention_ms` | For each tumbling-window chain the selected plan keeps beyond one evaluation, by node id, how long its tumbling windows are kept (`RealizedPlan::tumbling_window_retention_ms`) | | `PlanningRun` | `per_computation_candidates`, `logical_asap_candidates`, `physical_asap_candidates`, `selection`, `selected_physical_plan` (`RealizedPlan`), `selected_plan` (the roots to run) and `report` (Pass 2's and Stage 2's counts and each stage's time); `per_computation_candidates` is kept only with `PlanSearchSettings::inspect`; `asap_planner::selected_chosen_candidates` reads the selected plan as one candidate per group of `logical_asap_candidates.memo` | -| `PlanOutput::physical_asap_dag()` | The planned queries as one `PhysicalASAPDAG`, timed with the default `MaterializationAssignment` (see [Export](#export)) | +| `PlanOutput::physical_asap_dag()` | The planned queries as one `PhysicalASAPDAG`, timed with the default `PhysicalPropertiesAssignment` (see [Export](#export)) | Each candidate's `ResourceUsage` holds its usage without prices: `cpu_ops_per_second`, `scanned_bytes_per_second`, `average_memory_bytes`, @@ -360,8 +360,8 @@ optional also does not guarantee every planning operation can succeed without it | Function/type | Purpose | | --- | --- | -| `asap_types::ir::apply_materialization_timings(&root, &assignment, &mut NodeToTimedNodeCache::new())` | Write execution timing into every node from a `MaterializationAssignment` (default: all query time) and validate the data-state edges; `PlanOutput::physical_asap_dag()` applies the default to a planned workload | -| `asap_types::ir::physical_export::compile_physical_asap_dag(&timed_root)` | Export a timed DAG as a `PhysicalASAPDAG`; rejects an untimed node; `compile_physical_asap_workload(&roots)` exports a batch with one root per query; `.validate()` checks it explicitly | +| `asap_types::ir::apply_physical_properties(&root, &assignment)` | Complete a `PhysicalPropertiesAssignment` (default: all query time) into the physical properties of every node and validate the data-state edges; `apply_physical_properties_to_roots` does it for a workload; `PlanOutput::physical_asap_dag()` applies the pass's `physical_properties` to a planned workload | +| `asap_types::ir::physical_export::compile_physical_asap_dag(&root, &timed)` | Export a DAG as a `PhysicalASAPDAG`, each node with its properties from the completed assignment `timed`; rejects a node it does not list; `compile_physical_asap_workload(&roots, &timed)` exports a batch with one root per query; `.validate()` checks it explicitly | | `PhysicalASAPDAG::materialized_views()` and `.raw_data_retention()` (`asap_types::ir::materialized_view`) | What a runtime keeps for an installed DAG, derived from the DAG and the `retention_ms` installation sets: each materialized view (refresh mode, coverage, inputs, the node's `retention_ms`) and how far back each raw-data source is kept for query-time scans (each query-time `Scan`'s `retention_ms`) | Choose the export matching your intended handoff: an untimed semantic DAG is not diff --git a/docs/develop_docs/physical-compile-coverage.md b/docs/develop_docs/physical-compile-coverage.md index 9d68caec1..a0b221ddf 100644 --- a/docs/develop_docs/physical-compile-coverage.md +++ b/docs/develop_docs/physical-compile-coverage.md @@ -5,7 +5,7 @@ Audience: developers moving computation from ASAPQuery-backend into ## Contract -Logical selection decides what to compute. A `MaterializationAssignment` sets +Logical selection decides what to compute. A `PhysicalPropertiesAssignment` sets node timing (Stage 2 materialization chooses it per physical candidate). `physical_planner::compile` turns a timed `PhysicalASAPDAG` into physical operator DAGs. The backend owns ingestion, tumbling windows, storage, stored-state evaluation, external exact engines, pricing/selection, and execution scheduling. diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 5fcd74c4c..9a2d23dc5 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -33,8 +33,6 @@ pub struct OperatorNode { pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar pub schema: Schema, // output schema, derived at construction pub guarantee: Option, // None until accuracy assessment establishes one - pub timing: Option, // None until a materialization assignment is applied - pub materialization: Materialization, // NotMaterialized until Stage 2 assigns one } ``` @@ -47,12 +45,11 @@ pub struct OperatorNode { derived (a column reference out of range, a reserved ASAP operator). ASAP planning may retain a more specific schema through `OperatorNode::with_schema`. - `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. -- `timing` is `None` in every front-end DAG and every candidate. It is written by - `ir::timing::apply_materialization_timings` (see the Post-ASAP IR document); export rejects an - untimed node. -- `materialization` is `NotMaterialized` in every front-end DAG. Stage 2 writes it, with - `timing`, through `ir::timing::apply_materialization_timings` from its materialization - assignment. +- A node has no physical properties (timing, lifetime): a logical candidate has only its + logical properties (#1200). Physical stages keep each node's `PhysicalProperties` in a + `PhysicalPropertiesAssignment` keyed by node; `ir::timing::apply_physical_properties` + completes Stage 2's assignment over every node (see the Post-ASAP IR document), and + export reads it. `OperatorNode::children()` returns the operator's inputs in field order followed by the operator nodes its scalar expressions read (see "Scalar expressions"). Every DAG traversal — diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index 77cc5379d..7d89d4389 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -47,7 +47,7 @@ or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. | `SummaryExpr::ValueOperation { .. }` over an evaluation | An ordinary `NonASAPOp` (`Project`, `Filter`, `Sort`, `Limit`, `Aggregate`) reading an ASAP node; `FinalizeExactAccumulator`, `MaintainPopulation`, `EvaluatePopulation` are `ASAPOp` variants | | `Replacement::Summary(..)` / `Replacement::Rewrite(..)` | `Replacement::SubDAG(Rc)`; `is_logical_rewrite` tells them apart | | `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | -| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_materialization_timings` writes it from a `MaterializationAssignment` (default: all query time) | +| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_physical_properties` writes it from a `PhysicalPropertiesAssignment` (default: all query time) | | `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | | `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `share_common_sub_dags` (crate-private in `asap_logical_asap_optimization`, `stage0/cse.rs`) | | `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::ir::physical_export::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | diff --git a/docs/develop_docs/tumbling-window-search-compression-evaluation.md b/docs/develop_docs/tumbling-window-search-compression-evaluation.md index 2419cf97e..5ddcdfc26 100644 --- a/docs/develop_docs/tumbling-window-search-compression-evaluation.md +++ b/docs/develop_docs/tumbling-window-search-compression-evaluation.md @@ -110,7 +110,7 @@ The nine viewer workloads also passed **9/9**, with **40/40** query requirements ### Final regression follow-up -The regression suite found a genuine optimization regression outside window planning. Two identical SQL percentiles over an at-rest table were sent through progressive tumbling window rounds solely because they repeat every ten minutes. No computation had a tumbling-window-eligible window, so the rounds rebuilt the same 40-configuration, 150-connection model and restarted refinement. The 128-evaluation budget expired with two identical unmaterialized KLL builds, costing 0.007678692288679677 per second instead of the baseline's one shared query-materialized build at 0.006000017830006526 (+27.98%). The original pointer-sharing assertion failed both in the suite and in isolation; it was retained unchanged. +The regression suite found a genuine optimization regression outside window planning. Two identical SQL percentiles over an at-rest table were sent through progressive tumbling window rounds solely because they repeat every ten minutes. No computation had a tumbling-window-eligible window, so the rounds rebuilt the same 40-configuration, 150-connection model and restarted refinement. The 128-evaluation budget expired with two identical not_materialized KLL builds, costing 0.007678692288679677 per second instead of the baseline's one shared query-materialized build at 0.006000017830006526 (+27.98%). The original pointer-sharing assertion failed both in the suite and in isolation; it was retained unchanged. Revision `fe171f64` bypasses tumbling-window-domain expansion when the initial computations contain neither a positive range window nor an existing tumbling-window chain. It searches the complete candidate domain directly; at-rest status alone does not disable tumbling window planning. The same SQL fixture now passes the original sharing test and selects one shared KLL with exactly the baseline cost. Its report records 58 evaluations, `optimal`, one `Complete` round, and no pending widths. This fix avoids repeated identical-domain searches rather than increasing the budget or relaxing correctness checks. diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index c6800ee1c..f713f4a0f 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -9,7 +9,7 @@ document shows: - **Stage 1**, the logical candidates of the memo and the selected plan's logical DAG (`L1`); - **Stage 2**, the selected plan as an executable physical DAG (`P1`), with - each node's timing and materialization, and the physical candidates Stage 2 + each node's timing and lifetime, and the physical candidates Stage 2 pruned; - **Stage 3**, the selected plan's cost, the accuracy it accepted per query, its materialized views and the search report. @@ -78,7 +78,7 @@ To run it, see [RUNNING.md](RUNNING.md). candidates are filled green with `✓ selected`. Clicking a candidate or a group shows its nodes, children, parents and queries in the details. - **Details**: click a node for its operator, the queries reaching it, - timing, materialization, output schema, guarantee and Stage 3 cost; click + timing, lifetime, output schema, guarantee and Stage 3 cost; click an edge for its schema and data state. Other documents open with **Open stage document or workload…**, by dropping diff --git a/tools/dag-viewer/app.js b/tools/dag-viewer/app.js index 3c1b4e877..34aed44c5 100644 --- a/tools/dag-viewer/app.js +++ b/tools/dag-viewer/app.js @@ -124,7 +124,7 @@ if (d.tumblingWindowOf !== undefined) items.push(['tumbling window', `${d.tumblingWindowIndex + 1} of ${body.tumbling_window_count} of SummaryTumblingWindowChain ${d.tumblingWindowOf}, window ${tumblingWindowTimeText(d.tumbling_window, body.tumbling_window_width_ms)}`]); if (d.expandable) items.push(['tumbling windows', `${body.tumbling_window_count} × ${durationText(body.tumbling_window_width_ms)} (click the chain to show them, a tumbling window to hide them)`]); if (d.timing !== undefined) items.push(['runs at', formatTiming(d.timing)]); - if (n.materialization) items.push(['materialization', compactWire(n.materialization)]); + if (n.lifetime) items.push(['lifetime', compactWire(n.lifetime)]); if (n.output_state && n.output_state.primitive) items.push(['output primitive', compactWire(n.output_state.primitive)]); if (d.cost) { const value = typeof d.cost.cost_per_second === 'number' @@ -352,9 +352,9 @@ $('views-meta').innerHTML = selection.materialized_views_error ? `no installable plan: ${esc(selection.materialized_views_error)}` : `${count} materialized view${count === 1 ? '' : 's'} in ${groups.length} group${groups.length === 1 ? '' : 's'} of ${esc(selection.selected_candidate)}` + - (count ? '' : ': nothing is kept across evaluations; every query computes at query time'); + (count ? '' : ': nothing is materialized across evaluations; every query computes at query time'); const tumblingWindowsCell = (g) => (g.tumblingWindows.length > 1 - ? `
${esc(g.tumblingWindows.length)} installed nodes
    ${g.tumblingWindows.map((p) => `
  • node ${esc(p.nodeId)} · ${esc(p.tumblingWindow)} · window ${esc(p.window)} · kept ${esc(p.retention)}
    • `).join('')}
` + ? `
${esc(g.tumblingWindows.length)} installed nodes
    ${g.tumblingWindows.map((p) => `
  • node ${esc(p.nodeId)} · ${esc(p.tumblingWindow)} · window ${esc(p.window)} · materialized ${esc(p.retention)}
    • `).join('')}
` : `node ${esc(g.tumblingWindows[0].nodeId)} · window ${esc(g.tumblingWindows[0].window)}`); $('materializedViews').innerHTML = groups.length ? table( ['Stage 2 node', 'definition', 'kind', 'tumbling windows', 'refresh', 'retention', 'installed', 'Stage 3 cost', 'read by'], diff --git a/tools/dag-viewer/memo.js b/tools/dag-viewer/memo.js index 5216dbab3..d13aa801b 100644 --- a/tools/dag-viewer/memo.js +++ b/tools/dag-viewer/memo.js @@ -50,11 +50,11 @@ const memoIsNode = (nodes, id) => Number.isInteger(id) && id >= 0 && id < nodes. // One line per entry of a stage's `pruned_candidates` on group `group`: // Pass 1's `candidate`, or Stage 2's operator with its timing, -// materialization and scope copy; then the reason. 'none' without any. +// lifetime and scope copy; then the reason. 'none' without any. function memoPrunedText(entries, group) { const lines = (Array.isArray(entries) ? entries : []).filter((p) => p && p.group === group).map((p) => { const what = 'candidate' in p ? p.candidate - : `node ${p.operator_node} ${p.operator} ${p.timing} ${JSON.stringify(p.materialization)}`; + : `node ${p.operator_node} ${p.operator} ${p.timing} ${JSON.stringify(p.lifetime)}`; return `${what}: ${p.reason}`; }); return lines.length ? lines.join('; ') : 'none'; diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index aff3da8fa..d30b2c307 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -224,7 +224,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.payload); return { id: node.id, kind, body, schema: node.output_schema, guarantee: node.guarantee, - materialization: node.materialization, timing: node.output_state ? node.output_state.timing : undefined, raw: node, + lifetime: node.lifetime, timing: node.output_state ? node.output_state.timing : undefined, raw: node, }; }); const edges = dag.edges.map((edge) => ({ producer: edge.producer, consumer: edge.consumer, raw: edge })); @@ -241,8 +241,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.operator); stageLogicalChildren(body).forEach((producer) => edges.push({ producer, consumer: id, raw: null })); return { - id, kind, body, schema: node.schema, guarantee: node.guarantee, - materialization: node.materialization, timing: node.timing === null ? undefined : node.timing, raw: node, + id, kind, body, schema: node.schema, guarantee: node.guarantee, raw: node, }; }); const queries = dag.roots.map((root) => ('Scalar' in root ? { scalar: root.Scalar } : { node: root.Operator })); @@ -1079,7 +1078,7 @@ function materializedViewRows(doc) { definition: output.coverage.definition_id, kind: summary ? `${kind} · ${summary}` : kind, tumblingWindow, - refresh: output.refresh === 'at_ingestion' ? 'at ingestion' : 'on first read (kept across evaluations)', + refresh: output.refresh === 'at_ingestion' ? 'at ingestion' : 'on first read (materialized across evaluations)', inputs: `${output.inputs.raw_data_windows.length} raw-data window(s)` + (output.inputs.materialized_views.length ? `, views ${output.inputs.materialized_views.join(', ')}` : ''), retention: durationText(output.retention_ms), diff --git a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json index 633a16fa6..a5bcc3476 100644 --- a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json +++ b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} diff --git a/tools/dag-viewer/test-fixtures/memo-small.json b/tools/dag-viewer/test-fixtures/memo-small.json index cf7f96aea..2b8af0eb1 100644 --- a/tools/dag-viewer/test-fixtures/memo-small.json +++ b/tools/dag-viewer/test-fixtures/memo-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"materialization":"NotMaterialized","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"materialization":"NotMaterialized","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} From 99245c147a90f413a133f99e93342499ae5e7ba0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:34:09 +0000 Subject: [PATCH 02/30] refactor(types): CandidateForMemoGroupId.index -> index_in_group --- crates/types/src/stages/candidates.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index f77cefedb..b28020973 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -10,12 +10,12 @@ use crate::ir::schema::{SketchAlgorithm, SketchParams}; use crate::ir::Operator; use crate::ir::QueryRoot; -/// One candidate: index `index` in group `group`'s candidates +/// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct CandidateForMemoGroupId { pub group: MemoGroupId, - pub index: usize, + pub index_in_group: usize, } /// The memo, Stage 0's and Stage 1's one type: the memo groups, each with From 940c88baa5776f2fb4f04d3bc0fba1db80d28ee0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:15:39 +0000 Subject: [PATCH 03/30] =?UTF-8?q?refactor(types):=20drop=20the=20physical?= =?UTF-8?q?=20schedule=20=CF=83;=20PhysicalProperties=20{=20timing,=20life?= =?UTF-8?q?time=20}?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletes Schedule and the σ-only InvalidPhysicalProperties variants; the only invalid combination left is (IngestionTime, OneEvaluation). satisfies_physical checks timing x lifetime only; delivers_on is the per-context alignment check (w | I and w | φ). A OneEvaluation result runs in one evaluation context, so x_{κ,v} covers it as for ℓ = None. --- crates/plan-selection/src/milp/model.rs | 10 +- crates/types/src/physical/mod.rs | 7 +- crates/types/src/physical/properties.rs | 170 ++++-------------- .../src/stages/physical/memo_candidates.rs | 20 +-- 4 files changed, 51 insertions(+), 156 deletions(-) diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index 772384a6f..10d54cb2a 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -32,14 +32,14 @@ pub struct RefreshWindowId(pub usize); pub struct ContextId(pub usize); /// $\kappa$: a class of executions with a common owner and schedule, where -/// an unmaterialized candidate runs. +/// a candidate with $\ell$ = None or `OneEvaluation` runs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Context { /// $\kappa_q$: an evaluation of a query. QueryEvaluation(QueryId), /// $I_r$: the ingestion pipeline of a refresh window. IngestionPipeline(RefreshWindowId), - /// $\kappa_m$: inside a materialized candidate's own computation. + /// $\kappa_m$: inside an `AcrossEvaluations` candidate's own computation. Materialized(PhysicalCandidateId), } @@ -67,13 +67,13 @@ struct SelectionVariables { /// The execution-context variables (section 5), which replace $x$ and the /// unindexed $y$ of the baseline. struct ExecutionContextVariables { - /// $x^{\mathrm{mat}}_v$, for a materialized candidate. + /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. x_materialized: HashMap, - /// $x_{\kappa,v}$, for an unmaterialized candidate. + /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. x_in_context: HashMap<(ContextId, PhysicalCandidateId), Variable>, /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. y_in_context: HashMap<(ContextId, PhysicalEdgeId), Variable>, - /// $h_v$, for a materialized producer. + /// $h_v$, for an `AcrossEvaluations` producer. retention_amount: HashMap, } diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index 8d3b41089..de00983ee 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,8 +1,7 @@ //! #509 Stage 2 decision data: the physical properties of each candidate. //! -//! - [`properties`] — when a candidate runs, how long its output lives and -//! on which schedule it is delivered ([`PhysicalProperties`], [`Lifetime`], -//! [`Medium`], [`Schedule`]). +//! - [`properties`] — when a candidate runs and how long its output lives +//! ([`PhysicalProperties`], [`Lifetime`], [`Medium`]). //! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG //! ([`PhysicalWorkloadPlan`]). pub mod properties; @@ -10,6 +9,6 @@ pub mod workload_plan; pub use properties::{ EvaluationSchedule, ExecutionTiming, InvalidPhysicalProperties, Lifetime, Medium, - PhysicalProperties, Schedule, + PhysicalProperties, }; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 6080c1184..90f0d5000 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -1,16 +1,17 @@ -//! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell, \sigma)$ +//! A physical candidate's physical properties $p^{\mathrm{phys}}_v = (\tau, \ell)$ //! (#1200; concepts doc `memo-groups-properties-requirements.md`, sections 1 -//! and 3): when it runs ([`ExecutionTiming`]), how long its output lives -//! ([`Lifetime`]) and, for a materialized output, on which times or grid it -//! is delivered ([`Schedule`]). +//! and 3): when it runs ([`ExecutionTiming`]) and how long its output lives +//! ([`Lifetime`]). There is no schedule property: a `None` or `OneEvaluation` +//! output runs on the schedule of the context κ that runs it, and a tumbling +//! grid is anchored at phase 0 with its width in the logical window layout. //! -//! | timing | lifetime | schedule | +//! | timing | lifetime | runs | //! |---|---|---| -//! | query | `None` | none: runs in each context κ that reads it | -//! | query | `OneEvaluation` | `Periodic { interval, phase }` | -//! | query | `AcrossEvaluations` | `TumblingWindowGrid { phase }` | -//! | ingestion | `None` | none: runs inside the ingestion work that reads it | -//! | ingestion | `AcrossEvaluations` | `TumblingWindowGrid { phase: 0 }` | +//! | query | `None` | in each context κ that reads it | +//! | query | `OneEvaluation` | in one context κ; that evaluation's consumers share it | +//! | query | `AcrossEvaluations` | one window when each window ends | +//! | ingestion | `None` | inside the ingestion work that reads it | +//! | ingestion | `AcrossEvaluations` | one window when each window ends | //! | ingestion | `OneEvaluation` | invalid: there is no evaluation at ingestion time | //! //! There is no stored physical requirement: $r^{\mathrm{phys}}_{c,i}$ is a @@ -41,57 +42,30 @@ pub enum Lifetime { AcrossEvaluations(Medium), } -/// $\sigma_v$: what a materialized output ($\ell \ne$ None) is delivered on. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Schedule { - /// `OneEvaluation`: delivered at $\varphi + I\cdot\mathbb Z$. - Periodic { interval_ms: u64, phase_ms: u64 }, - /// `AcrossEvaluations`: the tumbling-window grid's phase, modulo the - /// grid's width. - TumblingWindowGrid { phase_ms: u64 }, -} - -/// A timing, lifetime and schedule that cannot go together (the invalid -/// combinations of the concepts doc, section 2). +/// A timing and lifetime that cannot go together (the invalid combination +/// of the concepts doc, section 2). #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum InvalidPhysicalProperties { #[error("an ingestion-time output cannot be materialized for one evaluation: there is no evaluation at ingestion time")] OneEvaluationAtIngestion, - #[error("an output that is not materialized has no schedule: it runs in its consumer's execution context")] - ScheduleWithoutMaterialization, - #[error("a materialized output needs a schedule")] - MaterializedWithoutSchedule, - #[error( - "one evaluation needs a periodic schedule; across evaluations needs a tumbling-window grid" - )] - ScheduleDoesNotMatchLifetime, - #[error("a periodic schedule needs a positive interval and a phase below it")] - PhaseNotBelowInterval, - #[error("an ingestion-time tumbling-window grid is anchored at the epoch: its phase is 0")] - IngestionGridPhaseNotZero, } -/// $p^{\mathrm{phys}}_v = (\tau, \ell, \sigma)$. [`Self::new`] rejects the -/// invalid combinations (concepts doc, section 2). A grid schedule's -/// phase below the tumbling width is checked where the width is known: when -/// a `PhysicalCandidate` is built. +/// $p^{\mathrm{phys}}_v = (\tau, \ell)$. [`Self::new`] rejects the invalid +/// combination (concepts doc, section 2). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalProperties { timing: ExecutionTiming, lifetime: Lifetime, - schedule: Option, } impl PhysicalProperties { - /// The properties `(timing, lifetime, schedule)`, unless they cannot go - /// together. + /// The properties `(timing, lifetime)`, unless they cannot go together. pub fn new( timing: ExecutionTiming, lifetime: Lifetime, - schedule: Option, ) -> Result { - let _ = (timing, lifetime, schedule); - todo!("#1207 PR 1: reject the invalid combinations of section 2") + let _ = (timing, lifetime); + todo!("#1207 PR 1: reject the invalid combination of section 2") } /// $\tau_v$. @@ -104,19 +78,13 @@ impl PhysicalProperties { self.lifetime } - /// $\sigma_v$: `None` exactly when $\ell_v$ = None. - pub fn schedule(&self) -> Option<&Schedule> { - self.schedule.as_ref() - } - /// Whether $\ell_v \ne$ None. Derived, not stored. pub fn is_materialized(&self) -> bool { self.lifetime != Lifetime::None } } -/// A periodic schedule $\varphi + I\cdot\mathbb Z$: a materialized consumer's -/// own times, or a context's times $T_\kappa$ (Stage 3). +/// When a context κ runs: $\varphi_\kappa + I_\kappa\cdot\mathbb Z$ (Stage 3). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct EvaluationSchedule { interval_ms: u64, @@ -130,98 +98,26 @@ mod tests { const MEMORY: Medium = Medium::Memory; - fn periodic(interval_ms: u64, phase_ms: u64) -> Option { - Some(Schedule::Periodic { - interval_ms, - phase_ms, - }) - } - - fn grid(phase_ms: u64) -> Option { - Some(Schedule::TumblingWindowGrid { phase_ms }) - } - - /// Every invalid combination of the concepts doc, section 2. + /// The one invalid combination of the concepts doc, section 2. #[test] - fn new_rejects_each_invalid_combination() { - use InvalidPhysicalProperties as E; - let cases = [ - ( - IngestionTime, - Lifetime::OneEvaluation(MEMORY), - periodic(10_000, 0), - E::OneEvaluationAtIngestion, - ), - ( - QueryTime, - Lifetime::None, - periodic(10_000, 0), - E::ScheduleWithoutMaterialization, - ), - ( - QueryTime, - Lifetime::OneEvaluation(MEMORY), - None, - E::MaterializedWithoutSchedule, - ), - ( - QueryTime, - Lifetime::OneEvaluation(MEMORY), - grid(0), - E::ScheduleDoesNotMatchLifetime, - ), - ( - QueryTime, - Lifetime::AcrossEvaluations(MEMORY), - periodic(10_000, 0), - E::ScheduleDoesNotMatchLifetime, - ), - ( - QueryTime, - Lifetime::OneEvaluation(MEMORY), - periodic(10_000, 10_000), - E::PhaseNotBelowInterval, - ), - ( - QueryTime, - Lifetime::OneEvaluation(MEMORY), - periodic(0, 0), - E::PhaseNotBelowInterval, - ), - ( - IngestionTime, - Lifetime::AcrossEvaluations(MEMORY), - grid(5_000), - E::IngestionGridPhaseNotZero, - ), - ]; - for (timing, lifetime, schedule, error) in cases { - assert_eq!( - PhysicalProperties::new(timing, lifetime, schedule), - Err(error) - ); - } + fn new_rejects_one_evaluation_at_ingestion() { + assert_eq!( + PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), + Err(InvalidPhysicalProperties::OneEvaluationAtIngestion) + ); } - /// The valid (τ, ℓ) pairs of section 2, each with a schedule of its kind. + /// The valid (τ, ℓ) pairs of section 2. #[test] fn new_accepts_each_valid_pair() { - for (timing, lifetime, schedule) in [ - (QueryTime, Lifetime::None, None), - (IngestionTime, Lifetime::None, None), - ( - QueryTime, - Lifetime::OneEvaluation(MEMORY), - periodic(10_000, 5_000), - ), - (QueryTime, Lifetime::AcrossEvaluations(MEMORY), grid(5_000)), - ( - IngestionTime, - Lifetime::AcrossEvaluations(Medium::Disk), - grid(0), - ), + for (timing, lifetime) in [ + (QueryTime, Lifetime::None), + (IngestionTime, Lifetime::None), + (QueryTime, Lifetime::OneEvaluation(MEMORY)), + (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), + (IngestionTime, Lifetime::AcrossEvaluations(Medium::Disk)), ] { - let properties = PhysicalProperties::new(timing, lifetime, schedule).unwrap(); + let properties = PhysicalProperties::new(timing, lifetime).unwrap(); assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); } } diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 213115d2a..6032e13c7 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -1,5 +1,5 @@ //! Stage 2's output: each logical candidate's physical alternatives -//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell, \sigma)$, and +//! $\mathcal C$, one per valid $p^{\mathrm{phys}} = (\tau, \ell)$, and //! the physical edges $\mathcal E$ with $\alpha^{\log}$ and //! $\alpha^{\mathrm{phys}}$ (#1200, #1201). Stage 3 selects among them; //! Stage 2 prices nothing. @@ -27,27 +27,27 @@ pub struct PhysicalCandidate { } impl PhysicalCandidate { - /// Whether this producer delivers on `times` (the concepts doc's - /// alignment table, section 2). Reads the grid width from the candidate's - /// logical window layout in `memo`. + /// Whether a consumer running on `times` can read this producer (the + /// concepts doc's alignment table, section 2): $w \mid I$ and + /// $w \mid \varphi$ for a windowed `AcrossEvaluations` producer, with $w$ + /// read from its logical window layout in `memo`; true for $\ell$ = None, + /// and for `OneEvaluation` when the result belongs to that context. pub fn delivers_on(&self, memo: &Memo, times: &EvaluationSchedule) -> bool { let _ = (memo, times); todo!("#1207 PR 1: the alignment table of section 2") } } -/// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime (the concepts doc's -/// table, section 2), and, for a materialized consumer, that the producer -/// delivers on the consumer's own schedule. Takes candidates rather than -/// bare properties because a tumbling grid's width lives in the logical -/// window layout. +/// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts +/// doc's table, section 2). Alignment is checked per context in Stage 3 with +/// [`PhysicalCandidate::delivers_on`]. pub fn satisfies_physical( memo: &Memo, producer: &PhysicalCandidate, consumer: &PhysicalCandidate, ) -> bool { let _ = (memo, producer, consumer); - todo!("#1207 PR 1: the timing x lifetime table and the alignment table of section 2") + todo!("#1207 PR 1: the timing x lifetime table of section 2") } /// $(u, c, i)$ over physical candidates. From cb6e9e1a0a1b6ee64846ef10da71e0604431f789 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:23:58 +0000 Subject: [PATCH 04/30] refactor(types): theta must be listed, floors need not be; satisfies_physical drops memo InvalidLogicalProperties gains SketchBuildWithoutParameters and renames FloorNotInParameterList to ParametersNotInList, returned by LogicalProperties::new. LogicalRequirement::new no longer checks list membership (a floor is any value of the family's parameter type, e.g. heap_size >= k), so it returns Self. satisfies_physical takes only the producer and consumer. --- crates/types/src/stages/candidates.rs | 32 ++++++++++--------- .../src/stages/physical/memo_candidates.rs | 8 ++--- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index b28020973..91a5c5c74 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -140,8 +140,9 @@ pub struct CandidateExpression { /// $p^{\log}_v = (\boldsymbol\theta_v, \omega_v)$. The family is the /// operator's argument, not a property. [`Self::new`] rejects parameters of -/// another family (or on an exact operator) and a window on an operator that -/// cannot be windowed. +/// another family, not in the family's list, missing on a sketch build or +/// present on an exact operator, and a window on an operator that cannot be +/// windowed. #[derive(Debug, Clone)] pub struct LogicalProperties { /// $\boldsymbol\theta_v$, from the family's predefined list; `None` for @@ -158,7 +159,7 @@ impl LogicalProperties { window_layout: Option, ) -> Result { let _ = (expression, parameters, window_layout); - todo!("#1207 PR 1: reject parameters of another family and invalid windows") + todo!("#1207 PR 1: reject invalid parameters and invalid windows") } /// $\boldsymbol\theta_v$. @@ -179,9 +180,8 @@ pub enum WindowLayout { Tumbling { width_ms: u64 }, } -/// Logical properties or a logical requirement whose dimensions cannot go -/// together: bad input to [`LogicalProperties::new`] or -/// [`LogicalRequirement::new`]. +/// Logical properties whose dimensions cannot go together: bad input to +/// [`LogicalProperties::new`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum InvalidLogicalProperties { #[error("the parameters belong to another family than the operator's")] @@ -192,8 +192,10 @@ pub enum InvalidLogicalProperties { WindowOnOperatorThatCannotBeWindowed, #[error("a tumbling window has a positive width")] ZeroWindowWidth, - #[error("the parameter floor is not in its family's predefined list")] - FloorNotInParameterList, + #[error("a sketch build needs parameters from its family's list")] + SketchBuildWithoutParameters, + #[error("the parameters are not in their family's predefined list")] + ParametersNotInList, } /// The family's predefined $\boldsymbol\theta$ list (#1207 step 6), each with @@ -205,19 +207,19 @@ pub fn parameter_list(family: SketchAlgorithm) -> &'static [SketchParams] { } /// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. -/// [`Self::new`] rejects a floor that is not in the family's predefined list. #[derive(Debug, Clone)] pub struct LogicalRequirement { - /// $\boldsymbol\theta^{\min}_{c,i}$: the smallest $\boldsymbol\theta$ in - /// the list whose $\varepsilon$, for the statistic the consumer reads, is - /// at most $\text{cap}_{c,i}$ = min(validity cap of $g_c$, pruning cap). + /// $\boldsymbol\theta^{\min}_{c,i}$: the weakest parameters whose + /// $\varepsilon$, for the statistic the consumer reads, is at most + /// $\text{cap}_{c,i}$ = min(validity cap of $g_c$, pruning cap), or a + /// direct need such as heap_size ≥ k. Any value of the family's parameter + /// type; it need not be in the family's list. parameter_floor: Option, } impl LogicalRequirement { - pub fn new(parameter_floor: Option) -> Result { - let _ = parameter_floor; - todo!("#1207 PR 1: reject a floor not in the family's predefined list") + pub fn new(parameter_floor: Option) -> Self { + Self { parameter_floor } } /// $\boldsymbol\theta^{\min}_{c,i}$. diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 6032e13c7..390945ba5 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -41,12 +41,8 @@ impl PhysicalCandidate { /// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts /// doc's table, section 2). Alignment is checked per context in Stage 3 with /// [`PhysicalCandidate::delivers_on`]. -pub fn satisfies_physical( - memo: &Memo, - producer: &PhysicalCandidate, - consumer: &PhysicalCandidate, -) -> bool { - let _ = (memo, producer, consumer); +pub fn satisfies_physical(producer: &PhysicalCandidate, consumer: &PhysicalCandidate) -> bool { + let _ = (producer, consumer); todo!("#1207 PR 1: the timing x lifetime table of section 2") } From ed44fc227799411fe0a014bac9d27913afb3c925 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:26:58 +0000 Subject: [PATCH 05/30] refactor(plan-selection): one evaluation context per schedule Context::QueryEvaluation(QueryId) becomes Context::Evaluation(EvaluationSchedule): queries with the same (interval, phase) are evaluated together and share their OneEvaluation results. evaluation_schedule returns that schedule. QueryId stays for z_{q,v}. A non-periodic query has no context yet. --- crates/plan-selection/src/milp/model.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index 10d54cb2a..d642fce79 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -35,8 +35,11 @@ pub struct ContextId(pub usize); /// a candidate with $\ell$ = None or `OneEvaluation` runs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Context { - /// $\kappa_q$: an evaluation of a query. - QueryEvaluation(QueryId), + /// The evaluation of every query with this schedule: queries with the + /// same (interval, phase) are evaluated together and share its + /// `OneEvaluation` results. A and B both every 10 s at phase 0 give one + /// context. A non-periodic query has no context yet (out of scope). + Evaluation(EvaluationSchedule), /// $I_r$: the ingestion pipeline of a refresh window. IngestionPipeline(RefreshWindowId), /// $\kappa_m$: inside an `AcrossEvaluations` candidate's own computation. @@ -44,11 +47,16 @@ pub enum Context { } impl Context { - /// $T_\kappa$: the context's periodic evaluation schedule when it has - /// one; `None` (a non-periodic query) is not an implicit compatibility - /// pass. + /// $T_\kappa$: an evaluation context's schedule; `None` for an ingestion + /// pipeline or a materialized candidate's own computation, which is not + /// an implicit compatibility pass. pub fn evaluation_schedule(&self) -> Option { - todo!("#1207 PR 7: each context's schedule from its query's 𝒯_q") + match self { + Context::Evaluation(schedule) => Some(*schedule), + Context::IngestionPipeline(_) | Context::Materialized(_) => { + todo!("#1207 PR 7: whether an ingestion or materialized context has times") + } + } } } From 9416f448a09f473987752056e1a6ecb8e00658d0 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:34:35 +0000 Subject: [PATCH 06/30] refactor(types): rename the #1207 interfaces as agreed (2026-11-06) InvalidLogicalProperties -> LogicalPropertiesError, InvalidPhysicalProperties -> PhysicalPropertiesError, LogicalRequirement -> LogicalRequirements (parameter_floor -> parameter_floor_for_required_accuracy), satisfies_logical / satisfies_physical -> producer_satisfies_logical_requirements / producer_satisfies_physical_requirements, Medium -> StorageMedium, Context / ContextId -> ExecutionContext / ExecutionContextId (x_in_context / y_in_context -> x_in_execution_context / y_in_execution_context). PhysicalCandidate and PhysicalCandidateKey fields -> logical_candidate_id, physical_properties; edge fields -> producer_candidate_id, consumer_candidate_id; PhysicalASAPMemoCandidates fields -> logical_candidates, physical_candidates. --- crates/cost/src/lib.rs | 2 +- crates/cost/src/usage.rs | 18 ++++++------ crates/executor/src/physical_planner/mod.rs | 4 +-- .../executor/tests/precompute_population.rs | 6 ++-- crates/plan-selection/src/milp/model.rs | 16 +++++----- crates/planner/src/run.rs | 2 +- crates/types/src/ir/physical_export.rs | 6 ++-- crates/types/src/physical/mod.rs | 6 ++-- crates/types/src/physical/properties.rs | 21 ++++++++------ crates/types/src/stages/candidates.rs | 27 ++++++++++------- .../src/stages/logical_memo_candidates.rs | 9 +++--- crates/types/src/stages/mod.rs | 12 ++++---- crates/types/src/stages/physical/graph.rs | 17 +++++++---- .../src/stages/physical/memo_candidates.rs | 29 ++++++++++--------- crates/types/src/stages/physical/mod.rs | 4 +-- crates/types/tests/physical_export.rs | 6 ++-- docs/design_docs/concepts/post-asap-ir.md | 2 +- 17 files changed, 102 insertions(+), 85 deletions(-) diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index f921d3bb7..208f2cd0c 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -83,7 +83,7 @@ pub struct ResourceUsage { /// per consumer that recomputes it, with the consumers materialized at ingestion time /// of one refresh window counted once. pub peak_memory_bytes: u64, - /// Bytes kept on disk: state materialized on `Medium::Disk`, plus the + /// Bytes kept on disk: state materialized on `StorageMedium::Disk`, plus the /// raw data query-time scans read (the longest window of each source). pub disk_bytes: u64, /// CPU cores to provision for the peak: the ingestion work's cores plus diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 1932529a1..109db3c7f 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -30,7 +30,7 @@ use asap_types::ir::schema::{ FieldDataType, GroupingStrategy, Schema, SketchParams, SketchStatistic, }; use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; -use asap_types::physical::{Lifetime, Medium}; +use asap_types::physical::{Lifetime, StorageMedium}; use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; use asap_types::workload::bounds::{ @@ -1137,7 +1137,7 @@ pub fn node_usage<'a>( if !ingestion && working_memory_bytes > 0 { held = Some((working_memory_bytes, per_second, true)); } - (0, Medium::Memory) + (0, StorageMedium::Memory) } // A tumbling-window chain materialized across evaluations holds the tumbling windows its // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window @@ -1158,7 +1158,7 @@ pub fn node_usage<'a>( ) .ok_or(AnalyticalCostError::Overflow)?; match medium { - Medium::Memory => { + StorageMedium::Memory => { average_memory_bytes = between as f64; peak_memory_bytes = match retained == tumbling_windows { true => state, @@ -1167,18 +1167,18 @@ pub fn node_usage<'a>( .ok_or(AnalyticalCostError::Overflow)?, }; } - Medium::Disk => disk_bytes = between, + StorageMedium::Disk => disk_bytes = between, } detail += &format!("; materializes {between} bytes across evaluations"); (0, medium) } (_, Lifetime::OneEvaluation(medium), _) => { match medium { - Medium::Memory => { + StorageMedium::Memory => { held = Some((state, per_second, false)); peak_memory_bytes = state; } - Medium::Disk => disk_bytes = state, + StorageMedium::Disk => disk_bytes = state, } (0, medium) } @@ -1206,16 +1206,16 @@ pub fn node_usage<'a>( medium, ), }; - if ingestion && node.lifetime == Lifetime::AcrossEvaluations(Medium::Memory) { + if ingestion && node.lifetime == Lifetime::AcrossEvaluations(StorageMedium::Memory) { maintained = Some(state / tumbling_windows_of(node)); } if materialized_bytes > 0 { match medium { - Medium::Memory => { + StorageMedium::Memory => { average_memory_bytes = materialized_bytes as f64; peak_memory_bytes = materialized_bytes; } - Medium::Disk => disk_bytes = materialized_bytes, + StorageMedium::Disk => disk_bytes = materialized_bytes, } detail += &format!("; keeps {materialized_bytes} bytes"); } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index f17a925b5..e32464736 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -1603,7 +1603,7 @@ mod compilation_tests { #[cfg(feature = "test-support")] pub mod test_support { use super::*; - use planner_types::physical::{Lifetime, Medium}; + use planner_types::physical::{Lifetime, StorageMedium}; use std::collections::HashMap; /// The materialized views at `nodes`, in that order: each node that is @@ -1646,7 +1646,7 @@ pub mod test_support { for node in &mut dag.nodes { let id = node.id as NodeId; if marked.contains(&id) { - node.lifetime = Lifetime::AcrossEvaluations(Medium::Memory); + node.lifetime = Lifetime::AcrossEvaluations(StorageMedium::Memory); } if at_ingestion.contains(&id) { node.output_state.timing = ExecutionTiming::IngestionTime; diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 432b38337..50cef473b 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -59,7 +59,7 @@ fn view_over_samples( [ PhysicalASAPDAGNode { lifetime: planner_types::physical::Lifetime::AcrossEvaluations( - planner_types::physical::Medium::Memory, + planner_types::physical::StorageMedium::Memory, ), ..node( 0, @@ -139,7 +139,7 @@ fn try_state_dag( }), output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: planner_types::physical::Lifetime::AcrossEvaluations( - planner_types::physical::Medium::Memory, + planner_types::physical::StorageMedium::Memory, ), parameters: None, output_schema: logical_schema(family.clone()), @@ -173,7 +173,7 @@ fn try_state_dag( }), output_state: ExecutionDataState::INGESTION_SUMMARY, lifetime: planner_types::physical::Lifetime::AcrossEvaluations( - planner_types::physical::Medium::Memory, + planner_types::physical::StorageMedium::Memory, ), parameters: None, output_schema: logical_schema(target), diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index d642fce79..75e11f09d 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -27,14 +27,14 @@ pub struct QueryId(pub usize); #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RefreshWindowId(pub usize); -/// Index of a [`Context`] among the workload's contexts. +/// Index of a [`ExecutionContext`] among the workload's contexts. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct ContextId(pub usize); +pub struct ExecutionContextId(pub usize); /// $\kappa$: a class of executions with a common owner and schedule, where /// a candidate with $\ell$ = None or `OneEvaluation` runs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Context { +pub enum ExecutionContext { /// The evaluation of every query with this schedule: queries with the /// same (interval, phase) are evaluated together and share its /// `OneEvaluation` results. A and B both every 10 s at phase 0 give one @@ -46,14 +46,14 @@ pub enum Context { Materialized(PhysicalCandidateId), } -impl Context { +impl ExecutionContext { /// $T_\kappa$: an evaluation context's schedule; `None` for an ingestion /// pipeline or a materialized candidate's own computation, which is not /// an implicit compatibility pass. pub fn evaluation_schedule(&self) -> Option { match self { - Context::Evaluation(schedule) => Some(*schedule), - Context::IngestionPipeline(_) | Context::Materialized(_) => { + ExecutionContext::Evaluation(schedule) => Some(*schedule), + ExecutionContext::IngestionPipeline(_) | ExecutionContext::Materialized(_) => { todo!("#1207 PR 7: whether an ingestion or materialized context has times") } } @@ -78,9 +78,9 @@ struct ExecutionContextVariables { /// $x^{\mathrm{mat}}_v$, for an `AcrossEvaluations` candidate. x_materialized: HashMap, /// $x_{\kappa,v}$, for a candidate with $\ell$ = None or `OneEvaluation`. - x_in_context: HashMap<(ContextId, PhysicalCandidateId), Variable>, + x_in_execution_context: HashMap<(ExecutionContextId, PhysicalCandidateId), Variable>, /// $y_{\kappa,u,c,i}$, indexed by the consumer's context. - y_in_context: HashMap<(ContextId, PhysicalEdgeId), Variable>, + y_in_execution_context: HashMap<(ExecutionContextId, PhysicalEdgeId), Variable>, /// $h_v$, for an `AcrossEvaluations` producer. retention_amount: HashMap, } diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 3f919230c..143db8a7b 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -192,7 +192,7 @@ pub fn run_planning( let physical = physical_asap_candidates.insert( physical.map_err(|error| SelectionError::PhysicalASAPCandidates(error.to_string()))?, ); - report.physical_candidates = physical.candidates.len(); + report.physical_candidates = physical.physical_candidates.len(); report.stage2_pruned_candidates = physical.pruned_candidates.len(); // Stage 3. let started = Instant::now(); diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 62af7acd7..caccbb4b2 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -22,7 +22,7 @@ use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::schema::{FieldDataType, Schema, SketchParams}; -use crate::physical::{InvalidPhysicalProperties, Lifetime}; +use crate::physical::{Lifetime, PhysicalPropertiesError}; /// A node's operator, with each child replaced by its node id. pub type PhysicalASAPOperatorPayload = Operator; @@ -174,7 +174,7 @@ pub enum PhysicalASAPDAGValidationError { InvalidPhysicalProperties { node: PhysicalASAPNodeId, #[source] - reason: InvalidPhysicalProperties, + reason: PhysicalPropertiesError, }, /// A summary build (or maintained population) that is not materialized but does /// not run at query time: a planner bug. @@ -554,7 +554,7 @@ impl PhysicalASAPDAG { { return Err(PhysicalASAPDAGValidationError::InvalidPhysicalProperties { node: node.id, - reason: InvalidPhysicalProperties::OneEvaluationAtIngestion, + reason: PhysicalPropertiesError::OneEvaluationAtIngestion, }); } if is_materializable_state_output(&node.payload) diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index de00983ee..2b9e04694 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,14 +1,14 @@ //! #509 Stage 2 decision data: the physical properties of each candidate. //! //! - [`properties`] — when a candidate runs and how long its output lives -//! ([`PhysicalProperties`], [`Lifetime`], [`Medium`]). +//! ([`PhysicalProperties`], [`Lifetime`], [`StorageMedium`]). //! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG //! ([`PhysicalWorkloadPlan`]). pub mod properties; pub mod workload_plan; pub use properties::{ - EvaluationSchedule, ExecutionTiming, InvalidPhysicalProperties, Lifetime, Medium, - PhysicalProperties, + EvaluationSchedule, ExecutionTiming, Lifetime, PhysicalProperties, PhysicalPropertiesError, + StorageMedium, }; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 90f0d5000..0c998bf5b 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -16,13 +16,13 @@ //! //! There is no stored physical requirement: $r^{\mathrm{phys}}_{c,i}$ is a //! function of the consumer's own properties -//! ([`crate::stages::satisfies_physical`]). +//! ([`crate::stages::producer_satisfies_physical_requirements`]). pub use crate::ir::properties::ExecutionTiming; /// Where a materialized output is stored. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -pub enum Medium { +pub enum StorageMedium { Memory, Disk, } @@ -37,15 +37,15 @@ pub enum Lifetime { #[default] None, /// Materialized until every consumer within one evaluation finishes. - OneEvaluation(Medium), + OneEvaluation(StorageMedium), /// Materialized across evaluations, so later evaluations reuse it. - AcrossEvaluations(Medium), + AcrossEvaluations(StorageMedium), } /// A timing and lifetime that cannot go together (the invalid combination /// of the concepts doc, section 2). #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum InvalidPhysicalProperties { +pub enum PhysicalPropertiesError { #[error("an ingestion-time output cannot be materialized for one evaluation: there is no evaluation at ingestion time")] OneEvaluationAtIngestion, } @@ -63,7 +63,7 @@ impl PhysicalProperties { pub fn new( timing: ExecutionTiming, lifetime: Lifetime, - ) -> Result { + ) -> Result { let _ = (timing, lifetime); todo!("#1207 PR 1: reject the invalid combination of section 2") } @@ -96,14 +96,14 @@ mod tests { use super::*; use ExecutionTiming::{IngestionTime, QueryTime}; - const MEMORY: Medium = Medium::Memory; + const MEMORY: StorageMedium = StorageMedium::Memory; /// The one invalid combination of the concepts doc, section 2. #[test] fn new_rejects_one_evaluation_at_ingestion() { assert_eq!( PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), - Err(InvalidPhysicalProperties::OneEvaluationAtIngestion) + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) ); } @@ -115,7 +115,10 @@ mod tests { (IngestionTime, Lifetime::None), (QueryTime, Lifetime::OneEvaluation(MEMORY)), (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), - (IngestionTime, Lifetime::AcrossEvaluations(Medium::Disk)), + ( + IngestionTime, + Lifetime::AcrossEvaluations(StorageMedium::Disk), + ), ] { let properties = PhysicalProperties::new(timing, lifetime).unwrap(); assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 91a5c5c74..0d93ba367 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -103,7 +103,7 @@ pub struct CandidateForMemoGroup { pub logical_properties: LogicalProperties, /// $r^{\log}_{c,i}$, one per input group: per child of /// `expression.operator`, in [`Operator::children`] order. - pub input_requirements: Vec, + pub input_requirements: Vec, /// The groups a fused computation also implements, beyond the group /// whose candidate it is; empty otherwise. pub fused_groups: Vec, @@ -157,7 +157,7 @@ impl LogicalProperties { expression: &CandidateExpression, parameters: Option, window_layout: Option, - ) -> Result { + ) -> Result { let _ = (expression, parameters, window_layout); todo!("#1207 PR 1: reject invalid parameters and invalid windows") } @@ -183,7 +183,7 @@ pub enum WindowLayout { /// Logical properties whose dimensions cannot go together: bad input to /// [`LogicalProperties::new`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -pub enum InvalidLogicalProperties { +pub enum LogicalPropertiesError { #[error("the parameters belong to another family than the operator's")] ParametersOfAnotherFamily, #[error("an exact operator has no summary parameters")] @@ -208,23 +208,25 @@ pub fn parameter_list(family: SketchAlgorithm) -> &'static [SketchParams] { /// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. #[derive(Debug, Clone)] -pub struct LogicalRequirement { +pub struct LogicalRequirements { /// $\boldsymbol\theta^{\min}_{c,i}$: the weakest parameters whose /// $\varepsilon$, for the statistic the consumer reads, is at most /// $\text{cap}_{c,i}$ = min(validity cap of $g_c$, pruning cap), or a /// direct need such as heap_size ≥ k. Any value of the family's parameter /// type; it need not be in the family's list. - parameter_floor: Option, + parameter_floor_for_required_accuracy: Option, } -impl LogicalRequirement { - pub fn new(parameter_floor: Option) -> Self { - Self { parameter_floor } +impl LogicalRequirements { + pub fn new(parameter_floor_for_required_accuracy: Option) -> Self { + Self { + parameter_floor_for_required_accuracy, + } } /// $\boldsymbol\theta^{\min}_{c,i}$. - pub fn parameter_floor(&self) -> Option<&SketchParams> { - self.parameter_floor.as_ref() + pub fn parameter_floor_for_required_accuracy(&self) -> Option<&SketchParams> { + self.parameter_floor_for_required_accuracy.as_ref() } /// $g_c$'s linear bound: $\beta_{c,i}$ and $b_{c,i}$ per input, read @@ -252,7 +254,10 @@ pub struct ErrorPropagation { /// $\alpha^{\log}_{u,c,i}$: $\boldsymbol\theta_u \succeq /// \boldsymbol\theta^{\min}_{c,i}$, compared per family on the parameters /// that set $\varepsilon$. No floor, or an exact producer, always satisfies. -pub fn satisfies_logical(producer: &LogicalProperties, requirement: &LogicalRequirement) -> bool { +pub fn producer_satisfies_logical_requirements( + producer: &LogicalProperties, + requirement: &LogicalRequirements, +) -> bool { let _ = (producer, requirement); todo!("#1207 PR 1: per-family dominance") } diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index 6a8205011..9a313ec76 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -3,12 +3,13 @@ use super::{CandidateForMemoGroupId, Memo}; -/// $(u, c, i)$ over logical candidates: `producer` is a candidate of -/// $G_{c,i}$, the group input `input_index_of_consumer` of `consumer` reads. +/// $(u, c, i)$ over logical candidates: `producer_candidate_id` is a +/// candidate of $G_{c,i}$, the group input `input_index_of_consumer` of +/// `consumer_candidate_id` reads. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct LogicalEdge { - pub producer: CandidateForMemoGroupId, - pub consumer: CandidateForMemoGroupId, + pub producer_candidate_id: CandidateForMemoGroupId, + pub consumer_candidate_id: CandidateForMemoGroupId, pub input_index_of_consumer: usize, /// $\alpha^{\log}_{u,c,i}$. pub satisfies_logical_requirement: bool, diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index b106be284..4425f594b 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -16,9 +16,9 @@ mod pruning; mod realizer; pub use candidates::{ - parameter_list, satisfies_logical, CandidateExpression, CandidateForMemoGroup, - CandidateForMemoGroupId, ErrorPropagation, InvalidLogicalProperties, LogicalProperties, - LogicalRequirement, Memo, OriginOfCandidateForMemoGroup, WindowLayout, + parameter_list, producer_satisfies_logical_requirements, CandidateExpression, + CandidateForMemoGroup, CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, + LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, WindowLayout, }; pub use logical_memo_candidates::{LogicalASAPMemoCandidates, LogicalEdge}; pub use memo::{queries_rooted_at, root_groups, MemoGroup, MemoGroupId}; @@ -26,9 +26,9 @@ pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, }; pub use physical::{ - output_references, satisfies_physical, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, - SelectedEdge, SelectedNodeId, + output_references, producer_satisfies_physical_requirements, DagOfSelectedCandidates, + PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, + PhysicalEdge, PhysicalEdgeId, SelectedEdge, SelectedNodeId, }; pub use pruning::{PruneReason, PrunedCandidate}; pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs index 69f6bc760..3f7910f0b 100644 --- a/crates/types/src/stages/physical/graph.rs +++ b/crates/types/src/stages/physical/graph.rs @@ -47,7 +47,7 @@ impl DagOfSelectedCandidates { /// some output reference, each output reference comes from a candidate /// of its group. Costs and accuracy are Stage 3's to check. pub fn validate(&self, physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let memo = &physical.logical.memo; + let memo = &physical.logical_candidates.memo; let output_groups: Vec<_> = super::output_references(&physical.query_roots) .into_iter() .map(|(_, group)| group) @@ -55,9 +55,14 @@ impl DagOfSelectedCandidates { let query_output_candidates: Vec> = output_groups .iter() .map(|&group| { - (0..physical.candidates.len()) + (0..physical.physical_candidates.len()) .map(PhysicalCandidateId) - .filter(|id| physical.candidates[id.0].logical.group == group) + .filter(|id| { + physical.physical_candidates[id.0] + .logical_candidate_id + .group + == group + }) .collect() }) .collect(); @@ -65,9 +70,9 @@ impl DagOfSelectedCandidates { query_output_candidates.iter().map(Vec::as_slice).collect(); let edges: Vec<_> = physical.edges().collect(); self.validate_against( - physical.candidates.len(), + physical.physical_candidates.len(), |candidate| { - memo.candidate(physical.candidates[candidate.0].logical) + memo.candidate(physical.physical_candidates[candidate.0].logical_candidate_id) .expression .operator .children() @@ -77,7 +82,7 @@ impl DagOfSelectedCandidates { physical .admissible_edges_into(candidate, input_index_of_consumer) .iter() - .any(|edge| edges[edge.0].1.producer == producer) + .any(|edge| edges[edge.0].1.producer_candidate_id == producer) }, &query_output_candidates, ) diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 390945ba5..7e25d5d0e 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -21,9 +21,9 @@ pub struct PhysicalEdgeId(pub usize); #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalCandidate { /// $e_v$ and $p^{\log}_v$. - pub logical: CandidateForMemoGroupId, + pub logical_candidate_id: CandidateForMemoGroupId, /// $p^{\mathrm{phys}}_v$. - pub properties: PhysicalProperties, + pub physical_properties: PhysicalProperties, } impl PhysicalCandidate { @@ -41,7 +41,10 @@ impl PhysicalCandidate { /// $\alpha^{\mathrm{phys}}_{u,c,i}$: timing and lifetime only (the concepts /// doc's table, section 2). Alignment is checked per context in Stage 3 with /// [`PhysicalCandidate::delivers_on`]. -pub fn satisfies_physical(producer: &PhysicalCandidate, consumer: &PhysicalCandidate) -> bool { +pub fn producer_satisfies_physical_requirements( + producer: &PhysicalCandidate, + consumer: &PhysicalCandidate, +) -> bool { let _ = (producer, consumer); todo!("#1207 PR 1: the timing x lifetime table of section 2") } @@ -49,8 +52,8 @@ pub fn satisfies_physical(producer: &PhysicalCandidate, consumer: &PhysicalCandi /// $(u, c, i)$ over physical candidates. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalEdge { - pub producer: PhysicalCandidateId, - pub consumer: PhysicalCandidateId, + pub producer_candidate_id: PhysicalCandidateId, + pub consumer_candidate_id: PhysicalCandidateId, pub input_index_of_consumer: usize, /// $\alpha^{\log}_{u,c,i}$. pub satisfies_logical_requirement: bool, @@ -63,8 +66,8 @@ pub struct PhysicalEdge { /// [`PhysicalCandidateId`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct PhysicalCandidateKey { - pub logical: CandidateForMemoGroupId, - pub properties: PhysicalProperties, + pub logical_candidate_id: CandidateForMemoGroupId, + pub physical_properties: PhysicalProperties, } /// Stage 2's output, which Stage 3 selects from. @@ -72,9 +75,9 @@ pub struct PhysicalCandidateKey { pub struct PhysicalASAPMemoCandidates { /// Pass 2's output: the memo, its candidates' $e_v$, $p^{\log}_v$, /// $r^{\log}_{c,i}$, and the logical edges. - pub logical: LogicalASAPMemoCandidates, + pub logical_candidates: LogicalASAPMemoCandidates, /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. - pub candidates: Vec, + pub physical_candidates: Vec, /// Each query's root over groups, by index in the workload, passed /// through from Pass 2. pub query_roots: Vec>, @@ -90,15 +93,15 @@ pub struct PhysicalASAPMemoCandidates { } impl PhysicalASAPMemoCandidates { - /// Stage 2's output over `candidates` and `edges` (grouped by + /// Stage 2's output over `physical_candidates` and `edges` (grouped by /// (consumer, input)), with the admissible-edge indexes built once. pub fn new( - logical: LogicalASAPMemoCandidates, - candidates: Vec, + logical_candidates: LogicalASAPMemoCandidates, + physical_candidates: Vec, query_roots: Vec>, edges: Vec, ) -> Self { - let _ = (logical, candidates, query_roots, edges); + let _ = (logical_candidates, physical_candidates, query_roots, edges); todo!("#1207 PR 5: index the admissible edges") } diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 3a49d0eb1..1b072bb99 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -6,6 +6,6 @@ mod memo_candidates; pub use graph::{DagOfSelectedCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId}; pub use memo_candidates::{ - output_references, satisfies_physical, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, + output_references, producer_satisfies_physical_requirements, PhysicalASAPMemoCandidates, + PhysicalCandidate, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, }; diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 0d9d2e1b3..d33f4f50a 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -7,7 +7,7 @@ use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::{InvalidPhysicalProperties, Lifetime, Medium}; +use asap_types::physical::{Lifetime, PhysicalPropertiesError, StorageMedium}; use std::rc::Rc; /// Scan(t) → SummaryAgg(sum by key) → FinalizeExactAccumulator, untimed. @@ -107,12 +107,12 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { }) .unwrap(); scalar_only.nodes[state].output_state.timing = ExecutionTiming::IngestionTime; - scalar_only.nodes[state].lifetime = Lifetime::OneEvaluation(Medium::Disk); + scalar_only.nodes[state].lifetime = Lifetime::OneEvaluation(StorageMedium::Disk); assert_eq!( scalar_only.validate(), Err(PhysicalASAPDAGValidationError::InvalidPhysicalProperties { node: scalar_only.nodes[state].id, - reason: InvalidPhysicalProperties::OneEvaluationAtIngestion, + reason: PhysicalPropertiesError::OneEvaluationAtIngestion, }) ); diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 9377eee5d..07f03ee49 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -116,7 +116,7 @@ decides, for each sub-DAG, whether its output is materialized, and if so with wh timing and a `Lifetime` (#1200). The lifetime is `None` (discarded after its reader), `OneEvaluation(medium)` (materialized until every reader of one evaluation finishes) or `AcrossEvaluations(medium)` (materialized for later evaluations), with -`Medium::Memory` or `Medium::Disk`; an ingestion-time output cannot be materialized +`StorageMedium::Memory` or `StorageMedium::Disk`; an ingestion-time output cannot be materialized for `OneEvaluation`. A [`PhysicalPropertiesAssignment`](../../../crates/types/src/ir/properties/timing.rs) records, per sub-DAG, the properties of its materialized output, named by the node whose output From b05cec1f84c3222837e1cda6fddebb10188f99b4 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:41:58 +0000 Subject: [PATCH 07/30] refactor(types): execution contexts reuse the workload's recurrence Deletes EvaluationSchedule and ExecutionContext::evaluation_schedule. An evaluation context is Evaluation { recurrence, range_evaluation }: queries with equal ones share it and its OneEvaluation results, one-off queries included. ExecutionContext, ExecutionContextId and RefreshWindowId move to asap_types::stages so PhysicalCandidate::delivers_on can take the context. Contexts are deduplicated by equality (QueryRecurrence holds an f64). --- crates/plan-selection/src/milp/model.rs | 42 ++----------------- crates/types/src/physical/mod.rs | 3 +- crates/types/src/physical/properties.rs | 7 ---- crates/types/src/stages/mod.rs | 5 ++- .../src/stages/physical/execution_context.rs | 36 ++++++++++++++++ .../src/stages/physical/memo_candidates.rs | 21 ++++++---- crates/types/src/stages/physical/mod.rs | 2 + 7 files changed, 58 insertions(+), 58 deletions(-) create mode 100644 crates/types/src/stages/physical/execution_context.rs diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index 75e11f09d..e976556c0 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -16,50 +16,14 @@ use std::collections::HashMap; use good_lp::{Constraint, Variable}; -use asap_types::physical::EvaluationSchedule; -use asap_types::stages::{PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId}; +use asap_types::stages::{ + ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, +}; /// A query, by index in the workload. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct QueryId(pub usize); -/// A refresh window of the workload's ingestion pipelines, by index. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct RefreshWindowId(pub usize); - -/// Index of a [`ExecutionContext`] among the workload's contexts. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct ExecutionContextId(pub usize); - -/// $\kappa$: a class of executions with a common owner and schedule, where -/// a candidate with $\ell$ = None or `OneEvaluation` runs. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ExecutionContext { - /// The evaluation of every query with this schedule: queries with the - /// same (interval, phase) are evaluated together and share its - /// `OneEvaluation` results. A and B both every 10 s at phase 0 give one - /// context. A non-periodic query has no context yet (out of scope). - Evaluation(EvaluationSchedule), - /// $I_r$: the ingestion pipeline of a refresh window. - IngestionPipeline(RefreshWindowId), - /// $\kappa_m$: inside an `AcrossEvaluations` candidate's own computation. - Materialized(PhysicalCandidateId), -} - -impl ExecutionContext { - /// $T_\kappa$: an evaluation context's schedule; `None` for an ingestion - /// pipeline or a materialized candidate's own computation, which is not - /// an implicit compatibility pass. - pub fn evaluation_schedule(&self) -> Option { - match self { - ExecutionContext::Evaluation(schedule) => Some(*schedule), - ExecutionContext::IngestionPipeline(_) | ExecutionContext::Materialized(_) => { - todo!("#1207 PR 7: whether an ingestion or materialized context has times") - } - } - } -} - /// The baseline variables (section 3). struct SelectionVariables { /// $x_v$, by [`PhysicalCandidateId`]. diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index 2b9e04694..28db939f6 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -8,7 +8,6 @@ pub mod properties; pub mod workload_plan; pub use properties::{ - EvaluationSchedule, ExecutionTiming, Lifetime, PhysicalProperties, PhysicalPropertiesError, - StorageMedium, + ExecutionTiming, Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium, }; pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 0c998bf5b..3bc12bf0a 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -84,13 +84,6 @@ impl PhysicalProperties { } } -/// When a context κ runs: $\varphi_\kappa + I_\kappa\cdot\mathbb Z$ (Stage 3). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct EvaluationSchedule { - interval_ms: u64, - phase_ms: u64, -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 4425f594b..a8500534b 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -27,8 +27,9 @@ pub use per_computation_logical_memo_candidates::{ }; pub use physical::{ output_references, producer_satisfies_physical_requirements, DagOfSelectedCandidates, - PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalCandidateKey, - PhysicalEdge, PhysicalEdgeId, SelectedEdge, SelectedNodeId, + ExecutionContext, ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidate, + PhysicalCandidateId, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, RefreshWindowId, + SelectedEdge, SelectedNodeId, }; pub use pruning::{PruneReason, PrunedCandidate}; pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs new file mode 100644 index 000000000..0ab1630e8 --- /dev/null +++ b/crates/types/src/stages/physical/execution_context.rs @@ -0,0 +1,36 @@ +//! $\kappa$: where a candidate with $\ell$ = None or `OneEvaluation` runs +//! (concepts doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and +//! $y_{\kappa,u,c,i}$ by it; [`super::PhysicalCandidate::delivers_on`] reads +//! it, so it lives with the physical candidates. + +use super::graph::PhysicalCandidateId; +use crate::workload::{QueryRecurrence, RangeEvaluation}; + +/// A refresh window of the workload's ingestion pipelines, by index. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct RefreshWindowId(pub usize); + +/// Index of an [`ExecutionContext`] in the workload's list of distinct +/// contexts. Contexts are deduplicated by equality: [`QueryRecurrence`] +/// holds an `f64` (`EstimatedRate`), so a context cannot be hashed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ExecutionContextId(pub usize); + +/// $\kappa$: a class of executions with a common owner and schedule, where +/// a candidate with $\ell$ = None or `OneEvaluation` runs. +#[derive(Debug, Clone, PartialEq)] +pub enum ExecutionContext { + /// The evaluation of every query with this recurrence and range + /// evaluation, the workload's own description of when it runs: queries + /// with equal ones are evaluated together and share its `OneEvaluation` + /// results. A and B both every 10 s at phase 0 give one context; a + /// one-off query's context is its `OneTime` recurrence. + Evaluation { + recurrence: QueryRecurrence, + range_evaluation: Option, + }, + /// $I_r$: the ingestion pipeline of a refresh window. + IngestionPipeline(RefreshWindowId), + /// $\kappa_m$: inside an `AcrossEvaluations` candidate's own computation. + Materialized(PhysicalCandidateId), +} diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 7e25d5d0e..5b64ece64 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -6,9 +6,10 @@ use std::collections::HashMap; +use super::execution_context::ExecutionContext; use super::graph::PhysicalCandidateId; use crate::ir::QueryRoot; -use crate::physical::{EvaluationSchedule, PhysicalProperties}; +use crate::physical::PhysicalProperties; use crate::stages::{ CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PrunedCandidate, }; @@ -27,13 +28,17 @@ pub struct PhysicalCandidate { } impl PhysicalCandidate { - /// Whether a consumer running on `times` can read this producer (the - /// concepts doc's alignment table, section 2): $w \mid I$ and - /// $w \mid \varphi$ for a windowed `AcrossEvaluations` producer, with $w$ - /// read from its logical window layout in `memo`; true for $\ell$ = None, - /// and for `OneEvaluation` when the result belongs to that context. - pub fn delivers_on(&self, memo: &Memo, times: &EvaluationSchedule) -> bool { - let _ = (memo, times); + /// Whether a consumer running in `context` can read this producer (the + /// concepts doc's alignment table, section 2). True for $\ell$ = None, + /// and for `OneEvaluation` in the same context. A windowed + /// `AcrossEvaluations` producer, with $w$ read from its logical window + /// layout in `memo`, needs every evaluation time of the context on its + /// grid: $w$ divides the context's evaluation grid (interval, range step + /// and phase), each scheduled time, or a one-off query's time; false + /// when the times are not known (an interval without a phase, an + /// estimated rate, an unknown recurrence). + pub fn delivers_on(&self, memo: &Memo, context: &ExecutionContext) -> bool { + let _ = (memo, context); todo!("#1207 PR 1: the alignment table of section 2") } } diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index 1b072bb99..b5ed3f780 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -1,9 +1,11 @@ //! The Stage 2 → Stage 3 boundary: the physical candidates and edges Stage 2 //! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201). +mod execution_context; mod graph; mod memo_candidates; +pub use execution_context::{ExecutionContext, ExecutionContextId, RefreshWindowId}; pub use graph::{DagOfSelectedCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId}; pub use memo_candidates::{ output_references, producer_satisfies_physical_requirements, PhysicalASAPMemoCandidates, From 868d2f8effb1b1cc7141b886d3b3349a4814301e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:46:36 +0000 Subject: [PATCH 08/30] refactor: naming sweep for the #1207 unit PhysicalASAPDAGValidationError::InvalidPhysicalProperties becomes PhysicalProperties { node, reason }. In the files this unit edits, "kept" describing a materialized result or retained data becomes "materialized" or "retained" (cost, materialized views, realizer, installable plan, deployment capabilities; locals kept/raw_kept_rows -> retained/ raw_retained_rows), and "reader" becomes "query" where it is a query (execution demand, ScheduleGroup::queries, cost usage, labels, tests) or "consumer" where it is a consuming candidate. --- crates/accuracy/src/estimators/kll.rs | 4 +- crates/accuracy/src/estimators/mod.rs | 28 ++++---- crates/accuracy/src/estimators/univmon.rs | 6 +- crates/accuracy/src/propagation.rs | 8 +-- crates/cost/src/execution_demand.rs | 72 +++++++++---------- crates/cost/src/lib.rs | 8 +-- crates/cost/src/usage.rs | 52 +++++++------- crates/devtools/src/bin/asap_planner.rs | 2 +- crates/devtools/src/candidate_labels.rs | 6 +- crates/devtools/src/planner_fuzz/mod.rs | 2 +- crates/devtools/tests/asap_planner.rs | 2 +- .../tests/planner_layering_common/mod.rs | 10 +-- .../tests/sharing_issue_regressions.rs | 6 +- .../src/deployment_capabilities.rs | 10 +-- crates/logical-asap-optimization/src/lib.rs | 2 +- .../src/pass1/per_computation_candidates.rs | 20 +++--- .../src/stage0/cse.rs | 6 +- crates/plan-selection/src/report.rs | 2 +- crates/planner/src/installable.rs | 8 +-- crates/planner/src/pass.rs | 4 +- crates/planner/tests/sql_table_scans.rs | 2 +- crates/types/src/ir/materialized_view.rs | 48 +++++++------ .../src/ir/operator/tumbling_window_chain.rs | 2 +- crates/types/src/ir/physical_export.rs | 14 ++-- crates/types/src/stages/physical/graph.rs | 2 +- crates/types/src/stages/pruning.rs | 8 +-- crates/types/src/stages/realizer.rs | 8 +-- crates/types/tests/physical_export.rs | 2 +- docs/design_docs/concepts/post-asap-ir.md | 2 +- 29 files changed, 175 insertions(+), 171 deletions(-) diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index 068a49121..11815e994 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -180,12 +180,12 @@ pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> Ske } } -/// The `k` of one KLL shared by two readers sized alone at `(k, δ)` and +/// The `k` of one KLL shared by two consumers sized alone at `(k, δ)` and /// `(k2, δ2)`: each `k` resized to the smaller failure probability at no /// larger rank error ([`with_failure_probability`]), then the larger. The /// shared build states its guarantee at that smaller δ, where a `k` sized /// for a larger δ has a larger rank error, so the larger `k` alone could -/// miss its reader's ε. +/// miss its consumer's ε. pub(crate) fn shared_k( k: u32, failure_probability: Option, diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 27b6197c6..67fca98d2 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -262,9 +262,9 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { /// /// A KLL's `k` and a distinct-count UnivMon's heap are the exceptions to /// the largest value: their error grows as their failure probability -/// shrinks, so each reader's `k` or heap is first resized to the shared +/// shrinks, so each consumer's `k` or heap is first resized to the shared /// (smallest) failure probability at no larger error (`kll::shared_k`, -/// `univmon::shared_heap`), and the shared build meets every reader. +/// `univmon::shared_heap`), and the shared build meets every consumer. pub fn size_for_all_readers(params: &[&SketchParams]) -> Option { use SketchParams as P; let (first, rest) = params.split_first()?; @@ -287,7 +287,7 @@ pub fn size_for_all_readers(params: &[&SketchParams]) -> Option { }, ) => P::UnivMon { // Not the largest heap: a distinct count's error grows as - // its failure probability shrinks, so each reader's heap + // its failure probability shrinks, so each consumer's heap // first grows to the shared one at no larger error. heap_size: univmon::shared_heap(heap_size, distinct_failure_probability, *h, *f), sketch_rows: sketch_rows.max(*r), @@ -476,7 +476,7 @@ mod shared_size_tests { else { unreachable!() }; - // The `k = 200` reader, sized at the calibration's δ = 0.01, is + // The `k = 200` consumer, sized at the calibration's δ = 0.01, is // resized to the shared δ = 0.001 at its own rank error. assert_eq!((k, failure_probability), (258, Some(0.001))); assert_eq!( @@ -499,18 +499,18 @@ mod shared_size_tests { assert_eq!(size_for_all_readers(&[]), None); } - /// A KLL shared by a reader at rank (0.01, δ = 0.01) and one at - /// (0.05, δ = 1e-4) meets both: each reader's `k` is resized to the + /// A KLL shared by a consumer at rank (0.01, δ = 0.01) and one at + /// (0.05, δ = 1e-4) meets both: each consumer's `k` is resized to the /// shared δ = 1e-4 at no larger rank error (A's 0.00997) before the /// larger is taken (k = 412), not 269 stated at 1e-4 (rank error 0.0151). #[test] fn a_kll_shared_across_failure_probabilities_meets_each_reader() { - let readers = [(0.01, 0.01), (0.05, 1e-4)]; - let params = readers.map(|(eps, delta)| kll::size_params(eps, delta)); + let consumers = [(0.01, 0.01), (0.05, 1e-4)]; + let params = consumers.map(|(eps, delta)| kll::size_params(eps, delta)); let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); let quantile = SketchStatistic::Quantile { q: 0.99 }; let guarantee = sketch_guarantee(&SketchAlgorithm::Kll, &shared, &quantile).unwrap(); - for (eps, delta) in readers { + for (eps, delta) in consumers { assert!( satisfies( &guarantee, @@ -555,15 +555,15 @@ mod shared_size_tests { ); } - /// A distinct-count UnivMon shared by a reader at relative (0.05, - /// δ = 0.01) and one at (0.2, δ = 1e-4) meets both: each reader's heap + /// A distinct-count UnivMon shared by a consumer at relative (0.05, + /// δ = 0.01) and one at (0.2, δ = 1e-4) meets both: each consumer's heap /// grows to the shared δ = 1e-4 at no larger error before the larger is /// taken (5 400 000), not B's 337 500 stated at 1e-4 (error 0.2). #[test] fn a_distinct_univmon_shared_across_failure_probabilities_meets_each_reader() { use asap_types::ir::operator::agg_intent::default_cardinality; - let readers = [(0.05, 0.01), (0.2, 1e-4)]; - let params = readers.map(|(eps, delta)| { + let consumers = [(0.05, 0.01), (0.2, 1e-4)]; + let params = consumers.map(|(eps, delta)| { size_params(SketchAlgorithm::UnivMon, &default_cardinality(), eps, delta) }); let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); @@ -575,7 +575,7 @@ mod shared_size_tests { panic!("{guarantee:?}") }; let chebyshev = terms[0].evaluate().unwrap(); - for (eps, delta) in readers { + for (eps, delta) in consumers { assert!( error <= eps && chebyshev <= delta, "{shared:?} misses ({eps}, {delta}): error {error}" diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index 899ced4c6..aa4dcacf0 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -370,14 +370,14 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar } } -/// The heap of one UnivMon shared by two readers with heaps `heap_size` +/// The heap of one UnivMon shared by two consumers with heaps `heap_size` /// and `heap_size2`, sized for a distinct count at the failure /// probabilities they record (`None`: not sized for one): each distinct -/// reader's heap grown to the smaller failure probability at no larger +/// consumer's heap grown to the smaller failure probability at no larger /// relative error ([`with_failure_probability`]), then the larger. The /// shared build states its distinct guarantee at that smaller δ, where a /// heap sized for a larger δ has a larger error, so the larger heap alone -/// could miss its reader's ε. A reader not sized for a distinct count keeps +/// could miss its consumer's ε. A consumer not sized for a distinct count keeps /// its heap. pub(crate) fn shared_heap( heap_size: u32, diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index be95ce58e..2d506c88e 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -510,7 +510,7 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> ) { return Some(None); } - let reader = OperatorNode::with_schema( + let consumer = OperatorNode::with_schema( node.operator.map_children(|child| { if Rc::ptr_eq(child, state) { Rc::clone(input) @@ -520,7 +520,7 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> }), node.schema.clone(), ); - Some(composed_guarantee(&reader)) + Some(composed_guarantee(&consumer)) } /// The guarantee of an estimate or finalized value read from merged @@ -573,7 +573,7 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option Option, - /// How often the node is computed per second for these readers. + pub queries: Vec, + /// How often the node is computed per second for these queries. pub evaluations_per_second: f64, /// Which kind of group it is. pub schedule: Schedule, } -/// One fixed (interval, phase) schedule's readers: each step in ms they -/// use with the longest span in ms at that step, and the readers. +/// One fixed (interval, phase) schedule's queries: each step in ms they +/// use with the longest span in ms at that step, and the queries. type FixedSchedule = (BTreeMap, Vec); -/// The groups of `readers` evaluated together (each a recurrence with its +/// The groups of `queries` evaluated together (each a recurrence with its /// range evaluation), in the order [`evaluation_rate`] adds them: fixed /// intervals by (interval, phase), then each estimated rate, then the /// non-repeating batch. A request with a range evaluation is /// [`RangeEvaluation::evaluations_per_request`] evaluations, one without it -/// one. Repeating readers with equal intervals and equal phases form one +/// one. Repeating queries with equal intervals and equal phases form one /// group, every request ending at the same instant: a node they share is /// computed once per distinct instant their steps fall on, -/// `end − k · step_ms` for `k · step_ms ≤ span_ms`. A reader of unknown +/// `end − k · step_ms` for `k · step_ms ≤ span_ms`. A query of unknown /// phase joins the group of the first known phase on its interval, if any. -/// A reader evaluated once per request (no range, or one step) evaluates +/// A query evaluated once per request (no range, or one step) evaluates /// only at the end. Different intervals or phases are different groups (an /// over-estimate for intervals whose instants coincide, e.g. every 20 s and /// every 30 s meet every 60 s); each estimated rate is its own group. -/// One-off, scheduled and unknown readers form one batch group: the most +/// One-off, scheduled and unknown queries form one batch group: the most /// evaluations among them, spread over `non_repeating_query_horizon_seconds`. pub fn schedule_groups<'r>( - readers: impl IntoIterator)>, + queries: impl IntoIterator)>, non_repeating_query_horizon_seconds: f64, ) -> Result, AnalyticalCostError> { // Per interval in ms and phase in ms on it (`None`: unknown), each step - // in ms its readers use and the longest span in ms at that step, with - // the readers; once per request is the interval with span 0. + // in ms its queries use and the longest span in ms at that step, with + // the queries; once per request is the interval with span 0. let mut by_schedule: BTreeMap<(u32, Option), FixedSchedule> = BTreeMap::new(); let mut estimated = Vec::new(); let mut invocations = 0u64; let mut non_repeating = Vec::new(); - for (reader, (recurrence, range_evaluation)) in readers.into_iter().enumerate() { + for (query, (recurrence, range_evaluation)) in queries.into_iter().enumerate() { let steps = range_evaluation.map_or(1, |range| range.evaluations_per_request()); match recurrence { QueryRecurrence::Repeated( @@ -97,7 +97,7 @@ pub fn schedule_groups<'r>( by_schedule.entry((interval.0, phase_ms)).or_default(); let longest_span_ms = longest_span_ms_by_step_ms.entry(step_ms).or_default(); *longest_span_ms = (*longest_span_ms).max(span_ms); - members.push(reader); + members.push(query); } QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { let rate = estimate.expected_rate.0; @@ -105,28 +105,28 @@ pub fn schedule_groups<'r>( return Err(AnalyticalCostError::InvalidRecurrence); } estimated.push(ScheduleGroup { - readers: vec![reader], + queries: vec![query], evaluations_per_second: rate * steps as f64, schedule: Schedule::EstimatedRate, }); } QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => { invocations = invocations.max(times.len() as u64 * steps); - non_repeating.push(reader); + non_repeating.push(query); } QueryRecurrence::OneTime { invocations: count, .. } => { invocations = invocations.max(*count * steps); - non_repeating.push(reader); + non_repeating.push(query); } QueryRecurrence::Unknown => { invocations = invocations.max(steps); - non_repeating.push(reader); + non_repeating.push(query); } } } - // Readers of unknown phase run with the first known phase on their + // Queries of unknown phase run with the first known phase on their // interval. let unknown_phase_intervals_ms: Vec = by_schedule .keys() @@ -153,8 +153,8 @@ pub fn schedule_groups<'r>( let mut groups: Vec = by_schedule .into_iter() .map( - |((interval_ms, _), (longest_span_ms_by_step_ms, readers))| ScheduleGroup { - readers, + |((interval_ms, _), (longest_span_ms_by_step_ms, queries))| ScheduleGroup { + queries, evaluations_per_second: 1_000.0 / f64::from(interval_ms) * distinct_instants_per_request(&longest_span_ms_by_step_ms) as f64, schedule: Schedule::FixedInterval, @@ -164,7 +164,7 @@ pub fn schedule_groups<'r>( groups.extend(estimated); if !non_repeating.is_empty() { groups.push(ScheduleGroup { - readers: non_repeating, + queries: non_repeating, evaluations_per_second: invocations as f64 / non_repeating_query_horizon_seconds, schedule: Schedule::NonRepeating, }); @@ -336,10 +336,10 @@ pub fn consumers(dag: &PhysicalASAPDAG) -> HashMap Self { diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 109db3c7f..7601756e5 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -1,5 +1,5 @@ //! Costing a physical plan DAG node by node: each node's resource usage -//! ([`node_usage`]), then the plan's usage, latency, retention, kept tumbling window +//! ([`node_usage`]), then the plan's usage, latency, retention, retained tumbling window //! chains and cost ([`plan_cost`]). Moved from Stage 3 (`asap-plan-selection`, //! whose module doc, "Cost model", describes the model), so Stage 3 computes //! no cost of its own. @@ -44,8 +44,8 @@ use asap_types::workload::bounds::{ /// reached. A query-time node is measured per computation over the rows its /// time ranges keep, times how often it is computed per second /// ([`computations`]); an ingestion-time node over one second of arriving -/// rows. A materialized node's kept state is counted in byte-seconds per -/// second; a tumbling-window chain kept beyond one evaluation keeps the tumbling windows its +/// rows. A materialized node's state is counted in byte-seconds per +/// second; a tumbling-window chain materialized beyond one evaluation keeps the tumbling windows its /// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `materialized_chains` are /// `dag`'s ([`node_map`], [`materialized_tumbling_window_chains`]); `recurrence[q]` and /// `range_evaluations[q]` say how often query `q` runs and how many steps @@ -100,7 +100,7 @@ pub fn plan_cost( // counted once the evaluations' latencies are known, in node order so // the sum is the same in every run. let mut held: BTreeMap = BTreeMap::new(); - // The raw data query-time scans read, kept on disk: per source, the + // The raw data query-time scans read, retained on disk: per source, the // largest scan's (bytes, node). let mut raw_data: HashMap<&Source, (u64, PhysicalASAPNodeId)> = HashMap::new(); // Each in-memory ingestion-time state's bytes per window (what it keeps @@ -425,7 +425,7 @@ fn each_node_usage<'a>( (Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { match dag.scan_window(node.id, materialized_chains) { // Priced as before #955 decides `@` anchors and - // unwindowed readers: the widest window read. + // unwindowed queries: the widest window read. ScanWindow::Unbounded => trailing_window( widest_window_read_ms(dag, nodes, node.id, materialized_chains) .or(root_scan_lookback_ms), @@ -572,8 +572,8 @@ pub fn node_usage<'a>( let mut held = None; let mut maintained = None; let mut raw_scan = None; - // A query-time scan's raw rows kept ([`ScanWindow::lookback_ms`]). - let mut raw_kept_rows = None; + // A query-time scan's raw rows retained ([`ScanWindow::lookback_ms`]). + let mut raw_retained_rows = None; let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; let inputs: Vec = producers.iter().map(|p| p.output).collect(); let sources: BTreeSet<&str> = match &node.payload { @@ -722,15 +722,15 @@ pub fn node_usage<'a>( (rows_per_ms * lookback as f64).round() as u64 }) }; - // The raw rows kept for a query-time scan: back to the + // The raw rows retained for a query-time scan: back to the // oldest row it reads, `end_offset + width` (an offset // window keeps more than it reads), at most those held. if !ingestion { - let kept_rows = match (by_rate(scan_window.lookback_ms()), rows_held) { + let retained_rows = match (by_rate(scan_window.lookback_ms()), rows_held) { (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), (by_rate, rows_held) => by_rate.or(rows_held), }; - raw_kept_rows = kept_rows.map(|rows| rows.max(1)); + raw_retained_rows = retained_rows.map(|rows| rows.max(1)); } let by_rate = by_rate(lookback); let rows = match (ingestion, by_rate, rows_held) { @@ -867,13 +867,13 @@ pub fn node_usage<'a>( // A range keeps its window's samples; an instant // selector the latest sample of each series. // Unknown rate or series: no fewer than its input. - let kept = match kind { + let retained = match kind { TimeRangeKind::Range => rows_per_ms.map_or(input.rows, |rows_per_ms| { (rows_per_ms * range.as_millis() as f64).round() as u64 }), TimeRangeKind::Instant => series.unwrap_or(input.rows), }; - let out = edge(input.rows.min(kept.max(1)))?; + let out = edge(input.rows.min(retained.max(1)))?; let out = EdgeStatistics { rows: out.rows, bytes: out.bytes.min(input.bytes), @@ -945,7 +945,7 @@ pub fn node_usage<'a>( tumbling_window_count, .. }) => { - // Each input row is kept by the one tumbling window its time falls in, + // Each input row is retained by the one tumbling window its time falls in, // as a time range keeps it (one operation), and updates // that tumbling window. let group_count = groups(reduction)?; @@ -954,7 +954,7 @@ pub fn node_usage<'a>( .ok_or(AnalyticalCostError::Overflow)?; // Each tumbling window's state sees its group's rows in one tumbling window: of // one second's rows at ingestion time, of the rows read at - // query time (all of them for a chain kept across + // query time (all of them for a chain materialized across // evaluations, whose scan Stage 2 bounds to its newest tumbling window, // the only one it builds). let rows_per_tumbling_window = match (ingestion, node.lifetime) { @@ -1117,7 +1117,7 @@ pub fn node_usage<'a>( let mut peak_memory_bytes = 0u64; let mut disk_bytes = 0u64; let mut detail = format!("{detail}; x {per_second:.4}/s"); - // State kept between computations: at ingestion time, the window + // State retained between computations: at ingestion time, the window // being built and the completed ones queries read (a chain of N // tumbling windows keeps N + 1), every second. At query time the output is // reused only within one evaluation, so it is held only while that @@ -1127,7 +1127,7 @@ pub fn node_usage<'a>( // keeps nothing holds its working memory while it runs, once per // consumer that recomputes it; at query time, that is held for the // evaluation too (counted in [`plan_cost`], per consumer: a query's - // latency, or a materialized consumer's longest reader's, as a + // latency, or a materialized consumer's longest query's, as a // OneEvaluation state is). let state = out.bytes.max(estimate.peak_memory_bytes()); let (materialized_bytes, medium) = match (ingestion, node.lifetime, &node.payload) { @@ -1141,7 +1141,7 @@ pub fn node_usage<'a>( } // A tumbling-window chain materialized across evaluations holds the tumbling windows its // retention covers ([`tumbling_window_retention_ms`]: N, plus one per tumbling window - // width of the readers' range-evaluation span), all but the + // width of the queries' range-evaluation span), all but the // newest between evaluations and all while one runs. ( false, @@ -1221,15 +1221,15 @@ pub fn node_usage<'a>( } if let Payload::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload { if !ingestion { - let kept = match raw_kept_rows { + let retained = match raw_retained_rows { Some(rows) => edge(rows)?.bytes, None => out.bytes, }; - raw_scan = Some((source, kept)); + raw_scan = Some((source, retained)); } } // One computation's CPU and scan price times its rate, then the - // kept state's. + // materialized state's. let cost = usage_prices.map(|prices| { (estimate.cpu_ops() * prices.cost_per_cpu_op + estimate.scan_bytes() as f64 * prices.cost_per_scanned_byte) @@ -1387,9 +1387,9 @@ fn ingest_workers( (ingest_ms_per_s / 1_000.0).ceil().max(1.0) as u64 } -/// For each tumbling-window chain of `dag` kept beyond one evaluation (at +/// For each tumbling-window chain of `dag` materialized beyond one evaluation (at /// ingestion time, or at query time across evaluations), how far back -/// from the newest evaluation its completed tumbling windows must be kept, in +/// from the newest evaluation its completed tumbling windows must be retained, in /// milliseconds: how long before the evaluation its newest tumbling window ends (the /// offset of its input, `chain_input_offset_ms`), plus its window /// `tumbling_window_count · tumbling_window_width_ms`, plus the longest @@ -1457,7 +1457,7 @@ fn chain_input_offset_ms( u64::try_from(offset_ms).unwrap_or(0) } -/// The tumbling-window retention of every kept tumbling-window chain of `dag` +/// The tumbling-window retention of every materialized tumbling-window chain of `dag` /// ([`tumbling_window_retention_ms`] over [`query_reach`]), for a plan whose query /// `q` has range evaluation `range_evaluations[q]` (none when missing): /// what Stage 2's realizer records for a realized plan. @@ -1601,9 +1601,9 @@ pub type NodeMap<'a> = /// The query-time tumbling-window chains of `dag` materialized across evaluations /// (lifetime `AcrossEvaluations` at query time, which Stage 2 offers only when the -/// deployment keeps query-time results and every reader builds one new +/// deployment keeps query-time results and every query builds one new /// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the -/// older ones, which complete the readers' look-back, are kept from +/// older ones, which complete the queries' look-back, are retained from /// earlier evaluations. pub fn materialized_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { dag.nodes @@ -1623,7 +1623,7 @@ pub fn materialized_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet String { } let mut shared: Vec<_> = producers .values() - .filter(|(_, readers)| readers.len() > 1) - .map(|(name, readers)| format!("shared {name} ({})", query_names(readers))) + .filter(|(_, queries)| queries.len() > 1) + .map(|(name, queries)| format!("shared {name} ({})", query_names(queries))) .collect(); shared.sort(); labels.extend(shared); @@ -157,7 +157,7 @@ pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String }; let label = match &node.payload { // Name the chain: chains of one family and tumbling-window count differ by - // their readers and tumbling-window width. + // their queries and tumbling-window width. Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, tumbling_window_count, diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index ab359d74f..c40a2ef4a 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -456,7 +456,7 @@ impl Planner { /// assignment implemented and priced, the cheapest selected, with the /// same models as [`plan`](Self::plan). Guarantees are composed again /// first, so a derived node carries the guarantee of what it reads, and - /// summaries are sized for their readers as Pass 1 sizes them. + /// summaries are sized for their consumers as Pass 1 sizes them. /// With `last_root_is_key_filter`, the last root is a `Filter` on /// group-key columns only and keeps its input's guarantee (whole groups /// selected by an exact key keep each remaining row's bound), which diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index dfa0b9512..c09f75a24 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -1299,7 +1299,7 @@ fn undeclared_supported_summaries_default_to_the_executors_families() { /// Two windowed quantiles over one metric (Q1 over 1 m, Q2 over 5 m, both /// every 10 s): with a shared scan, Stage 2 also offers the scan built once /// per evaluation, which reads the 5 m window once (≈0.1364/s) instead of -/// once per reader (0.2203/s); the selected plan costs at most 0.143/s. +/// once per query (0.2203/s); the selected plan costs at most 0.143/s. #[test] fn a_scan_two_queries_share_is_offered_built_once_per_evaluation() { let workload = json!({ diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index a62cba38b..ee5aa4f5d 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -264,7 +264,7 @@ pub fn closure(dag: &impl ExportedDag, root: NodeId) -> HashSet { } /// The queries (indexes into `query_roots`) whose result depends on `node`. -pub fn readers(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { +pub fn queries(dag: &impl ExportedDag, query_roots: &[NodeId], node: NodeId) -> BTreeSet { query_roots .iter() .enumerate() @@ -338,7 +338,7 @@ pub fn estimates_of(dag: &impl ExportedDag, build: NodeId) -> Vec<(NodeId, Sketc pub fn cross_query_nodes(dag: &impl ExportedDag, query_roots: &[NodeId]) -> BTreeSet { dag.node_ids() .into_iter() - .filter(|&id| readers(dag, query_roots, id).len() > 1) + .filter(|&id| queries(dag, query_roots, id).len() > 1) .collect() } @@ -421,9 +421,9 @@ pub enum Materializing { MaterializedAtIngestion, /// Runs when a query first needs it; output reused by the other /// queries of the evaluation, or, as query-time `AcrossEvaluations`, - /// kept for later evaluations. + /// retained for later evaluations. MaterializedAtQueryTime, - /// Output discarded after its reader (lifetime `None`). + /// Output discarded after its query (lifetime `None`). NotMaterialized, } @@ -475,7 +475,7 @@ pub fn window_form(dag: &impl ExportedDag, build: NodeId) -> WindowForm { } } -/// How far back from an evaluation a materialized node's kept output +/// How far back from an evaluation a materialized node's materialized output /// reaches in event time, derived the way plan selection prices what it /// keeps: a tumbling-window chain keeps its `N` /// tumbling windows of width `w`, plus the one being built when maintained at diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs index 2ad36086e..02221f75a 100644 --- a/crates/integration-tests/tests/sharing_issue_regressions.rs +++ b/crates/integration-tests/tests/sharing_issue_regressions.rs @@ -101,7 +101,7 @@ fn slower_repeat_reuses_faster_sum() { ); } -/// #902: the filtered reader avoids its independent scan and aggregation. +/// #902: the filtered query avoids its independent scan and aggregation. #[test] fn grouped_filtered_quantile_plans() { assert!( @@ -110,7 +110,7 @@ fn grouped_filtered_quantile_plans() { ); } -/// #922: duplicate exact TopK readers share their result and inner window work. +/// #922: duplicate exact TopK queries share their result and inner window work. #[test] fn duplicate_exact_topk_shares_work() { let mut w = workload(924); @@ -124,7 +124,7 @@ fn duplicate_exact_topk_shares_work() { assert!(duplicate <= single * 1.001, "{duplicate} vs {single}"); } -/// #924/#925 SQL variants: nested windows and slower readers reuse one sum. +/// #924/#925 SQL variants: nested windows and slower queries reuse one sum. #[test] fn sql_sum_shares_nested_windows_and_slower_readers() { let mut w = workload(922); diff --git a/crates/logical-asap-optimization/src/deployment_capabilities.rs b/crates/logical-asap-optimization/src/deployment_capabilities.rs index 5caa09a78..add7d3ddd 100644 --- a/crates/logical-asap-optimization/src/deployment_capabilities.rs +++ b/crates/logical-asap-optimization/src/deployment_capabilities.rs @@ -13,16 +13,16 @@ //! accumulators and plain exact execution are always supported; //! * whether raw data samples are retained. If they are, Stage 3 counts them on //! disk once per source, at the longest window a query-time scan of it -//! reads. If they are not, raw data samples are not kept after they arrive, so +//! reads. If they are not, raw data samples are not retained after they arrive, so //! a candidate that scans raw data at query time is invalid (the //! rejection lists every such scan): only ingestion-time work reads raw //! rows. Stored data (at rest or mixed) with no raw data retained is a //! contradiction, refused as bad input ([`DeploymentCapabilities::validate`]); //! * whether query-time results are retained across evaluations. If they -//! are, Stage 2 also offers a tumbling-window chain whose readers evaluate +//! are, Stage 2 also offers a tumbling-window chain whose queries evaluate //! together at its tumbling-window width as query-time `AcrossEvaluations`: each //! evaluation builds one new tumbling window and keeps the others for the next ones, -//! covering the readers' longest look-back. A chain two or more queries +//! covering the queries' longest look-back. A chain two or more queries //! read keeps its `OneEvaluation` option too (shared within one evaluation), //! and Stage 3 picks the cheaper. If they are not, a query-time result is //! reused only within one evaluation. @@ -50,9 +50,9 @@ pub struct DeploymentCapabilities { /// it can answer from one; `None` (the default) supports every family /// and readout the planner can build. pub supported_summaries: Option>, - /// Whether raw data samples are kept for query-time scans; default `true`. + /// Whether raw data samples are retained for query-time scans; default `true`. pub raw_data_retained: bool, - /// Whether a query-time result may be kept across evaluations + /// Whether a query-time result may be materialized across evaluations /// (query-time `AcrossEvaluations`); default `false`. pub query_time_results_retained: bool, } diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index b9b7a8d4d..185147b38 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -55,7 +55,7 @@ //! candidate. Pass 1 takes the deployment's `AccuracyModel` and evidence //! only to prune summaries no query reading them can use (when they cannot //! accept an output without its guarantee) and sizes every summary for its -//! readers; plan selection (Stage 3) checks each selected candidate's +//! consumers; plan selection (Stage 3) checks each selected candidate's //! guarantee against its query's `AccuracyTarget` and never resizes. /// The deployment's capabilities: supported summaries and what it retains. diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 0c0f53a40..9dbe23a1d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -31,7 +31,7 @@ //! - A measure filter or `HAVING` prunes nothing. The target records the //! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] //! per consumer `Filter` directly above the aggregate: a key-only predicate -//! applied by every reader of the aggregate's group, across queries, may go +//! applied by every consumer of the aggregate's group, across queries, may go //! below the aggregate, anything else stays above the output (approximate //! outputs then get guarantee `None`). So every candidate of a group //! computes the group's own rows. @@ -83,11 +83,11 @@ //! `1 − (1 + ε)^{−1/n}`) and `δ/n` for `n` inputs (#669b). Only the inputs //! approximate in this candidate count. This composition sizes for ε and δ //! per output value; attached to the memo groups, each candidate's -//! approximate inputs are sized for the values its readers' targets cover +//! approximate inputs are sized for the values its consumers' targets cover //! (`δ / (n · V)`, //! [`pass1_memo`](crate::pass1::logical_memo_candidates::pass1_memo)), //! their final sizes. Pass 2 inherits them or takes the strictest shared -//! reader's, and Stage 3 only checks the composed accuracy. +//! consumer's, and Stage 3 only checks the composed accuracy. //! //! Every node of the candidate gets a theoretical guarantee, composed with //! the built-in accuracy model through operators (#669): exact over exact @@ -665,9 +665,9 @@ fn placement(below: bool) -> HavingPlacement { /// An aggregate measure is eligible when it has a realization besides exact /// execution, even if every such realization is pruned; the prune reasons /// are reported on the target. Measure filters and `HAVING` keep every -/// candidate; the target records how they are applied. A group's readers +/// candidate; the target records how they are applied. A group's consumers /// are its parent groups and the query roots that read it; a consumer -/// `Filter`'s key-only `HAVING` is pushed below only when every reader of +/// `Filter`'s key-only `HAVING` is pushed below only when every consumer of /// the group applies the same predicate, so every candidate of a shared /// aggregate computes the group's own rows (#1089). pub fn generate_per_computation_candidates( @@ -697,7 +697,7 @@ pub fn generate_per_computation_candidates( let mut seen = HashSet::new(); // Each aggregate group with the first query that reaches it. let mut aggregates = Vec::new(); - // Every reader of each group's node: `Some(parent)`, or `None` for a + // Every consumer of each group's node: `Some(parent)`, or `None` for a // query root or scalar reference. let mut consumers: HashMap<*const OperatorNode, Vec>>> = HashMap::new(); for (query, root) in roots.iter().enumerate() { @@ -763,7 +763,7 @@ pub fn generate_per_computation_candidates( } } for (filter, predicate) in &filters_above { - // Every reader of the group, across queries, applies it. + // Every consumer of the group, across queries, applies it. let shared_by_all = filters_above.len() == parents.len() && filters_above.iter().all(|(_, other)| other == predicate); entries.push(HavingEntry { @@ -864,7 +864,7 @@ fn close_absorptions(targets: &mut [ComputationTarget]) { /// sum of the inner groups' sums is the sum of their rows), absorbing the /// inner target. Offered whatever else reads the inner sum: it is one more /// candidate of the outer sum's group, and the inner sum's group keeps its -/// own candidates for its other readers. Applied when neither has a filter +/// own candidates for its other consumers. Applied when neither has a filter /// or `HAVING` /// ([`nested_sum_input`]), and when the outer sum is not itself absorbed by /// a merge and has no fused computation yet, and the inner sum has none, so @@ -990,7 +990,7 @@ fn nested_sum_input(target: &OperatorNode) -> Option<(Rc, AggInten /// counts of raw data samples, the heap reads those samples and absorbs the /// per-series target too. Offered whatever else reads the inner aggregate: /// it is one more candidate of the top-k's group, and the absorbed groups -/// keep their own candidates for their other readers. Applied when neither +/// keep their own candidates for their other consumers. Applied when neither /// has a filter or `HAVING`, and when the keyed-additive rule realizes it /// ([`whole_expression_input`]). fn add_whole_expression_targets( @@ -1546,7 +1546,7 @@ fn whole_expression( /// `topk_k` the estimated Float64 weight, then the item columns. This is /// the exact top-k's physical shape (its ranked rows' value, then the /// columns naming each ranked item: PromQL labels, SQL group keys), so -/// readers of `[keys.., topk_k]` find those columns where they were. +/// consumers of `[keys.., topk_k]` find those columns where they were. fn ranked_as_topk( target: &OperatorNode, estimate: Rc, diff --git a/crates/logical-asap-optimization/src/stage0/cse.rs b/crates/logical-asap-optimization/src/stage0/cse.rs index d90241d4a..085b0793c 100644 --- a/crates/logical-asap-optimization/src/stage0/cse.rs +++ b/crates/logical-asap-optimization/src/stage0/cse.rs @@ -39,7 +39,7 @@ //! same input rows returns the same output rows //! ([`NonASAPOp::is_shareable_when_equal`]). Equal nodes are //! interned children first, so two equal nodes read one input, and one copy -//! returns each reader the rows it would have computed alone. A unique key +//! returns each consumer the rows it would have computed alone. A unique key //! is not needed: equal per-series quantiles without a provable key are one //! node. An operator that picks rows by input order alone (an unordered //! `LIMIT`) is always inserted fresh, because two copies may return @@ -428,7 +428,7 @@ mod tests { } /// Two equal scans of a keyless table (SQL rows) are one node: each - /// reader reads every row the scan outputs. + /// consumer reads every row the scan outputs. #[test] fn equal_scans_without_a_unique_key_are_shared() { let table_scan = || { @@ -720,7 +720,7 @@ mod tests { } /// `LIMIT 10` without an order may return any 10 rows, so two copies may - /// differ; one shared copy would make both readers see the same rows. + /// differ; one shared copy would make both consumers see the same rows. #[test] fn unordered_limits_are_not_shared() { let limit = || { diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index d5eb6298e..7cac6899e 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -89,7 +89,7 @@ pub enum RejectionCause { /// order. TiedLostOnEnumerationOrder, /// No plan meets every query's accuracy target, latency bound and the - /// catalog's reader constraints together: the selection MILP is + /// catalog's consumer constraints together: the selection MILP is /// infeasible. The reason names a query when it has no plan even alone. Infeasible, } diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index 68e827cac..a2dcf5f48 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -10,7 +10,7 @@ //! Everything else a runtime needs is derived from the DAG //! (`asap_types::ir::materialized_view`): the views to maintain, with what //! each is refreshed from and when ([`InstallablePlan::materialized_views`]), -//! and how far back each raw-data source is kept for query-time scans +//! and how far back each raw-data source is retained for query-time scans //! ([`InstallablePlan::raw_data_retention`]). A runtime reads the file with //! [`InstallablePlan::from_json`]. Raw data is read only at the DAG's //! `Scan`s, by their `Source`, and a view by its `MaterializedView`: the @@ -101,7 +101,7 @@ pub struct PredictedUsage { pub average_memory_bytes: f64, /// The most memory live at once. pub peak_memory_bytes: u64, - /// Bytes kept on disk, raw data query-time scans read included. + /// Bytes retained on disk, raw data query-time scans read included. pub disk_bytes: u64, /// CPU cores to provision for the peak. pub peak_cpu_cores: u32, @@ -285,7 +285,7 @@ impl InstallablePlan { Ok(self.dag.ingestion_refresh_groups()?) } - /// How far back each raw-data source is kept for query-time scans + /// How far back each raw-data source is retained for query-time scans /// ([`PhysicalASAPDAG::raw_data_retention`]): the raw data a runtime /// keeps, the raw-data counterpart of each view's retention. pub fn raw_data_retention(&self) -> Result, InstallablePlanError> { @@ -692,7 +692,7 @@ mod tests { pub(super) static COMPILED_NODES: std::cell::Cell = const { std::cell::Cell::new(0) }; } - /// A p99 panel over a 300 s window every second, with no raw data kept: + /// A p99 panel over a 300 s window every second, with no raw data retained: /// one DDSketch per series in each of 300 one-second tumbling windows built at /// ingestion, about 300 materialized views in a DAG of about 900 nodes. /// Each refresh (here one refresh group of all the tumbling windows) is compiled diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 07e6596fd..539ac72e8 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -212,9 +212,9 @@ pub struct PlanOutput { /// How plan selection chose the plan, when the pass selects among priced /// candidates. pub selection: Option, - /// For each tumbling-window chain of the selected plan kept beyond one + /// For each tumbling-window chain of the selected plan materialized beyond one /// evaluation, by node id in its physical DAG, how long its completed - /// tumbling windows are kept ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); + /// tumbling windows are retained ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); /// empty for a pass that does not set it. pub tumbling_window_retention_ms: BTreeMap, } diff --git a/crates/planner/tests/sql_table_scans.rs b/crates/planner/tests/sql_table_scans.rs index 6dda13659..9dd9b41a2 100644 --- a/crates/planner/tests/sql_table_scans.rs +++ b/crates/planner/tests/sql_table_scans.rs @@ -298,7 +298,7 @@ fn run_data_plane(plan: &InstallablePlan) -> Vec>> { .collect() } -/// A 5 and a 15 minute distinct count kept in tumbling windows at ingestion +/// A 5 and a 15 minute distinct count materialized in tumbling windows at ingestion /// time (the deployment keeps no raw data): each tumbling window is built in an ingestion run from the table rows of its /// window, and both queries answer exactly from the tumbling windows (the sketch is /// exact below its capacity), as the query-time reference does. diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index 8dc00ba18..9a8b45ffc 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -5,12 +5,12 @@ //! `retention_ms` installation sets on its nodes //! ([`PhysicalASAPDAG::set_retention`]). //! -//! A **materialized view** is a node whose output is kept across +//! A **materialized view** is a node whose output is materialized across //! evaluations: its lifetime is `AcrossEvaluations`, at ingestion time or //! at query time, and it carries its `retention_ms`. A node materialized for //! `OneEvaluation` is not a view: its output is discarded after one //! evaluation. A query-time `Scan` carries how long its source's rows are -//! kept for it (`None` when it reads them unbounded). +//! retained for it (`None` when it reads them unbounded). //! //! ## Refreshing a view //! @@ -82,11 +82,11 @@ use crate::ir::properties::summary_coverage::{CoverageError, SelectionBox, Summa use crate::physical::Lifetime; /// One materialized view of a DAG: what its stored state is, what it is -/// refreshed from, when, and for how long it is kept. +/// refreshed from, when, and for how long it is retained. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MaterializedView { - /// The node whose output is kept. + /// The node whose output is materialized. pub node_id: PhysicalASAPNodeId, /// Which run refreshes it. pub(crate) refresh: MaterializedViewRefresh, @@ -240,7 +240,7 @@ pub enum MaterializedViewRefresh { /// samples have arrived and the views it reads are refreshed), and /// keeps the result. AtIngestion, - /// Refreshed at query time and kept across evaluations + /// Refreshed at query time and materialized across evaluations /// (query-time `AcrossEvaluations`): when an evaluation reads this /// view for a selection it has not been refreshed for yet, the runtime /// runs the node in a query run over its inputs, keeps the result, and @@ -375,18 +375,18 @@ pub enum InstallableDagError { /// Bad input (a plan file): view `node_id` has no `retention_ms`. #[error("materialized view {node_id} has no retention_ms")] ViewWithoutRetention { node_id: PhysicalASAPNodeId }, - /// Bad input (a plan file): view `node_id` is kept for less than the + /// Bad input (a plan file): view `node_id` is retained for less than the /// data it summarizes reaches back, so a later evaluation would miss /// it. - #[error("materialized view {node_id} is kept for less than its data reaches back")] + #[error("materialized view {node_id} is retained for less than its data reaches back")] ViewRetentionTooShort { node_id: PhysicalASAPNodeId }, /// Bad input (a plan file): query-time `Scan` `node_id` reads a bounded /// window but has no `retention_ms`. #[error("query-time scan {node_id} reads a bounded window but has no retention_ms")] ScanWithoutRetention { node_id: PhysicalASAPNodeId }, /// Bad input (a plan file): query-time `Scan` `node_id` keeps its rows - /// for less than its readers' window reaches back. - #[error("query-time scan {node_id} is kept for less than its window reaches back")] + /// for less than its consumers' window reaches back. + #[error("query-time scan {node_id} is retained for less than its window reaches back")] ScanRetentionTooShort { node_id: PhysicalASAPNodeId }, } @@ -569,25 +569,25 @@ impl PhysicalASAPDAG { /// /// A view takes `tumbling_window_retention_ms_by_node`'s value (Stage 3's retention /// of a tumbling window's chain: its offset plus its window plus the longest - /// range-evaluation span of its readers); otherwise how far back its - /// data reaches, plus, for a view built from other views, its readers' + /// range-evaluation span of its consumers); otherwise how far back its + /// data reaches, plus, for a view built from other views, its consumers' /// longest range-evaluation span (`reader_span_ms_by_node`). Then each - /// view is kept at least as long as every view built from it needs it: + /// view is retained at least as long as every view built from it needs it: /// as long as that view's own retention when it is refreshed on first /// read (a range request may refresh it for its oldest step), else as /// far back as that view's data reaches. This is each node's own /// requirement: a runtime that stores identical outputs of several /// nodes once keeps them for the largest requirement among those nodes. /// - /// A query-time `Scan` keeps its rows for its readers' window + /// A query-time `Scan` keeps its rows for its consumers' window /// ([`Self::scan_window`]'s `lookback_ms`) plus their longest /// range-evaluation span, and at least the retention of every view /// refreshed on first read from it (which may be refreshed for its /// oldest step); `None` when its window is unbounded. In an installed - /// DAG a chain kept across evaluations is its expanded tumbling windows, so its + /// DAG a chain materialized across evaluations is its expanded tumbling windows, so its /// scan keeps the reach of all of them: reusing a tumbling window refreshed at an /// earlier evaluation is optional, and a restart or an older range step - /// refreshes older tumbling windows from raw data. (Stage 3 prices a kept chain on + /// refreshes older tumbling windows from raw data. (Stage 3 prices a materialized chain on /// the unexpanded DAG as reading its newest tumbling window.) pub fn set_retention( &mut self, @@ -623,8 +623,8 @@ impl PhysicalASAPDAG { MaterializedViewRefresh::AtIngestion => reach_ms_by_node[&view.node_id], }; for input in view.inputs.materialized_views.clone() { - let kept = &mut views[position_of[&input]].retention_ms; - *kept = (*kept).max(needed_ms); + let retained = &mut views[position_of[&input]].retention_ms; + *retained = (*retained).max(needed_ms); } } let mut retention_ms_by_node: HashMap> = views @@ -685,9 +685,13 @@ impl PhysicalASAPDAG { return Err(InstallableDagError::ScanWithoutRetention { node_id: node.id }) } }; - match retention.iter_mut().find(|kept| &kept.source == source) { - Some(kept) => { - kept.lookback_ms = kept.lookback_ms.zip(lookback_ms).map(|(a, b)| a.max(b)) + match retention + .iter_mut() + .find(|retained| &retained.source == source) + { + Some(retained) => { + retained.lookback_ms = + retained.lookback_ms.zip(lookback_ms).map(|(a, b)| a.max(b)) } None => retention.push(RawDataRetention { source: source.clone(), @@ -698,7 +702,7 @@ impl PhysicalASAPDAG { Ok(retention) } - /// [`Self::scan_window`] of each query-time `Scan`, with no chain kept, + /// [`Self::scan_window`] of each query-time `Scan`, with no chain materialized, /// by id. fn query_time_scan_windows(&self) -> HashMap { let scans = self @@ -817,7 +821,7 @@ impl PhysicalASAPDAG { /// `inputs` summarizes reaches: the farthest of its raw-data windows and of /// the views it reads (`reach_ms_by_node`, each already computed). /// Whether `node` is a `Scan` run at query time, which carries how long its -/// source's rows are kept for it. +/// source's rows are retained for it. pub(crate) fn is_query_time_scan(node: &PhysicalASAPDAGNode) -> bool { matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. })) && node.output_state.timing == ExecutionTiming::QueryTime diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index ca69afbfd..a0974bf3e 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -8,7 +8,7 @@ //! the family merges (the family-capability table), and the executor runs //! the chain as its tumbling windows (`expand_tumbling_window_chains`). A merge of only the //! newest `m < n` tumbling windows (`SummaryMerge::newest_tumbling_windows`) is the state of one -//! build over `TimeRange(m·w)` ([`merged_window_build`]): one chain kept for +//! build over `TimeRange(m·w)` ([`merged_window_build`]): one chain materialized for //! the longest window serves every query whose window, on the same tumbling windows, //! ends at the same time and is shorter. #[cfg(any(test, feature = "test-support"))] diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index caccbb4b2..f6d383b28 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -171,7 +171,7 @@ pub struct PhysicalASAPDAG { pub enum PhysicalASAPDAGValidationError { /// A node whose timing and lifetime cannot go together: a planner bug. #[error("node {node:?}: {reason}")] - InvalidPhysicalProperties { + PhysicalProperties { node: PhysicalASAPNodeId, #[source] reason: PhysicalPropertiesError, @@ -225,7 +225,7 @@ pub enum PhysicalASAPDAGValidationError { SummaryGroupingMismatch { node: PhysicalASAPNodeId }, } -/// Which rows of a `Scan` its readers read, back from an evaluation time `T` +/// Which rows of a `Scan` its consumers read, back from an evaluation time `T` /// ([`PhysicalASAPDAG::scan_window`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ScanWindow { @@ -377,8 +377,8 @@ impl PhysicalASAPDAG { Ok(rebuilt) } - /// The rows of the `Scan` `scan` its readers read, back from the - /// evaluation time `T`: the union of each reader's window. A reader + /// The rows of the `Scan` `scan` its consumers read, back from the + /// evaluation time `T`: the union of each consumer's window. A consumer /// reaches the `Scan` through `TimeShift`s without an `@` anchor /// (offsets summing to `s`) and is /// - a `TimeRange(w)`, range or instant: `(T - s - w, T - s]`; @@ -387,7 +387,7 @@ impl PhysicalASAPDAG { /// chain in `materialized_chains` (materialized across evaluations, so each /// evaluation builds one tumbling window). /// - /// `Unbounded` when a root, any other reader, or a shift with an `@` + /// `Unbounded` when a root, any other consumer, or a shift with an `@` /// anchor reads the `Scan` or one of those shifts, or nothing reads it. pub fn scan_window( &self, @@ -398,7 +398,7 @@ impl PhysicalASAPDAG { } /// [`Self::scan_window`] of each of `scans`, by id, with the DAG - /// indexed once for all of them: O(nodes + edges + the readers walked). + /// indexed once for all of them: O(nodes + edges + the consumers walked). pub fn scan_windows( &self, scans: impl IntoIterator, @@ -552,7 +552,7 @@ impl PhysicalASAPDAG { if node.output_state.timing == ExecutionTiming::IngestionTime && matches!(node.lifetime, Lifetime::OneEvaluation(_)) { - return Err(PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + return Err(PhysicalASAPDAGValidationError::PhysicalProperties { node: node.id, reason: PhysicalPropertiesError::OneEvaluationAtIngestion, }); diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs index 3f7910f0b..6e886c505 100644 --- a/crates/types/src/stages/physical/graph.rs +++ b/crates/types/src/stages/physical/graph.rs @@ -26,7 +26,7 @@ pub struct SelectedEdge { /// The selected plan as a DAG with one node per run of a candidate: what /// Stage 3 selects and the facade builds through /// [`AssignmentRealizer::realize`](crate::stages::AssignmentRealizer::realize). -/// A candidate that runs twice (recomputed by two readers, or two shared +/// A candidate that runs twice (recomputed by two consumers, or two shared /// instances over different inputs) is two nodes. #[derive(Debug, Clone, PartialEq)] pub struct DagOfSelectedCandidates { diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index 82984ba3b..a2f8446f4 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,6 +1,6 @@ //! What a stage removed, and why: one shape for every stage (#1047 P6). //! Each stage removes only what its own knowledge rules out (option C): -//! Stage 2 what cannot be implemented, built or kept. Stage 0 fails with an +//! Stage 2 what cannot be implemented, built or materialized. Stage 0 fails with an //! error, Pass 1 records nothing it leaves out (#1205 will show it), and //! Pass 2 only adds candidates. Stage 3 prunes before //! its selection MILP (dominated and uncostable candidates); its @@ -21,7 +21,7 @@ pub struct PrunedCandidate { } /// Why a stage removed a candidate: first Stage 2's (the physical candidate -/// cannot be built, used or kept), then Stage 3's, before its selection +/// cannot be built, used or materialized), then Stage 3's, before its selection /// MILP. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PruneReason { @@ -40,14 +40,14 @@ pub enum PruneReason { /// cycle, on an input that cannot be built. InputHasNoBuildableProducer, /// Stage 2: no query output can read it, directly or through other - /// candidates, once each candidate's allowed readers are applied. + /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, /// Stage 2: it is shared within one query evaluation /// (`OneEvaluation`), which needs at least two reading queries, and fewer /// queries can reach it. TooFewQueryReaders, /// Stage 3: the candidate it names, in the same memo group with the - /// same producers and readers, is at least as good in every respect + /// same producers and consumers, is at least as good in every respect /// (guarantee, usage, latency, output, held state), so no optimal plan /// needs this one. DominatedBy(PhysicalCandidateId), diff --git a/crates/types/src/stages/realizer.rs b/crates/types/src/stages/realizer.rs index ab1258f16..96ad555a1 100644 --- a/crates/types/src/stages/realizer.rs +++ b/crates/types/src/stages/realizer.rs @@ -33,21 +33,21 @@ impl From<&str> for PhysicalCandidateError { } /// A realized selection (was `SelectedPhysicalCandidate`): the physical -/// plan Stage 2 built for it, how long each kept tumbling-window chain of that plan +/// plan Stage 2 built for it, how long each materialized tumbling-window chain of that plan /// keeps its tumbling windows, and the operator node each selected node became. #[derive(Debug, Clone)] pub struct RealizedPlan { /// The selection rebuilt as one physical plan, one operator node per /// selected node, each with its candidate's timing and lifetime. pub plan: PhysicalWorkloadPlan, - /// For each tumbling-window chain of `plan` kept beyond one evaluation - /// (maintained at ingestion time, or built at query time and kept + /// For each tumbling-window chain of `plan` materialized beyond one evaluation + /// (maintained at ingestion time, or built at query time and materialized /// across evaluations), by node id in `plan.dag`: how far back from the /// newest evaluation the runtime must keep its completed tumbling windows, in /// milliseconds. It is how far its tumbling windows reach back: the offset its /// newest tumbling window ends before the evaluation (`[60m] offset 10m`: 10 min), /// plus the chain's window, plus the longest range-evaluation span of - /// its readers, so the earliest step of the next request still finds + /// its queries, so the earliest step of the next request still finds /// its tumbling windows; without an offset or range evaluations, just the window. /// Empty when the plan keeps no tumbling windows. pub tumbling_window_retention_ms: BTreeMap, diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index d33f4f50a..10a46370a 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -110,7 +110,7 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { scalar_only.nodes[state].lifetime = Lifetime::OneEvaluation(StorageMedium::Disk); assert_eq!( scalar_only.validate(), - Err(PhysicalASAPDAGValidationError::InvalidPhysicalProperties { + Err(PhysicalASAPDAGValidationError::PhysicalProperties { node: scalar_only.nodes[state].id, reason: PhysicalPropertiesError::OneEvaluationAtIngestion, }) diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 07f03ee49..6caa8b0a8 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -202,7 +202,7 @@ deployment knows which outputs to materialize, for how long and where; validatio rejects a timing and lifetime that cannot go together (an ingestion-time output materialized for `OneEvaluation`) and a `SummaryAgg` or `MaintainPopulation` that is not materialized outside query time -(`PhysicalASAPDAGValidationError::InvalidPhysicalProperties`). Ingestion +(`PhysicalASAPDAGValidationError::PhysicalProperties`). Ingestion work cannot depend on a future query result. The deployment executes the plan as given and does not change its lifetime or timing ([#509 Assumption 8](../proposals/planner-layering.md#assumptions)); it must From f4c03975b878c4f962f926bf6fad5faaa6956f6f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:55:30 +0000 Subject: [PATCH 09/30] refactor(types): Memo methods return Result; CandidateError (#1207 PR 0) find_or_add_group returns the node rules' SchemaDerivationError and add_candidate a CandidateError (the node rules, or an output that differs from the group's). Bodies stay todo!() until PR 2. --- crates/types/src/stages/candidates.rs | 28 +++++++++++++++++++++++---- crates/types/src/stages/mod.rs | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 0d93ba367..9cb82ad91 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -9,6 +9,7 @@ use super::{MemoGroup, MemoGroupId}; use crate::ir::schema::{SketchAlgorithm, SketchParams}; use crate::ir::Operator; use crate::ir::QueryRoot; +use crate::ir::SchemaDerivationError; /// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`]). @@ -29,18 +30,24 @@ pub struct Memo { } impl Memo { - /// Hash-consing: the group with this expression, added if new. - pub fn find_or_add_group(&mut self, expression: Operator) -> MemoGroupId { + /// Hash-consing: the group with this expression, added if new. Fails + /// when the expression breaks the node rules (schema derivation, input + /// validation, merge check). + pub fn find_or_add_group( + &mut self, + expression: Operator, + ) -> Result { let _ = expression; todo!("#1207 PR 2: hash-consing over groups") } - /// Adds `candidate` to `group`'s candidates. + /// Adds `candidate` to `group`'s candidates. Fails when its expression + /// breaks the node rules or its output differs from the group's. pub fn add_candidate( &mut self, group: MemoGroupId, candidate: CandidateForMemoGroup, - ) -> CandidateForMemoGroupId { + ) -> Result { let _ = (group, candidate); todo!("#1207 PR 2") } @@ -94,6 +101,19 @@ impl Memo { } } +/// A candidate [`Memo::add_candidate`] rejects; the memo is unchanged. +#[derive(Debug, thiserror::Error)] +pub enum CandidateError { + /// The candidate's expression fails the node rules (schema derivation, + /// input validation, merge check): a bug in the rule that built it. + #[error("the candidate's expression breaks the node rules: {0}")] + Schema(SchemaDerivationError), + /// Its output schema or output kind differs from the group's: it does + /// not compute the group's result (a bug in the rule that built it). + #[error("the candidate's output differs from group {group:?}'s")] + OutputDiffersFromGroup { group: MemoGroupId }, +} + /// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. #[derive(Debug, Clone)] pub struct CandidateForMemoGroup { diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index a8500534b..13cb000d3 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -16,7 +16,7 @@ mod pruning; mod realizer; pub use candidates::{ - parameter_list, producer_satisfies_logical_requirements, CandidateExpression, + parameter_list, producer_satisfies_logical_requirements, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, WindowLayout, }; From 98f4eb25395c814ce4177f48195d4ecb1ee3ea21 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:00:43 +0000 Subject: [PATCH 10/30] refactor: Memo groups private behind groups(); Stage 0 through the Memo (#1207 PR 0) Memo::groups is private, read through groups(); groups are added only through find_or_add_group and queries through add_query (todo!() until PR 2). memo_of_queries returns Result and its body is todo!() until PR 2, which builds Stage 0 on find_or_add_group; Stage 0's node-level CSE (cse.rs), its test_support export and the dead share_identical helper are deleted. --- crates/devtools/src/bin/asap_planner.rs | 2 +- crates/devtools/src/bin/sketch_coverage.rs | 8 +- crates/devtools/src/candidate_labels.rs | 20 +- crates/devtools/src/planning_stages.rs | 3 +- crates/devtools/src/planning_stages_text.rs | 4 +- crates/executor/tests/common/mod.rs | 9 +- crates/frontend-common/src/canonicalize.rs | 2 +- crates/frontend-promql/tests/support.rs | 9 +- crates/frontend-sql/tests/frequency_l2.rs | 9 +- crates/integration-tests/src/lib.rs | 12 +- .../tests/per_computation_candidates_sql.rs | 15 +- .../tests/planner_layering_common/mod.rs | 35 - .../tests/stage1_logical_asap_optimization.rs | 55 +- crates/logical-asap-optimization/src/lib.rs | 2 +- .../src/pass1/per_computation_candidates.rs | 20 +- .../pass1/per_computation_candidates_tests.rs | 181 ++-- .../logical-asap-optimization/src/stage0.rs | 238 +---- .../src/stage0/cse.rs | 886 ------------------ .../src/test_support.rs | 9 +- crates/planner/src/run.rs | 6 +- crates/types/src/ir/schema/schema_support.rs | 2 +- crates/types/src/stages/candidates.rs | 17 +- 22 files changed, 224 insertions(+), 1320 deletions(-) delete mode 100644 crates/logical-asap-optimization/src/stage0/cse.rs diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 2c4c75e03..b49689ec5 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -872,7 +872,7 @@ impl MemoNodes { /// before) has no such candidate. fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) -> Vec { let ids = |groups: &[MemoGroupId]| groups.iter().map(|g| g.0).collect::>(); - memo.groups + memo.groups() .iter() .enumerate() .map(|(id, group)| { diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index ebf91c9e0..e11557d33 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -60,8 +60,12 @@ fn analyze_corpus( let mut rejected = 0; let mut summaries = Vec::new(); for (id, root) in roots { + let Ok(memo) = memo_of_queries(&[QueryRoot::Operator(root)]) else { + rejected += 1; + continue; + }; let Ok(pass1) = pass1_memo( - memo_of_queries(&[QueryRoot::Operator(root)]), + memo, std::slice::from_ref(accuracy), &IgnoreAccuracyGuarantees, &NoAccuracyEvidence, @@ -76,7 +80,7 @@ fn analyze_corpus( // excluded. A way to compute a measure that Pass 1 offers but cannot // compose into a candidate is not counted. let mut offered = BTreeSet::new(); - for candidate in pass1.groups.iter().flat_map(|group| &group.candidates) { + for candidate in pass1.groups().iter().flat_map(|group| &group.candidates) { if candidate.fused_groups.is_empty() { summary_family(&candidate.expression.operator, &mut offered); } diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 7736878eb..8cf029620 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -398,11 +398,13 @@ mod tests { #[test] fn connected_labels_describe_selected_states() { let workload = example1(); - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries( + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries( &lower_promql_roots(&workload).unwrap(), - )) - .unwrap(); + ) + .unwrap(), + ) + .unwrap(); let choice = vec![0; inventory.targets.len()]; let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); let label = connected_candidate_label(&roots); @@ -414,11 +416,13 @@ mod tests { #[test] fn connected_labels_name_actual_shared_state() { let workload = example1(); - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries( + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries( &lower_promql_roots(&workload).unwrap(), - )) - .unwrap(); + ) + .unwrap(), + ) + .unwrap(); let roots = compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); let label = connected_candidate_label(&[roots[1].clone(), roots[1].clone()]); diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index fff254628..9efdd4fd9 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -266,7 +266,8 @@ pub fn run_stages_until( until: RunUntil, ) -> Result { validate(&roots)?; - let stage0_dag = memo_of_queries(&roots); + let stage0_dag = + memo_of_queries(&roots).map_err(|e| PlanningStagesError::Validation(e.to_string()))?; let mut stages = PlanningStagesRun { run_until: until, stage0_memo_groups: Some(stage0_dag.clone()), diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index 4742965f0..1692de316 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -186,7 +186,7 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi // The group lines share one set of column widths, with each group's // candidates printed under its line. let group_rows: Vec> = memo - .groups + .groups() .iter() .enumerate() .map(|(id, group)| { @@ -209,7 +209,7 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi if !with_candidates { continue; } - let _ = (previous, &memo.groups[id].candidates); + let _ = (previous, &memo.groups()[id].candidates); todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index cfacc2731..31187af65 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -28,11 +28,10 @@ pub fn computation_candidates(root: Rc) -> Vec> { use asap_logical_asap_optimization::test_support::{ compose_logical_candidate, generate_per_computation_candidates, }; - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) - .expect("Pass 1 accepts the query"); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) + .expect("Pass 1 accepts the query"); let mut choices = vec![vec![0; inventory.targets.len()]]; for (index, target) in inventory.targets.iter().enumerate() { for choice in 1..target.choice_count() { diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 707acd89e..43495769f 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_logical_asap_optimization`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 28cd4fc27..d6a3912e1 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -148,11 +148,10 @@ fn local_computation_candidates(root: Rc) -> Result bool, ) -> Vec<(usize, usize)> { let mut found: Vec<_> = memo - .groups + .groups() .iter() .enumerate() .filter(|(_, group)| group.reached_by_queries() == [q]) @@ -332,9 +332,9 @@ pub mod planning { .collect(); // Group indices don't give the order: Pass 2 appends groups at the // end. Depth from the root does. - let mut depth = vec![usize::MAX; memo.groups.len()]; + let mut depth = vec![usize::MAX; memo.groups().len()]; let mut pending: std::collections::VecDeque<_> = memo - .groups + .groups() .iter() .enumerate() .filter(|(group, _)| { @@ -350,7 +350,7 @@ pub mod planning { } depth[group] = d; pending.extend( - memo.groups[group] + memo.groups()[group] .child_groups .iter() .map(|child| (child.0 as usize, d + 1)), diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 46fa6c962..564381f0a 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -25,9 +25,9 @@ async fn targets(sql: &str) -> Vec { let root = lower_sql(sql, &catalog, accuracy) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) + generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() .targets } @@ -211,11 +211,10 @@ async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) + .unwrap(); let name = |expr: &SummaryInputExpr| match expr { SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), SummaryInputExpr::Column(ColumnRef::Qualified { table, name }) => { diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index ee5aa4f5d..283b0f173 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -40,25 +40,6 @@ pub fn needs_merge(form: WindowForm, query_window_ms: u64) -> bool { } } -/// `roots` with every identical sub-DAG made one node (the identical- -/// expression rule, as the memo's group identity applies it). -#[allow(dead_code)] -pub fn share_identical(roots: &[QueryRoot]) -> Vec { - asap_logical_asap_optimization::test_support::share_common_sub_dags( - roots - .iter() - .enumerate() - .map(|(id, root)| match root { - QueryRoot::Operator(node) => (id, std::rc::Rc::clone(node)), - QueryRoot::Scalar(_) => panic!("operator root"), - }) - .collect(), - ) - .into_iter() - .map(|(_, node)| QueryRoot::Operator(node)) - .collect() -} - /// `roots` with no node shared between queries: each query reads its own /// copy of every node it reaches (sharing within one query is kept). Pass 1 /// composes a candidate from memo groups, so queries that read an identical @@ -92,22 +73,6 @@ pub fn unshare(roots: &[QueryRoot]) -> Vec { .collect() } -/// The number of distinct nodes (by identity) `roots` reach: smaller after -/// [`share_identical`] exactly when some sub-DAG was shared. -#[allow(dead_code)] -pub fn distinct_nodes(roots: &[QueryRoot]) -> usize { - roots - .iter() - .filter_map(|root| match root { - QueryRoot::Operator(node) => Some(node), - QueryRoot::Scalar(_) => None, - }) - .flat_map(asap_types::ir::OperatorNode::reachable) - .map(|node| std::rc::Rc::as_ptr(&node)) - .collect::>() - .len() -} - // ── Query-language frontends ────────────────────────────────────────── /// PromQL workload roots, each series' identity as a column. diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index 2fdaf76ce..fab612462 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -97,11 +97,10 @@ fn output_columns(root: &QueryRoot) -> Vec<(String, String, bool)> { /// Compose every candidate of `root`; returns how many there were. fn compose_all(label: &str, root: QueryRoot) -> usize { let expected = output_columns(&root); - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - root, - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[root]).unwrap(), + ) + .unwrap(); let radix: Vec<_> = inventory .targets .iter() @@ -177,11 +176,10 @@ fn promql_candidates_compose() { /// The summary estimates of `root`'s candidates that vary one target's /// choice, every other target taking its first. fn composed_estimates(root: QueryRoot) -> Vec> { - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - root, - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[root]).unwrap(), + ) + .unwrap(); let targets = inventory.targets.len(); let choices: Vec<_> = inventory .targets @@ -268,11 +266,10 @@ fn topk_over_an_aggregate_ranks_its_labels() { async fn subquery_in_a_measure_filter_is_substituted() { let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ FROM flows GROUP BY src"; - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - sql(query).await, - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[sql(query).await]).unwrap(), + ) + .unwrap(); assert_eq!(inventory.targets.len(), 2); let inner = inventory .targets @@ -403,11 +400,11 @@ async fn avg_as_sum_and_count( query: &str, ) -> (Rc, Rc) { let exact = lower_sql(query, catalog, accuracy()).await.unwrap(); - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - QueryRoot::Operator(exact.clone()), - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(exact.clone())]) + .unwrap(), + ) + .unwrap(); let choice: Vec<_> = inventory .targets .iter() @@ -538,11 +535,10 @@ async fn filtered_sql_avg_is_null_without_matching_rows() { /// PromQL `avg` divides its `sum` and `count` vectors with a `BinaryOp`. #[test] fn promql_avg_divides_sum_and_count_vectors() { - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - promql("avg by (job) (m)"), - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[promql("avg by (job) (m)")]).unwrap(), + ) + .unwrap(); assert_eq!( inventory.targets[0].ways_to_compute[0], HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]) @@ -581,11 +577,10 @@ async fn every_candidate( Result, asap_logical_asap_optimization::PerComputationCandidateError>, )> { let root = lower_sql(query, catalog, accuracy()).await.unwrap(); - let inventory = - generate_per_computation_candidates(&asap_logical_asap_optimization::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) + .unwrap(); assert_eq!(inventory.targets.len(), 1, "{query}"); inventory.targets[0] .ways_to_compute diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index 185147b38..a94771df1 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -32,7 +32,7 @@ //! re-exports the internals other crates' tests call. //! //! - `stage0` — Stage 0: hash-consing the workload's logical DAG into memo -//! groups (`stage0::cse`), behind [`memo_of_queries`]. +//! groups through `Memo::find_or_add_group`, behind [`memo_of_queries`]. //! - `pass1::per_computation_candidates` — Pass 1: per-computation candidate //! generation and composition, behind //! [`pass1_memo`]. diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 9dbe23a1d..f5a81d896 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -674,7 +674,7 @@ pub fn generate_per_computation_candidates( memo: &Memo, ) -> Result { let logical_nodes: Rc<[Rc]> = memo - .groups + .groups() .iter() .map(|group| Rc::clone(&group.logical_node)) .collect(); @@ -2893,9 +2893,9 @@ mod tests { // as a column; a query over aggregated rows is planned over labels. let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) .unwrap_or(root); - generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) + generate_per_computation_candidates( + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() } @@ -3009,9 +3009,9 @@ mod tests { "topk by (job) (2, sum by (service, job) (rate(m[1m])))", AccuracyTarget::max_error(0.1), ); - let inventory = generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) + let inventory = generate_per_computation_candidates( + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap(); let topk = topk_target(&inventory); let mut heaps = 0; @@ -3781,9 +3781,9 @@ mod tests { /// (closed) rows a grouped top-k gets no whole-expression heap. fn inventory_at(query: &str, accuracy: AccuracyTarget) -> PerComputationCandidates { let root = crate::test_support::lower_promql(query, accuracy); - generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) + generate_per_computation_candidates( + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() } diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs index 757c781fc..15a95ad23 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs @@ -128,7 +128,8 @@ fn scalar_root_producers_are_discovered_once() { QueryRoot::Scalar(ScalarExpr::ScalarSubquery(producer.clone())), QueryRoot::Operator(producer.clone()), ]; - let candidates = generate_per_computation_candidates(&memo_of_queries(&roots)).unwrap(); + let candidates = + generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); assert_eq!(candidates.roots.len(), 2); for root in &candidates.roots { root.validate_structure().unwrap(); @@ -175,7 +176,8 @@ fn exact_promql(query: &str) -> Vec { #[test] fn example_1_exact_q1_keeps_exact_accumulators_only() { let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); - let candidates = generate_per_computation_candidates(&memo_of_queries(&roots)).unwrap(); + let candidates = + generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); let found: Vec<_> = candidates .targets .iter() @@ -208,7 +210,9 @@ fn asap_input_and_invalid_accuracy_are_rejected() { })) .unwrap(); assert!(matches!( - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(summary)])), + generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(summary)]).unwrap() + ), Err(PerComputationCandidateError::ContainsAsap) )); let epsilon_delta = @@ -248,9 +252,10 @@ fn asap_input_and_invalid_accuracy_are_rejected() { vec![], None, ); - let error = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(invalid)])) - .unwrap_err(); + let error = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(invalid)]).unwrap(), + ) + .unwrap_err(); assert!( matches!( &error, @@ -308,9 +313,10 @@ fn multi_measure_aggregate_has_one_target_per_measure() { vec![], None, ); - let inventory = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(node)])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(node)]).unwrap(), + ) + .unwrap(); let candidates = &inventory.targets; assert_eq!( candidates @@ -335,12 +341,12 @@ fn aggregate_without_realization_is_not_a_target() { col: Some(1), population: true, }); - assert!( - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(node)])) - .unwrap() - .targets - .is_empty() - ); + assert!(generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(node)]).unwrap() + ) + .unwrap() + .targets + .is_empty()); } /// A filtered SQL count keeps its executable accumulator and records its @@ -356,9 +362,10 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { vec![Some(always()), None], None, ); - let inventory = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(filtered)])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(filtered)]).unwrap(), + ) + .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 2); for (index, target) in targets.iter().enumerate() { @@ -440,9 +447,10 @@ fn equals(column: usize, value: ScalarValue) -> Predicate { fn key_only_having_is_pushed_below_and_keeps_candidates() { let node = grouped_count(None); let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); - let inventory = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root.clone())])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root.clone())]).unwrap(), + ) + .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 1); assert_eq!(**inventory.aggregate(&targets[0]), *node); @@ -462,10 +470,13 @@ fn key_only_having_is_pushed_below_and_keeps_candidates() { #[test] fn value_having_stays_above_output_and_keeps_candidates() { let value = || equals(1, ScalarValue::Int64(10)); - let inventory = generate_per_computation_candidates(&memo_of_queries(&[ - QueryRoot::Operator(having_filter(grouped_count(None), value())), - QueryRoot::Operator(grouped_count(Some(value()))), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(having_filter(grouped_count(None), value())), + QueryRoot::Operator(grouped_count(Some(value()))), + ]) + .unwrap(), + ) .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 2); @@ -487,10 +498,13 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { let node = grouped_count(None); let key = equals(0, ScalarValue::Utf8("a".into())); let filtered = having_filter(node.clone(), key.clone()); - let targets = generate_per_computation_candidates(&memo_of_queries(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(node.clone()), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(node.clone()), + ]) + .unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1); @@ -506,10 +520,9 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { // Two consumers applying the same key-only predicate push it below. let first = having_filter(node.clone(), key.clone()); let second = having_filter(node, key); - let targets = generate_per_computation_candidates(&memo_of_queries(&[ - QueryRoot::Operator(first), - QueryRoot::Operator(second), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(first), QueryRoot::Operator(second)]).unwrap(), + ) .unwrap() .targets; assert!(targets[0] @@ -535,10 +548,13 @@ fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { negated: false, }), ); - let inventory = generate_per_computation_candidates(&memo_of_queries(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(exists), - ])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(exists), + ]) + .unwrap(), + ) .unwrap(); let targets = &inventory.targets; let target = targets @@ -563,10 +579,11 @@ fn only_the_adjacent_of_stacked_filters_is_having() { let inner_key = equals(0, ScalarValue::Utf8("a".into())); let inner = having_filter(node.clone(), inner_key.clone()); let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); - let targets = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(outer)])) - .unwrap() - .targets; + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(outer)]).unwrap(), + ) + .unwrap() + .targets; assert_eq!(targets.len(), 1); assert_eq!( targets[0].having_filters, @@ -588,13 +605,16 @@ fn global_aggregate_having_stays_above() { }] }; let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); - let targets = generate_per_computation_candidates(&memo_of_queries(&[ - QueryRoot::Operator(having_filter( - aggregate_over(scan(false), count(), vec![], None), - never.clone(), - )), - QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[ + QueryRoot::Operator(having_filter( + aggregate_over(scan(false), count(), vec![], None), + never.clone(), + )), + QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), + ]) + .unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 2); @@ -645,9 +665,10 @@ fn scalar_reference_filter_is_not_having() { }; let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(scan(false), above_scalar(1, agg.clone())); - let inventory = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root)])) - .unwrap(); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) + .unwrap(); let targets = &inventory.targets; assert_eq!(targets.len(), 1); assert_eq!(**inventory.aggregate(&targets[0]), *agg); @@ -659,10 +680,11 @@ fn scalar_reference_filter_is_not_having() { let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); - let targets = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root.clone())])) - .unwrap() - .targets; + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root.clone())]).unwrap(), + ) + .unwrap() + .targets; assert_eq!(targets.len(), 1); assert_eq!( targets[0].having_filters, @@ -678,7 +700,7 @@ fn scalar_reference_filter_is_not_having() { #[test] fn promql_scalar_operand_is_not_having() { let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); - let targets = generate_per_computation_candidates(&memo_of_queries(&roots)) + let targets = generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()) .unwrap() .targets; assert!(!targets.is_empty()); @@ -691,7 +713,7 @@ fn promql_scalar_operand_is_not_having() { } let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); - let inventory = generate_per_computation_candidates(&memo_of_queries(&roots)).unwrap(); + let inventory = generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); assert!(having_of(&inventory, |intent| matches!( intent, AggIntent::Count { .. } @@ -717,10 +739,9 @@ fn having_shared_with_a_project_consumer_stays_above() { child: node, })) .unwrap(); - let targets = generate_per_computation_candidates(&memo_of_queries(&[ - QueryRoot::Operator(filtered), - QueryRoot::Operator(project), - ])) + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(filtered), QueryRoot::Operator(project)]).unwrap(), + ) .unwrap() .targets; assert_eq!(targets.len(), 1); @@ -754,10 +775,11 @@ fn without_and_per_entity_having_stays_above() { })) .unwrap(); let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); - let targets = - generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator(root)])) - .unwrap() - .targets; + let targets = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) + .unwrap() + .targets; assert_eq!(targets.len(), 1, "{reduction:?}"); let placements: Vec<_> = targets[0] .having_filters @@ -782,9 +804,9 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { cols: vec![0], accuracy: approximate(), }); - let inventory = generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator( - producer.clone(), - )])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(producer.clone())]).unwrap(), + ) .unwrap(); let exact = compose_logical_candidate(&inventory, &[0]).unwrap(); let QueryRoot::Operator(kept) = &exact[0] else { @@ -819,12 +841,15 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { /// The one root of the workload candidate composed from `roots` and `choice`. fn composed(roots: Vec>, choice: &[usize]) -> Vec> { use crate::pass1::per_computation_candidates::compose_logical_candidate; - let inventory = generate_per_computation_candidates(&memo_of_queries( - &roots - .into_iter() - .map(QueryRoot::Operator) - .collect::>(), - )) + let inventory = generate_per_computation_candidates( + &memo_of_queries( + &roots + .into_iter() + .map(QueryRoot::Operator) + .collect::>(), + ) + .unwrap(), + ) .unwrap(); compose_logical_candidate(&inventory, choice) .unwrap() @@ -1066,9 +1091,9 @@ fn own_having_subquery_is_substituted() { use crate::pass1::per_computation_candidates::compose_logical_candidate; let inner = aggregate(quantile()); let outer = grouped_quantile(Some(above_scalar(1, inner.clone()))); - let inventory = generate_per_computation_candidates(&memo_of_queries(&[QueryRoot::Operator( - outer.clone(), - )])) + let inventory = generate_per_computation_candidates( + &memo_of_queries(&[QueryRoot::Operator(outer.clone())]).unwrap(), + ) .unwrap(); assert_eq!(inventory.targets.len(), 2); let kll = 1; @@ -1177,7 +1202,7 @@ fn memo_cached_rules_preserve_occurrence_specific_having() { QueryRoot::Operator(shared), QueryRoot::Operator(having_filter(other, key)), ]; - let memo = memo_of_queries(&roots); + let memo = memo_of_queries(&roots).unwrap(); let cached = generate_per_computation_candidates(&memo).unwrap(); assert_eq!(cached.targets.len(), 1); // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. diff --git a/crates/logical-asap-optimization/src/stage0.rs b/crates/logical-asap-optimization/src/stage0.rs index d2749a7dd..5764cfeaf 100644 --- a/crates/logical-asap-optimization/src/stage0.rs +++ b/crates/logical-asap-optimization/src/stage0.rs @@ -1,230 +1,16 @@ -//! Stage 0: the memo groups of the workload's query roots -//! (a [`Memo`] without candidates), by hash-consing identical logical -//! sub-expressions (the `cse` module). Stage 1 starts from these groups. +//! Stage 0: the memo groups of the workload's query roots (a [`Memo`] +//! without candidates). Each node of the queries' DAGs becomes the group of +//! its expression, found or added by [`Memo::find_or_add_group`], so +//! identical sub-expressions are one group. Stage 1 starts from these groups. -/// Structural common-subexpression elimination (hash-consing) over the -/// logical DAG. -pub(crate) mod cse; - -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use crate::stage0::cse::share_common_sub_dags; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{Memo, MemoGroup, MemoGroupId}; +use asap_types::ir::{QueryRoot, SchemaDerivationError}; +use asap_types::stages::Memo; /// Stage 0: the memo groups of the workload's query roots, query `i` -/// being `roots[i]`, by hash-consing identical logical sub-expressions -/// (`share_common_sub_dags`). -pub fn memo_of_queries(roots: &[QueryRoot]) -> Memo { - let (groups, group_of) = hash_consed(roots); - let group = |node: &Rc| group_of[&Rc::as_ptr(node)]; - let query_roots = roots - .iter() - .map(|root| match root { - QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) - } - }) - .collect(); - Memo { - groups, - query_roots, - } -} - -/// Each original node (of `roots`), by address, to its group. -type GroupOf = HashMap<*const OperatorNode, MemoGroupId>; - -/// The groups of `roots`, children before parents, and each original node's -/// group. A scalar query reaches the groups of the operator nodes its -/// expression reads. -fn hash_consed(roots: &[QueryRoot]) -> (Vec, GroupOf) { - let originals: Vec<(usize, Rc)> = roots - .iter() - .enumerate() - .flat_map(|(query, root)| { - operator_roots(root) - .into_iter() - .map(move |node| (query, node)) - }) - .collect(); - let interned = share_common_sub_dags(originals.clone()); - // Number the interned nodes children first. - let mut groups: Vec = Vec::new(); - let mut id_of: HashMap<*const OperatorNode, MemoGroupId> = HashMap::new(); - for (_, root) in &interned { - number(root, &mut groups, &mut id_of); - } - for (query, root) in &interned { - let g = id_of[&Rc::as_ptr(root)]; - let mut reached = HashSet::new(); - reach(&groups, g, &mut reached); - for reached in reached { - let queries = &mut groups[reached.0 as usize].queries; - if !queries.contains(query) { - queries.push(*query); - } - } - } - for group in &mut groups { - group.queries.sort_unstable(); - } - // Interning rebuilds every node over its children in order, so the two - // DAGs are walked side by side. - let mut group_of = GroupOf::new(); - for ((_, original), (_, root)) in originals.iter().zip(&interned) { - pair(original, root, &id_of, &mut group_of); - } - (groups, group_of) -} - -/// The operator nodes a query root reads. -fn operator_roots(root: &QueryRoot) -> Vec> { - root.operator_refs().into_iter().cloned().collect() -} - -/// Number `node` (interned) and its descendants, children first. -fn number( - node: &Rc, - groups: &mut Vec, - id_of: &mut HashMap<*const OperatorNode, MemoGroupId>, -) -> MemoGroupId { - if let Some(&id) = id_of.get(&Rc::as_ptr(node)) { - return id; - } - let children: Vec = node - .children() - .into_iter() - .map(|child| number(child, groups, id_of)) - .collect(); - let id = MemoGroupId(groups.len() as u32); - for &child in &children { - let parents = &mut groups[child.0 as usize].parents; - if !parents.contains(&id) { - parents.push(id); - } - } - groups.push(MemoGroup { - logical_node: Rc::clone(node), - child_groups: children, - parents: Vec::new(), - queries: Vec::new(), - candidates: Vec::new(), - }); - id_of.insert(Rc::as_ptr(node), id); - id -} - -/// `g` and every group below it. -fn reach(groups: &[MemoGroup], g: MemoGroupId, reached: &mut HashSet) { - if reached.insert(g) { - for &child in &groups[g.0 as usize].child_groups { - reach(groups, child, reached); - } - } -} - -/// Map `original` and its descendants to the groups of `interned`, its -/// interned copy. -fn pair( - original: &Rc, - interned: &Rc, - id_of: &HashMap<*const OperatorNode, MemoGroupId>, - group_of: &mut GroupOf, -) { - if group_of - .insert(Rc::as_ptr(original), id_of[&Rc::as_ptr(interned)]) - .is_some() - { - return; - } - for (child, interned_child) in original.children().into_iter().zip(interned.children()) { - pair(child, interned_child, id_of, group_of); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::operator::operator_properties::BinaryOpKind; - use asap_types::ir::properties::AccuracyTarget; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::{NonASAPOp, ScalarExpr}; - use asap_types::stages::queries_rooted_at; - - fn query(text: &str) -> QueryRoot { - QueryRoot::Operator(lower_promql(text, AccuracyTarget::max_error(0.01))) - } - - /// Logical interning shares equivalent results without needing Pass 1 choices. - #[test] - fn repeated_results_share_groups_but_quantile_levels_do_not() { - let roots = vec![ - query("sum by (job) (m)"), - query("sum by (job) (m)"), - query("quantile_over_time(0.95, m[5m])"), - query("quantile_over_time(0.99, m[5m])"), - ]; - let memo = memo_of_queries(&roots); - let root_group = |q: usize| *memo.query_roots[q].as_operator().unwrap(); - assert_eq!(root_group(0), root_group(1)); - assert_ne!(root_group(2), root_group(3)); - for (index, group) in memo.groups.iter().enumerate() { - assert!(group - .child_groups - .iter() - .all(|child| (child.0 as usize) < index)); - for child in &group.child_groups { - assert!(memo.groups[child.0 as usize] - .parents - .contains(&MemoGroupId(index as u32))); - } - } - } - - /// A per-series quantile has no provable unique key, but it is - /// deterministic: both operands of `G2 / G2` read the same rows, so the - /// two equal sub-expressions are one group read twice. - #[test] - fn equal_keyless_deterministic_results_share_one_group() { - let roots = vec![QueryRoot::Operator(lower_promql( - "avg by (job) (quantile_over_time(0.5, data[5m]) / quantile_over_time(0.5, data[5m]) + quantile_over_time(0.9, data[5m]))", - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ))]; - let memo = memo_of_queries(&roots); - let division = memo - .groups - .iter() - .find(|group| { - matches!(group.logical_node.non_asap(), Some(NonASAPOp::BinaryOp { operator, .. }) - if operator.kind == BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)) - }) - .expect("a `G2 / G3` group"); - assert_eq!( - division.child_groups[0], division.child_groups[1], - "both operands of the division are one group" - ); - assert_eq!(memo.groups.len(), 7); - } - - /// Scalar references are roots of their query; literals need no invented node. - #[test] - fn scalar_and_literal_roots_keep_workload_positions() { - let QueryRoot::Operator(node) = query("sum by (job) (m)") else { - unreachable!() - }; - let roots = vec![ - QueryRoot::Scalar(ScalarExpr::literal_f64(1.0)), - QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(node.clone())), - QueryRoot::Operator(node.clone()), - ]; - let memo = memo_of_queries(&roots); - let id = *memo.query_roots[2].as_operator().unwrap(); - assert_eq!(queries_rooted_at(&memo.query_roots, id), vec![1, 2]); - assert_eq!(memo.groups[id.0 as usize].reached_by_queries(), [1, 2]); - assert!(memo_of_queries(&roots[..1]).groups.is_empty()); - } +/// being `roots[i]`, every group without candidates. Fails with the first +/// node that breaks the node rules over its children's groups +/// ([`Memo::find_or_add_group`]). +pub fn memo_of_queries(roots: &[QueryRoot]) -> Result { + let _ = roots; + todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group") } diff --git a/crates/logical-asap-optimization/src/stage0/cse.rs b/crates/logical-asap-optimization/src/stage0/cse.rs deleted file mode 100644 index 085b0793c..000000000 --- a/crates/logical-asap-optimization/src/stage0/cse.rs +++ /dev/null @@ -1,886 +0,0 @@ -//! Structural common-subexpression elimination over the unified operator IR: -//! bottom-up hash-consing of [`OperatorNode`] DAGs across a workload's roots. -//! -//! CSE only runs on already-bound, already-canonicalized plans — structural -//! matching is meaningless before canonicalization has converged -//! semantically-equivalent queries onto one shape. [`share_common_sub_dags`] -//! is the single entry point, run once per workload batch (a batch of one -//! still deduplicates a query's own repeated sub-DAGs, see below). -//! -//! ## Algorithm: classic hash-consing / value-numbering -//! -//! Bottom-up: every child is interned before its parent, so two parents whose -//! children were independently deduplicated down to the same `Rc`s are -//! structurally identical iff their own fields also match, without re-walking -//! the sub-DAGs. "Child" means everything [`OperatorNode::children`] returns: -//! the operator inputs *and* the operator nodes a scalar expression reads -//! (`PromqlScalarFromVector`, `ScalarSubquery`, `Exists`, `InSubquery`), so a -//! vector read by `scalar(v)` in two queries is shared like any other input. -//! The scalar expressions themselves stay opaque data on their owning node. -//! -//! ## Correctness: hash is a filter, `PartialEq` is the decision -//! -//! This is the one non-negotiable rule. A **false positive** here — two -//! sub-DAGs wrongly judged shareable — is a wrong query answer, not a missed -//! optimization: two different queries would read each other's data. -//! [`structural_hash`] (SipHash over a canonical serialization, no -//! collision-freedom guarantee) may only narrow the candidate set within one -//! bucket; the typed equality check on that bucket ([`same_node`]) is what -//! actually decides sharing, every time, no exceptions for "the hash probably -//! didn't collide." Equality is intentionally conservative: it recognizes -//! *exact* structural matches only, never "a stricter-accuracy summary could -//! also answer a looser request" (that subsumption question belongs to the -//! ASAP matcher, not here). -//! -//! ## Legality -//! -//! Structural equality is necessary but not sufficient. A non-ASAP node is -//! only ever *returned* as a match for another when every execution over the -//! same input rows returns the same output rows -//! ([`NonASAPOp::is_shareable_when_equal`]). Equal nodes are -//! interned children first, so two equal nodes read one input, and one copy -//! returns each consumer the rows it would have computed alone. A unique key -//! is not needed: equal per-series quantiles without a provable key are one -//! node. An operator that picks rows by input order alone (an unordered -//! `LIMIT`) is always inserted fresh, because two copies may return -//! different rows. -//! An ASAP node (summary state and its evaluations) has no such gate: equal -//! operator, schema and guarantee make it shareable, exactly as post-ASAP -//! sharing decided before this IR. -//! -//! ## Single-query CSE falls out for free -//! -//! A repeated sub-expression within *one* query (the same grouped aggregate on -//! both `BinaryOp` branches) is deduplicated by the same bottom-up interning — -//! a workload of size one still interns bottom-up within that one DAG. - -use std::collections::hash_map::DefaultHasher; -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use asap_types::ir::operator::node::{Operator, OperatorNode}; -use asap_types::ir::NonASAPOp; - -/// [`structural_hash`]'s memoization cache: an already-hashed node's `Rc` -/// pointer to its hash. A fresh cache is always correct; what matters is -/// letting it persist across every node of one bottom-up pass rather than -/// starting a new one per call. The caller must keep every cached node alive -/// for the cache's lifetime, or a reused address would alias a stale entry. -pub(crate) type NodeToStructuralHashCache = HashMap<*const OperatorNode, u64>; - -/// The operator with every child (operator inputs and the operator nodes -/// referenced from its scalar expressions alike) replaced by `()`. What -/// remains is the node's own data: variant tag, scalar expressions, -/// parameters. -fn own_fields(node: &OperatorNode) -> Operator<()> { - node.operator.map_children(|_| ()) -} - -/// Coarse structural hash used only to bucket [`InternTable::intern`]'s -/// candidate search — never the sharing decision ([`same_node`] is). -/// -/// `OperatorNode` carries `f64`s (`ScalarValue::Float64`, quantile targets, -/// `AccuracyGuarantee` bounds, …), so it cannot derive `std::hash::Hash`. The -/// hash is SipHash over two parts: -/// -/// 1. the canonical JSON of [`own_fields`] plus `result_kind`, `schema`, -/// `guarantee` — every field `PartialEq` compares except the -/// children. A scalar expression is serialized as data with each operator -/// node it reads replaced by `()`, so a reference to an -/// interned sub-DAG contributes nothing of its own here; -/// 2. for every child in [`OperatorNode::children`] order (operator inputs, -/// then scalar-referenced nodes), the child's own `structural_hash`, -/// memoized in `cache` by `Rc` pointer identity. -/// -/// Part 2 is what makes equal sub-DAGs hash equal whether they are reached -/// through an operator input or through a `scalar(v)`, and what keeps the -/// pass linear: a node is generally a DAG, and re-serializing a shared -/// descendant once per parent would cost `O(sub-DAG)` per node instead of -/// `O(1)` beyond the children's already-known hashes. A non-finite `f64` -/// serializes as `null`, merely widening one (still equality-checked) bucket. -pub(crate) fn structural_hash(node: &OperatorNode, cache: &mut NodeToStructuralHashCache) -> u64 { - fn child_hash(child: &Rc, cache: &mut NodeToStructuralHashCache) -> u64 { - let ptr = Rc::as_ptr(child); - if let Some(&h) = cache.get(&ptr) { - return h; - } - let h = structural_hash(child, cache); - cache.insert(ptr, h); - h - } - - let mut hasher = DefaultHasher::new(); - let own = ( - own_fields(node), - node.result_kind, - &node.schema, - &node.guarantee, - ); - serde_json::to_string(&own) - .unwrap_or_default() - .hash(&mut hasher); - for child in node.children() { - child_hash(child, cache).hash(&mut hasher); - } - hasher.finish() -} - -/// Numeric `PartialEq` alone conflates signed zeros. The serialized check is -/// additional evidence, never a replacement for typed equality (JSON maps -/// non-finite floats to `null`). Used for the guarantee, whose bounds are -/// floats a shared node must preserve bit-for-bit, and for summary families, -/// whose `SketchParams` equality omits the recorded error and probability. -pub(crate) fn same_value(left: &T, right: &T) -> bool { - left == right - && match (serde_json::to_string(left), serde_json::to_string(right)) { - (Ok(left), Ok(right)) => left == right, - _ => false, - } -} - -/// Cache of child-pair comparisons already decided by [`same_node`], keyed by -/// pointer pair. Only interned (table-owned, hence alive) nodes are keys. -type NodePairToIsSameNodeCache = HashMap<(*const OperatorNode, *const OperatorNode), bool>; - -/// The sharing decision: typed equality of two nodes. -/// -/// `OperatorNode`'s derived `PartialEq` would recurse into children by value -/// even when both sides hold the same `Rc` (`OperatorNode` is not `Eq`, so -/// `Rc` gets no pointer shortcut), expanding a shared diamond once per path. -/// Children are therefore compared by pointer first; only when the pointers -/// differ (an equal child that was not legal to share) are the values -/// compared, memoized per pair so a diamond is still walked once. -fn same_node( - left: &OperatorNode, - right: &OperatorNode, - cache: &mut NodePairToIsSameNodeCache, -) -> bool { - let (lc, rc) = (left.children(), right.children()); - if lc.len() != rc.len() { - return false; - } - let children_equal = lc.iter().zip(&rc).all(|(a, b)| { - if Rc::ptr_eq(a, b) { - return true; - } - let key = (Rc::as_ptr(a), Rc::as_ptr(b)); - if let Some(&eq) = cache.get(&key) { - return eq; - } - let eq = same_node(a, b, cache); - cache.insert(key, eq); - eq - }); - children_equal - && left.result_kind == right.result_kind - && left.schema == right.schema - && same_value(&left.guarantee, &right.guarantee) - && same_value(&own_fields(left), &own_fields(right)) -} - -/// Bottom-up hash-consing table: structurally-equal, sharing-legal nodes -/// collapse onto one `Rc`. -/// -/// `buckets` is keyed by [`structural_hash`] — a coarse candidate filter -/// only. Every entry within one bucket is a full node kept around for the -/// [`same_node`] comparison that actually decides a match; a hash collision -/// between structurally different nodes just means a harmless linear scan of -/// a few extra candidates. -struct InternTable { - buckets: HashMap>>, - /// Persisted for the table's whole lifetime so hashing is `O(1)` per node - /// beyond its children; every cached node is owned by `buckets`. - structural_hash_cache: NodeToStructuralHashCache, - same_node_cache: NodePairToIsSameNodeCache, -} - -impl InternTable { - fn new() -> Self { - Self { - buckets: HashMap::new(), - structural_hash_cache: HashMap::new(), - same_node_cache: HashMap::new(), - } - } - - /// Intern one node whose children are already interned: look it up by - /// [`structural_hash`], confirm with [`same_node`], and — only when - /// sharing is legal (module doc, "Legality") — return the existing `Rc` - /// instead of allocating a new one. - fn intern(&mut self, node: OperatorNode) -> Rc { - let hash = structural_hash(&node, &mut self.structural_hash_cache); - // A node that is not legal to share is never *returned* as a match - // for something else; it still occupies a fresh slot in the bucket - // (harmless: later scans require legality of the new node too). - let reusable = node.is_asap() - || node - .non_asap() - .is_some_and(NonASAPOp::is_shareable_when_equal); - let bucket = self.buckets.entry(hash).or_default(); - if reusable { - if let Some(existing) = bucket - .iter() - .find(|candidate| same_node(candidate, &node, &mut self.same_node_cache)) - { - return Rc::clone(existing); - } - } - let rc = Rc::new(node); - bucket.push(Rc::clone(&rc)); - rc - } -} - -/// Input pointer → (input `Rc`, interned result). The input `Rc` is retained -/// so its address cannot be freed and reused by a fresh allocation while the -/// cache still maps it. -type NodeToInternedNodeCache = HashMap<*const OperatorNode, (Rc, Rc)>; - -/// Intern `node`'s children (recursively), then `node` itself. The rebuilt -/// node keeps `node`'s retained schema, result kind and guarantee: -/// every child is replaced by an equal node, so each derived property stays -/// valid, and the result is `PartialEq`-equal to the input. -fn intern_bottom_up( - table: &mut InternTable, - visited: &mut NodeToInternedNodeCache, - node: &Rc, -) -> Rc { - if let Some((_, interned)) = visited.get(&Rc::as_ptr(node)) { - return Rc::clone(interned); - } - let operator = node - .operator - .map_children(|child| intern_bottom_up(table, visited, child)); - let rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()); - let interned = table.intern(rebuilt); - visited.insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&interned))); - interned -} - -/// Share structurally-identical, sharing-legal sub-DAGs across a workload's -/// roots (or within one root). Every root's *value* is unchanged -/// (`PartialEq`-equal to its input) — only its internal `Rc` structure may -/// now alias another root's, or another part of its own DAG. A node already -/// reached through two paths is visited once. -/// -/// `Id` is caller-chosen — a workload entry's key, an index, a query name. -pub fn share_common_sub_dags( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Rc)> { - let mut table = InternTable::new(); - let mut visited = NodeToInternedNodeCache::new(); - roots - .into_iter() - .map(|(id, root)| (id, intern_bottom_up(&mut table, &mut visited, &root))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::AggIntent; - use asap_types::ir::operator::asap::ASAPOp; - use asap_types::ir::operator::operator_properties::{ - BinaryOpKind, GroupKeys, Reduction, Source, - }; - use asap_types::ir::properties::guarantee::AccuracyGuarantee; - use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; - use asap_types::ir::schema::sketch::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; - use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; - use asap_types::ir::ScalarExpr; - use asap_types::ir::{BinaryOperator, NonASAPOp}; - - use asap_types::ir::properties::AccuracyTarget; - - fn node(op: NonASAPOp) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(op)).unwrap() - } - - /// `[ts, service, value, latency]`, no unique key. - fn scan() -> Rc { - node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - }) - } - - fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Quantile { - col, - q, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - } - - fn compare(lhs: Rc, rhs: Rc) -> Rc { - node(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Compare(CompareOpKind::Eq), - vector_match: None, - }, - return_bool: false, - lhs, - rhs, - }) - } - - fn two_roots(a: Rc, b: Rc) -> (Rc, Rc) { - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - (Rc::clone(ra), Rc::clone(rb)) - } - - #[test] - fn distinct_column_quantiles_do_not_merge() { - // Grouped (unique key present) so only the differing `col` blocks it. - let (ra, rb) = two_roots( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(3), 0.5), - ); - assert!(!Rc::ptr_eq(&ra, &rb)); - assert_ne!(ra, rb); - } - - // Two aggregates that differ only in one measure's `FILTER` predicate - // compute different values, so structural sharing keeps them apart. - #[test] - fn filtered_and_unfiltered_aggregates_do_not_merge() { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - let unfiltered = quantile_agg(vec![1], Some(2), 0.5); - let filtered = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.5, - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![Some(Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(3)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: asap_types::ir::ExprSemantics::Sql, - }))], - having: None, - child: scan(), - }); - let (ra, rb) = two_roots(unfiltered, filtered); - assert!( - !Rc::ptr_eq(&ra, &rb), - "a filtered measure must not be shared" - ); - assert_ne!(ra, rb); - } - - #[test] - fn identical_global_scalar_aggregates_share_their_single_result() { - let a = quantile_agg(vec![], Some(2), 0.9); - let b = quantile_agg(vec![], Some(2), 0.9); - assert_eq!(a, b, "fixture sanity: structurally equal"); - assert!( - !a.schema.has_unique_key(), - "fixture sanity: a global aggregate has no provable unique key" - ); - let (ra, rb) = two_roots(a, b); - assert!( - Rc::ptr_eq(&ra, &rb), - "a global scalar aggregate has one stable result per evaluation" - ); - } - - /// A global scalar result can be shared only with the same computation. - #[test] - fn distinct_global_quantiles_do_not_merge() { - let (a, b) = two_roots( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(2), 0.9), - ); - assert!(!Rc::ptr_eq(&a, &b)); - let (a, b) = two_roots( - quantile_agg(vec![], Some(2), 0.5), - quantile_agg(vec![], Some(3), 0.5), - ); - assert!(!Rc::ptr_eq(&a, &b)); - } - - /// Two equal scans of a keyless table (SQL rows) are one node: each - /// consumer reads every row the scan outputs. - #[test] - fn equal_scans_without_a_unique_key_are_shared() { - let table_scan = || { - node(NonASAPOp::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("city", DataType::Utf8, true), - Field::plain("value", DataType::Float64, true), - ], - 0, - vec![], - ), - }) - }; - let a = table_scan(); - assert!(!a.schema.has_unique_key(), "fixture sanity: keyless rows"); - let (ra, rb) = two_roots(a, table_scan()); - assert!(Rc::ptr_eq(&ra, &rb)); - } - - /// Grouped quantiles differing only in accuracy keep two roots over one shared scan. - #[test] - fn differing_accuracy_is_never_merged_but_the_scan_child_is_shared() { - let shared_scan = scan(); - let quantile = |epsilon: f64| { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Quantile { - col: Some(2), - q: 0.99, - accuracy: AccuracyTarget::max_error(epsilon), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(&shared_scan), - }) - }; - let a = quantile(0.01); - assert!(a.schema.has_unique_key(), "fixture sanity: grouped"); - let (ra, rb) = two_roots(a, quantile(0.05)); - assert!(!Rc::ptr_eq(&ra, &rb)); - assert_ne!(ra, rb); - let ( - Some(NonASAPOp::Aggregate { child: child_a, .. }), - Some(NonASAPOp::Aggregate { child: child_b, .. }), - ) = (ra.non_asap(), rb.non_asap()) - else { - panic!("expected Aggregate roots"); - }; - assert!(Rc::ptr_eq(child_a, child_b)); - } - - #[test] - fn median_and_explicit_half_percentile_merge() { - // Two spellings that lower to the identical grouped `Quantile { q: 0.5 }`. - let (m, p) = two_roots( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - assert!(Rc::ptr_eq(&m, &p)); - } - - #[test] - fn single_query_shares_its_own_repeated_sub_dag() { - // One root with the same grouped aggregate on both branches, built as - // two separately-allocated sub-DAGs (no sharing yet). - let root = compare( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = root.non_asap() else { - panic!("expected BinaryOp root, got {root:?}"); - }; - assert!(Rc::ptr_eq(lhs, rhs)); - } - - #[test] - fn shared_root_value_is_unchanged() { - let a = quantile_agg(vec![1], Some(2), 0.5) - .as_ref() - .clone() - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); - let before = Rc::new(a); - let (ra, _) = two_roots(Rc::clone(&before), Rc::clone(&before)); - assert_eq!(ra.as_ref(), before.as_ref()); - assert!( - ra.guarantee.is_some(), - "retained properties survive the rebuild" - ); - } - - // ── scalar-referenced sub-DAGs ────────────────────────────────────── - - /// `vector(scalar(sum by (service) (up)))`. - fn scalar_of_vector() -> Rc { - let sum_up = node(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: Some(2) }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }); - assert!(sum_up.schema.has_unique_key(), "fixture sanity"); - node(NonASAPOp::PromqlVectorFromScalar( - ScalarExpr::PromqlScalarFromVector(sum_up), - )) - } - - fn bridged_vector(root: &Rc) -> &Rc { - match root.non_asap() { - Some(NonASAPOp::PromqlVectorFromScalar(ScalarExpr::PromqlScalarFromVector(v))) => v, - other => panic!("expected vector(scalar(v)), got {other:?}"), - } - } - - #[test] - fn scalar_referenced_vector_is_shared_across_queries() { - let (ra, rb) = two_roots(scalar_of_vector(), scalar_of_vector()); - assert!( - Rc::ptr_eq(bridged_vector(&ra), bridged_vector(&rb)), - "the vector read by scalar(v) is a child and must be interned" - ); - assert!( - Rc::ptr_eq(&ra, &rb), - "the scalar bridge has no unique key but returns the same rows" - ); - } - - #[test] - fn structural_hash_sees_through_a_scalar_reference() { - // Two equal bridges must hash equal whether or not their referenced - // vector is the same Rc — the reference contributes the vector's - // memoized hash, not its identity. - let a = scalar_of_vector(); - let b = scalar_of_vector(); - let mut cache = HashMap::new(); - assert_eq!( - structural_hash(&a, &mut cache), - structural_hash(&b, &mut cache) - ); - assert_eq!( - cache.len(), - 4, - "aggregate + scan cached once per root: {cache:?}" - ); - let other = node(NonASAPOp::PromqlVectorFromScalar( - ScalarExpr::PromqlScalarFromVector(quantile_agg(vec![1], Some(2), 0.5)), - )); - assert_ne!( - structural_hash(&a, &mut cache), - structural_hash(&other, &mut cache) - ); - } - - // ── ASAP nodes ────────────────────────────────────────────────────── - - fn summary_agg(_alpha: f64, guarantee: Option) -> Rc { - let family = FieldDataType::Sketch(SketchAlgorithm::DDSketch, GroupingStrategy::default()); - let schema = Schema::lifted(vec![Field::new("state", family.clone(), false)], None); - assert!(!schema.has_unique_key(), "fixture sanity"); - Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryAgg { - child: scan(), - family, - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }), - schema, - ) - .with_guarantee(guarantee), - ) - } - - #[test] - fn asap_nodes_share_without_a_unique_key() { - let exact = || Some(AccuracyGuarantee::exact("fixture")); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, exact())); - assert!(Rc::ptr_eq(&ra, &rb)); - assert!(ra.guarantee.is_some()); - } - - /// θ is not part of a node's identity (#1207): two sizes of one family - /// over one input are one group. A different guarantee still keeps them - /// apart. - #[test] - fn asap_nodes_with_distinct_parameters_are_shared_but_distinct_guarantees_are_not() { - let exact = || Some(AccuracyGuarantee::exact("fixture")); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.001, exact())); - assert!(Rc::ptr_eq(&ra, &rb), "θ is not part of the group identity"); - let (ra, rb) = two_roots(summary_agg(0.01, exact()), summary_agg(0.01, None)); - assert!( - !Rc::ptr_eq(&ra, &rb), - "an unknown guarantee never borrows an exact one" - ); - assert!(rb.guarantee.is_none()); - } - - #[test] - fn evaluations_share_their_producer_but_not_each_other() { - use asap_types::ir::schema::sketch::SketchStatistic; - let evaluation = |q: f64| { - Rc::new(OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: summary_agg(0.01, None), - query: SketchStatistic::Quantile { q }, - }), - Schema::lifted( - vec![Field::plain("quantile", DataType::Float64, false)], - None, - ), - )) - }; - let (p95, p99) = two_roots(evaluation(0.95), evaluation(0.99)); - let producer = |n: &Rc| Rc::clone(n.children()[0]); - assert!(!Rc::ptr_eq(&p95, &p99)); - assert!(Rc::ptr_eq(&producer(&p95), &producer(&p99))); - } - - // ── structural_hash (DAG-aware memoization) ───────────────────────── - - #[test] - fn structural_hash_is_stable_across_cache_states() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let mut cold = HashMap::new(); - let mut warm = HashMap::new(); - structural_hash(&scan(), &mut warm); - assert_eq!( - structural_hash(&agg, &mut cold), - structural_hash(&agg, &mut warm), - "hash must be independent of unrelated cache state" - ); - } - - #[test] - fn structural_hash_of_an_internally_shared_dag_matches_the_unshared_equivalent() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared_root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let unshared_root = compare( - quantile_agg(vec![1], Some(2), 0.5), - quantile_agg(vec![1], Some(2), 0.5), - ); - assert_eq!( - structural_hash(&shared_root, &mut HashMap::new()), - structural_hash(&unshared_root, &mut HashMap::new()), - ); - } - - #[test] - fn structural_hash_memoizes_a_shared_descendant_exactly_once() { - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let mut cache = HashMap::new(); - structural_hash(&root, &mut cache); - assert_eq!( - cache.len(), - 2, - "one entry per unique node in the shared branch (Aggregate + Scan): {cache:?}" - ); - } - - /// `DISTINCT` (every column) returns the same rows each time; a `Dedup` - /// on some columns keeps an unspecified row per key. - #[test] - fn dedup_shares_only_on_every_column() { - let dedup = |cols: Vec| { - node(NonASAPOp::Dedup { - cols, - child: scan(), - }) - }; - let (ra, rb) = two_roots(dedup(vec![]), dedup(vec![])); - assert!(Rc::ptr_eq(&ra, &rb)); - let (ra, rb) = two_roots(dedup(vec![1]), dedup(vec![1])); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// `LIMIT 10` without an order may return any 10 rows, so two copies may - /// differ; one shared copy would make both consumers see the same rows. - #[test] - fn unordered_limits_are_not_shared() { - let limit = || { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: scan(), - }) - }; - let (ra, rb) = two_roots(limit(), limit()); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows - /// of a fixed order; over a `Sort` partitioned differently, which rows - /// come first still depends on input order. - #[test] - fn limits_over_a_sort_with_their_partitions_are_shared() { - let limit = |sort_partitions: GroupKeys| { - node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: node(NonASAPOp::Sort { - keys: vec![asap_types::ir::scalar::SortKey { - expr: ScalarExpr::column(2), - ascending: false, - nulls_first: false, - }], - partition_by: sort_partitions, - child: scan(), - }), - }) - }; - let (ra, rb) = two_roots(limit(GroupKeys::default()), limit(GroupKeys::default())); - assert!(Rc::ptr_eq(&ra, &rb)); - let (ra, rb) = two_roots(limit(GroupKeys::by(vec![1])), limit(GroupKeys::by(vec![1]))); - assert!(!Rc::ptr_eq(&ra, &rb)); - } - - /// `count without (service)` has no provable unique key, but it computes - /// the same rows from the same input, so equal copies are one node. - #[test] - fn without_grouping_is_shared() { - let without_agg = || { - node(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: scan(), - }) - }; - let a = without_agg(); - assert!(!a.schema.has_unique_key()); - let (ra, rb) = two_roots(a, without_agg()); - assert!(Rc::ptr_eq(&ra, &rb)); - } - - #[test] - fn already_shared_nodes_are_visited_once() { - // A diamond already present in the input stays one node and is not - // re-interned per path. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = compare(Rc::clone(&agg), Rc::clone(&agg)); - let shared = share_common_sub_dags(vec![("q", root)]); - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = shared[0].1.non_asap() else { - panic!("expected BinaryOp root"); - }; - assert!(Rc::ptr_eq(lhs, rhs)); - assert_eq!(OperatorNode::reachable(&shared[0].1).len(), 3); - } - - // Comparing a shareable node whose equal-but-unshareable children form a - // deep diamond must not expand the diamond once per path. The timeout is - // a coarse runaway guard, not a performance SLA. - #[test] - fn shared_diamond_does_not_expand_during_comparison() { - let (done, completion) = std::sync::mpsc::channel(); - let worker = std::thread::spawn(move || { - fn keyed_diamond() -> Rc { - // An unordered `Limit` at each of the 24 levels is never - // shared, so comparing the `Dedup` on top walks the diamond. - let mut current = scan(); - for _ in 0..24 { - let limited = node(NonASAPOp::Limit { - n: Some(10), - offset: 0, - partition_by: GroupKeys::default(), - child: current, - }); - current = compare(Rc::clone(&limited), limited); - } - node(NonASAPOp::Dedup { - cols: vec![], - child: current, - }) - } - let (ra, rb) = two_roots(keyed_diamond(), keyed_diamond()); - assert!(Rc::ptr_eq(&ra, &rb)); - done.send(()).unwrap(); - }); - completion - .recv_timeout(std::time::Duration::from_secs(5)) - .expect("comparison expanded the shared DAG"); - worker.join().unwrap(); - } - - /// A keyed (hence shareable) projection emitting the literal `value`. - fn keyed_literal(value: f64) -> Rc { - let keyed = node(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - ], - 0, - vec![vec![1]], - ), - }); - node(NonASAPOp::Project { - cols: vec![ - asap_types::ir::ProjectItem { - alias: None, - expr: ScalarExpr::Column(1), - }, - asap_types::ir::ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::literal_f64(value), - }, - ], - qualifier: None, - child: keyed, - }) - } - - /// Sharing preserves IEEE signed zero, and JSON's `null` encoding of - /// non-finite floats never becomes the equality decision. - #[test] - fn signed_zero_and_nonfinite_values_remain_distinct() { - assert!( - keyed_literal(0.0).schema.has_unique_key(), - "fixture is shareable" - ); - for (a, b) in [ - (0.0, -0.0), - (-0.0, 0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - let (ra, rb) = two_roots(keyed_literal(a), keyed_literal(b)); - assert!(!Rc::ptr_eq(&ra, &rb), "{a} and {b} must not be shared"); - } - let (ra, rb) = two_roots(keyed_literal(f64::INFINITY), keyed_literal(f64::INFINITY)); - assert!(Rc::ptr_eq(&ra, &rb)); - } -} diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index 37c029953..7eceb36ee 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -5,11 +5,10 @@ use std::rc::Rc; // Stage 1 internals other crates' tests call: Pass 1's working form, -// single-candidate composition and Stage 0 hash-consing. +// and single-candidate composition. pub use crate::pass1::per_computation_candidates::{ compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, }; -pub use crate::stage0::cse::share_common_sub_dags; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::OperatorNode; @@ -183,9 +182,9 @@ pub fn cms_summary(query: &str) -> OperatorNode { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); let root = lower_promql(query, target); // Inspect each local computation choice under the built-in deployment. - let per_computation = generate_per_computation_candidates(&crate::stage0::memo_of_queries(&[ - QueryRoot::Operator(root), - ])) + let per_computation = generate_per_computation_candidates( + &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + ) .unwrap() .without_unsupported_summaries(None); let summary = per_computation diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 143db8a7b..269ba9405 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -118,7 +118,7 @@ pub fn run_planning( root.validate_structure()?; } // Stage 0: the memo of logical groups. - let initial = memo_of_queries(&roots); + let initial = memo_of_queries(&roots)?; // Pass 1: the per-computation candidates the deployment can run and // some query can use. let accuracy_targets: Vec<_> = entries @@ -172,10 +172,10 @@ pub fn run_planning( let logical = logical_asap_candidates.insert( logical.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?, ); - report.memo_groups = logical.memo.groups.len(); + report.memo_groups = logical.memo.groups().len(); report.logical_candidates = logical .memo - .groups + .groups() .iter() .map(|group| group.candidates.len()) .sum(); diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index d63742553..61bc53e69 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_logical_asap_optimization`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 9cb82ad91..540d02d2c 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -24,12 +24,27 @@ pub struct CandidateForMemoGroupId { /// query's root over them. #[derive(Debug, Clone, Default)] pub struct Memo { - pub groups: Vec, + /// The groups, by id ([`Self::groups`]); added only through + /// [`Self::find_or_add_group`]. + groups: Vec, /// Each query's root over groups, by index in the workload. pub query_roots: Vec>, } impl Memo { + /// The groups, by id. + pub fn groups(&self) -> &[MemoGroup] { + &self.groups + } + + /// Adds a query whose root is `root` and returns its index: the root's + /// groups and every group below them list it + /// ([`MemoGroup::reached_by_queries`]). Stage 0 adds each query once. + pub fn add_query(&mut self, root: QueryRoot) -> usize { + let _ = root; + todo!("#1207 PR 2") + } + /// Hash-consing: the group with this expression, added if new. Fails /// when the expression breaks the node rules (schema derivation, input /// validation, merge check). From f104f6f989339e6904853b0f8b2f09ca079f158a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:28:29 +0000 Subject: [PATCH 11/30] refactor(types): a memo group is its expression and GroupProperties; OperatorInput; coverage only in the memo (#1207 PR 0) MemoGroup { expression, group_properties: GroupProperties, parents, queries, candidates } replaces logical_node and child_groups; GroupProperties { schema, result_kind, summary_coverage, shareable_when_equal } is derived once by derive_group_properties, which applies the existing rules to the expression over MemoGroupAsInputs. The rules (output_schema, output_kind, validate_inputs, is_shareable_when_equal, summary builds, window chains, scalar subqueries, top-k, populations) are generic over OperatorInput, implemented for Rc and MemoGroupAsInput; only how they read an input changed. BuiltOrInput carries the builds a rule makes while deriving (one build over a chain's window, the build a rollup or a selection stands for). SummaryCoverage { definition: SummaryStateDefinition { input_group, unlifted_operators }, selection } is derived only from memo groups (derive_from_inputs) and stored in GroupProperties; node-tree coverage, NodeToSummaryCoverageCache, derive_cached and OperatorNode::coverage are deleted, so node trees have no merge check. Memo::logical_node_trees is deleted: Pass 1 target discovery (PR 3), devtools group labels (PR 9) and the export's view coverage (PR 6) are todo!(). find_or_add_group and add_candidate derive the properties, then todo!() until PR 2. --- crates/accuracy/src/propagation.rs | 19 +- crates/devtools/src/bin/asap_planner.rs | 120 +- crates/devtools/src/planner_fuzz/relations.rs | 6 +- crates/devtools/src/planning_stages_text.rs | 24 +- crates/executor/src/physical_planner/mod.rs | 8 +- .../frontend-sql/src/sql/collection_udfs.rs | 2 +- crates/integration-tests/src/lib.rs | 5 +- .../tests/frontend_timestamps.rs | 2 +- crates/integration-tests/tests/main.rs | 1 - .../integration-tests/tests/sketch_rollup.rs | 9 +- .../tests/summary_coverage_e2e.rs | 262 ---- .../src/pass1/per_computation_candidates.rs | 16 +- .../pass1/per_computation_candidates_tests.rs | 18 +- .../src/pass1/realization.rs | 4 +- crates/types/src/ir/materialized_view.rs | 74 +- crates/types/src/ir/operator/asap.rs | 425 ++++--- crates/types/src/ir/operator/input.rs | 137 +++ .../src/ir/operator/maintained_population.rs | 31 +- crates/types/src/ir/operator/mod.rs | 3 + crates/types/src/ir/operator/node.rs | 67 +- crates/types/src/ir/operator/non_asap.rs | 162 +-- .../src/ir/operator/tumbling_window_chain.rs | 207 ++-- .../src/ir/properties/summary_coverage.rs | 642 +++++----- crates/types/src/ir/scalar/mod.rs | 28 +- crates/types/src/ir/schema/topk.rs | 48 +- crates/types/src/stages/candidates.rs | 48 +- crates/types/src/stages/memo.rs | 83 +- crates/types/src/stages/mod.rs | 3 +- crates/types/src/test_support.rs | 15 +- crates/types/src/workload/bounds.rs | 14 +- crates/types/tests/main.rs | 1 - crates/types/tests/structure_contract.rs | 4 +- crates/types/tests/summary_coverage.rs | 1049 ----------------- crates/types/tests/summary_merge_structure.rs | 25 - crates/types/tests/tumbling_window_chain.rs | 74 +- 35 files changed, 1179 insertions(+), 2457 deletions(-) delete mode 100644 crates/integration-tests/tests/summary_coverage_e2e.rs create mode 100644 crates/types/src/ir/operator/input.rs delete mode 100644 crates/types/tests/summary_coverage.rs diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index 2d506c88e..7e55e8be4 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::{merged_chain, merged_window_build}; +use asap_types::ir::operator::tumbling_window_chain::{merged_chain, merged_window_build_node}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::guarantee::{ AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, @@ -497,7 +497,8 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> return Some(None); }; // Kept whole, a selected state keeps the guarantee it was built with. - let input = asap_types::ir::operator::asap::selected_from(input).unwrap_or(input); + let input = + asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); let family = state.asap()?.produced_state()?; let merged = match node.asap()? { ASAPOp::SummaryEstimate { query, .. } => FamilyCapability::of(family).merged_error(query), @@ -513,7 +514,7 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> let consumer = OperatorNode::with_schema( node.operator.map_children(|child| { if Rc::ptr_eq(child, state) { - Rc::clone(input) + Rc::clone(&input) } else { Rc::clone(child) } @@ -525,7 +526,7 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> /// The guarantee of an estimate or finalized value read from merged /// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range ([`merged_window_build`]: the whole +/// build over the merged tumbling windows' time range ([`merged_window_build_node`]: the whole /// chain's, or its newest tumbling windows') when the /// family's merged error for that read is exact or the same as that build's /// (the family-capability table); none when it is approximate (evidence @@ -537,14 +538,16 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option { - let family = merged_chain(summary_input)?.asap()?.produced_state()?; + let chain = merged_chain(summary_input)?; + let family = chain.asap()?.produced_state()?; ( summary_input, FamilyCapability::of(family).merged_error(query), ) } ASAPOp::FinalizeExactAccumulator { child } => { - let family = merged_chain(child)?.asap()?.produced_state()?; + let chain = merged_chain(child)?; + let family = chain.asap()?.produced_state()?; // An exact accumulator's merged state is exact. let merged = FamilyCapability::of(family).merge_inputs; (child, merged.map(|_| MergedError::Exact)) @@ -558,7 +561,7 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option Option match merged_chain(summary_input) - .unwrap_or(summary_input) + .unwrap_or_else(|| Rc::clone(summary_input)) .asap()? { ASAPOp::SummaryAgg { diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index b49689ec5..2e9d883b5 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -97,8 +97,7 @@ // A section of a stage after `--run-until` is null. use std::collections::btree_map::Entry; -use std::collections::{BTreeSet, HashMap}; -use std::rc::Rc; +use std::collections::BTreeSet; use asap_cost::{CostInputs, ResourceUsage}; use asap_devtools::candidate_labels::node_label; @@ -112,12 +111,11 @@ use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; use asap_logical_asap_optimization::DeploymentCapabilities; use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; -use asap_types::ir::flat::{flatten, FlatDag, FlatNode, NodeId}; -use asap_types::ir::operator::node::OperatorNode; +use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::properties::{default_metric_for_root, AccuracyTarget}; use asap_types::ir::QueryRoot; use asap_types::stages::PhysicalASAPMemoCandidates; -use asap_types::stages::{queries_rooted_at, MemoGroupId}; +use asap_types::stages::{queries_rooted_at, MemoGroup, MemoGroupId}; use asap_types::stages::{Memo, OriginOfCandidateForMemoGroup}; use asap_types::workload::{ DataArrival, DataWorkload, PlanningWorkload, QueryLanguage, QueryRecurrence, @@ -827,44 +825,10 @@ fn with_default_metrics(mut queries: Vec, roots: &[QueryRoot]) -> Vec, - index_of: HashMap, -} - -impl MemoNodes { - /// Adds `roots` and every node below them. Returns each root's index - /// and, for each node of that DAG, the original node and its index. - fn add( - &mut self, - roots: &[Rc], - ) -> (Vec, Vec<(Rc, NodeId)>) { - let roots: Vec<_> = roots.iter().cloned().map(QueryRoot::Operator).collect(); - let (dag, originals) = flatten(&roots); - let mut index_in_table: Vec = Vec::with_capacity(dag.nodes.len()); - for mut node in dag.nodes { - node.operator = node.operator.map_children(|&child| index_in_table[child]); - let key = serde_json::to_string(&node).expect("a flat node serializes"); - let index = *self.index_of.entry(key).or_insert_with(|| { - self.nodes.push(node); - self.nodes.len() - 1 - }); - index_in_table.push(index); - } - let roots = dag - .roots - .iter() - .map(|root| match root { - QueryRoot::Operator(id) => index_in_table[*id], - QueryRoot::Scalar(_) => unreachable!("memo DAGs have operator roots"), - }) - .collect(); - (roots, originals.into_iter().zip(index_in_table).collect()) - } } /// `memo`'s groups, each with its candidates, their nodes added to @@ -877,7 +841,7 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .enumerate() .map(|(id, group)| { let candidates = &group.candidates; - let node = table.add(std::slice::from_ref(&group.logical_node)).0[0]; + let node = group_node(table, group); let mut by_origin = serde_json::Map::new(); for candidate in candidates { let count = by_origin @@ -896,7 +860,7 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) json!({ "id": id, "label": node_label(&table.nodes[node]), - "children": ids(&group.child_groups), + "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), "roots_of": queries_rooted_at(&memo.query_roots, MemoGroupId(id as u32)), @@ -910,6 +874,12 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) .collect() } +/// The node of `group`'s own operator, added to `table`. +fn group_node(table: &mut MemoNodes, group: &MemoGroup) -> usize { + let _ = (table, group); + todo!("#1207 PR 9: a group's node from its expression") +} + /// `OriginOfCandidateForMemoGroup` in snake case. fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { @@ -922,69 +892,3 @@ fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { } .into() } - -#[cfg(test)] -mod tests { - use super::*; - - /// A node two DAGs share is stored once in the memo's node table, even - /// when each DAG built it as its own `Rc`, and each DAG rebuilt from the - /// table equals its own `FlatDag` export. - #[test] - fn memo_node_table_stores_a_shared_node_once() { - let root = |query: &str| { - let workload: PlanningWorkload = serde_json::from_value(json!({ - "query_workload": {"language": "promql", "query_batch": [{ - "query": query, "invocations": 1, - "requirements": {"accuracy": {"max_error": 0.0, "failure_probability": 0.0}}, - "predictability": {"predictable": {"known_at": null}}, - }]}, - "data_workload": { - "arrival": {"ingesting": {"rows_per_second": 1.0}}, - "sources": {"m": {"series_count": 10, "sample_interval": 15000, "metric_type": "gauge"}}, - }, - })) - .unwrap(); - match lower_promql_roots(&workload).unwrap().remove(0) { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => unreachable!(), - } - }; - let (sum, max) = (root("sum(m)"), root("max(m)")); - let mut table = MemoNodes::default(); - let (sum_root, _) = table.add(std::slice::from_ref(&sum)); - let after_sum = table.nodes.len(); - let (max_root, _) = table.add(std::slice::from_ref(&max)); - // `max(m)` adds only its own aggregate: the scan is already there. - assert_eq!(table.nodes.len(), after_sum + 1); - // The DAG below `root`, renumbered in table order. - let rebuilt = |root: NodeId| { - let mut reached = BTreeSet::new(); - let mut pending = vec![root]; - while let Some(id) = pending.pop() { - if reached.insert(id) { - pending.extend(table.nodes[id].operator.children().into_iter().copied()); - } - } - let local: HashMap<_, _> = reached.iter().enumerate().map(|(i, &id)| (id, i)).collect(); - let nodes: Vec<_> = reached - .iter() - .map(|&id| { - let mut node = table.nodes[id].clone(); - node.operator = node.operator.map_children(|child| local[child]); - node - }) - .collect(); - FlatDag { - nodes, - roots: vec![QueryRoot::Operator(local[&root])], - } - }; - for (node, root) in [(sum, sum_root[0]), (max, max_root[0])] { - assert_eq!( - serde_json::to_value(rebuilt(root)).unwrap(), - serde_json::to_value(export(&[QueryRoot::Operator(node)])).unwrap() - ); - } - } -} diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index 1b72e8cc9..3bc274793 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -987,11 +987,11 @@ fn with_root_estimate( } /// The family of the build whose state `state` carries. -fn state_family(state: &OperatorNode) -> Result { +fn state_family(state: &Rc) -> Result { match state.summary_build() { Some( ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => Ok(family.clone()), + ) => Ok(family), _ => Err("q's estimate reads no single summary build".into()), } } @@ -1226,7 +1226,7 @@ pub fn witness_root( }) } Witness::OtherStatistic { statistic } => { - let answers = |state: &OperatorNode| -> Result<(), String> { + let answers = |state: &Rc| -> Result<(), String> { let family = state_family(state)?; if FamilyCapability::of(&family).answers(statistic) { Ok(()) diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index 1692de316..e136e4e9c 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -21,11 +21,10 @@ //! ``` use std::fmt::Write; -use std::rc::Rc; use asap_types::ir::flat::flatten; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{Memo, PhysicalASAPMemoCandidates}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{Memo, MemoGroup, PhysicalASAPMemoCandidates}; use crate::candidate_labels::node_label; use crate::planning_stages::{PlanningStagesRun, RunUntil}; @@ -191,15 +190,12 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi .enumerate() .map(|(id, group)| { let children: Vec = group - .child_groups - .iter() + .expression + .children() + .into_iter() .map(|child| format!("G{}", child.0)) .collect(); - vec![ - format!("G{id}:"), - root_label(&group.logical_node), - reads(&children), - ] + vec![format!("G{id}:"), group_label(group), reads(&children)] }) .collect(); let mut group_lines = String::new(); @@ -237,10 +233,10 @@ fn write_selection(out: &mut String, stages: &PlanningStagesRun) { todo!("#1207 PR 9: print the selected DAG over the new physical candidates") } -/// The label of `node`'s own operator, without its children. -fn root_label(node: &Rc) -> String { - let (flat, _) = flatten(&[QueryRoot::Operator(node.clone())]); - flat.nodes.last().map(node_label).unwrap_or_default() +/// The label of `group`'s own operator, without its input groups. +fn group_label(group: &MemoGroup) -> String { + let _ = group; + todo!("#1207 PR 9: label a group by its expression") } #[cfg(test)] diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index e32464736..8c9afa66a 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -661,17 +661,15 @@ fn compile_internal( /// Whether every summary state `state` merges was built from a relation's /// rows (a SQL table's), where a global aggregate has a row over no input /// row; not from a PromQL vector's, where it has none. -fn built_from_relation(state: &OperatorNode) -> bool { +fn built_from_relation(state: &std::rc::Rc) -> bool { match state.asap() { - Some(ASAPOp::SummaryMerge { children, .. }) => { - children.iter().all(|child| built_from_relation(child)) - } + Some(ASAPOp::SummaryMerge { children, .. }) => children.iter().all(built_from_relation), Some( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, ) => child.result_kind == planner_types::ir::OperatorResultKind::Relation, // A filter keeping whole states reads them from their build. _ => planner_types::ir::operator::asap::selected_from(state) - .is_some_and(|states| built_from_relation(states)), + .is_some_and(|states| built_from_relation(&states)), } } diff --git a/crates/frontend-sql/src/sql/collection_udfs.rs b/crates/frontend-sql/src/sql/collection_udfs.rs index 17711eb83..c4892190c 100644 --- a/crates/frontend-sql/src/sql/collection_udfs.rs +++ b/crates/frontend-sql/src/sql/collection_udfs.rs @@ -118,7 +118,7 @@ impl CollectionUdf { Ok(ScalarExpr::Column(index)) } }) - .collect::>>()?; + .collect::>>()?; match self.function { CollectionFunction::Element => element_access_type(&args, &schema), CollectionFunction::StructField => struct_field_type(&args, &schema), diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 5ddb8a08b..d145c8228 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -351,8 +351,9 @@ pub mod planning { depth[group] = d; pending.extend( memo.groups()[group] - .child_groups - .iter() + .expression + .children() + .into_iter() .map(|child| (child.0 as usize, d + 1)), ); } diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 5a1fd282b..4890c85fb 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -20,7 +20,7 @@ async fn promql_eval_time_and_sql_current_timestamp_remain_distinct() { "expected a bare evaluation-time scalar, got {promql:?}" ); assert_eq!( - ScalarExpr::EvalTimestamp + ScalarExpr::>::EvalTimestamp .scalar_type(&Schema::default()) .unwrap() .0, diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index 08e7d6f74..dfbe200d2 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -34,5 +34,4 @@ mod sql_cardinality; mod sql_frequency; mod sql_to_post_asap; mod stage1_logical_asap_optimization; -mod summary_coverage_e2e; mod time_range; diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs index 6546ede02..a49628beb 100644 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ b/crates/integration-tests/tests/sketch_rollup.rs @@ -241,11 +241,6 @@ async fn direct_and_rolled_up( continue; }; let merge = rolled_up(&build); - assert_eq!( - asap_types::test_support::definition_id(&merge.coverage().expect("a valid rollup")), - asap_types::test_support::definition_id(&build.coverage().unwrap()), - "{kind:?}: the rolled-up states cover what the direct build does" - ); let rolled = replace(&candidate, &build, &merge); rolled.validate_structure().unwrap(); out.push((*kind, candidate, rolled)); @@ -363,9 +358,9 @@ async fn invalid_rollups_are_rejected() { assert!(regroup(keys.clone()).is_err(), "{keys:?}"); } // No keys: the global build over the same rows. - assert!(regroup(vec![]).unwrap().coverage().is_some()); + assert!(regroup(vec![]).is_ok()); // Both keys, swapped, are still group keys of the input. - assert!(regroup(vec![1, 0]).unwrap().coverage().is_some()); + assert!(regroup(vec![1, 0]).is_ok()); // Rate states merge consecutive windows of one series, not groups. let Some(ASAPOp::SummaryAgg { child, diff --git a/crates/integration-tests/tests/summary_coverage_e2e.rs b/crates/integration-tests/tests/summary_coverage_e2e.rs deleted file mode 100644 index dc5446817..000000000 --- a/crates/integration-tests/tests/summary_coverage_e2e.rs +++ /dev/null @@ -1,262 +0,0 @@ -//! Query string → planned summary state → `coverage()`, for the examples of -//! the ASAP primitive schema design doc (#573 §4.2.2). Each query is lowered -//! and its logical candidates composed, and the quantile's KLL `SummaryAgg` -//! is checked: what its `definition` keeps and what its `selection` takes, -//! row by row of §4.2.2's "More examples". The tests read the candidates, not -//! plan selection's choice, so they do not depend on which candidate is -//! cheapest. - -use std::ops::Bound; -use std::rc::Rc; - -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::planning::explicit_summary_candidate; -use asap_types::ir::properties::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ScalarValue; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, Predicate, ScalarExpr}; - -fn accuracy() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -/// The single KLL `SummaryAgg` in an explicit KLL rule fixture for `pre`. -/// Coverage semantics do not depend on the capacities visited by search. -fn kll_state(pre: Rc) -> Rc { - let dag = explicit_summary_candidate(pre, SketchAlgorithm::Kll); - let mut states: Vec<_> = OperatorNode::reachable(&dag) - .into_iter() - .filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) - if kind == &SketchAlgorithm::Kll) - }) - .collect(); - assert_eq!( - states.len(), - 1, - "expected one KLL candidate state: {states:#?}" - ); - states.pop().unwrap() -} - -async fn sql_state(sql: &str) -> Rc { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]), - ); - let pre = lower_sql(sql, &catalog, accuracy()) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - kll_state(pre) -} - -fn promql_state(query: &str) -> Rc { - kll_state(lower_promql(query, accuracy()).expect("lowering failed")) -} - -fn column(table: Option<&str>, name: &str) -> ColumnIdentity { - ColumnIdentity::new(table.map(str::to_string), name.to_string()) -} - -fn utf8(value: &str) -> ScalarValue { - ScalarValue::Utf8(value.to_string()) -} - -/// The child of the definition's `SummaryAgg`. -fn definition_child(state: &OperatorNode) -> Rc { - state.coverage().unwrap().definition().children()[0].clone() -} - -/// The definition's `SummaryAgg` carries no filter of its own. -fn assert_definition_unfiltered(state: &OperatorNode) { - let coverage = state.coverage().unwrap(); - let definition = coverage.definition(); - assert!( - matches!( - definition.asap(), - Some(ASAPOp::SummaryAgg { filter: None, .. }) - ), - "{definition:#?}" - ); -} - -/// The definition keeps the whole sub-DAG below the state: nothing moved -/// into the selection, which is the one unrestricted box. -fn assert_nothing_selected(state: &OperatorNode) { - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::default()] - ); - assert!(Rc::ptr_eq(&definition_child(state), state.children()[0])); -} - -#[tokio::test] -async fn sql_where_value_set_moves_to_selection() { - let state = - sql_state("SELECT approx_percentile_cont(latency, 0.99) FROM t WHERE region = 'us'").await; - let coverage = state.coverage().unwrap(); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [( - column(Some("t"), "region"), - Constraint::In(vec![utf8("us")]) - )] - .into(), - None - )] - ); - assert_definition_unfiltered(&state); - let child = definition_child(&state); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty()) - ); -} - -#[tokio::test] -async fn sql_where_range_moves_to_selection() { - let state = - sql_state("SELECT approx_percentile_cont(latency, 0.99) FROM t WHERE latency < 100").await; - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column(Some("t"), "latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Float64(100.0)), - } - )] - .into(), - None - )] - ); - assert_definition_unfiltered(&state); - let child = definition_child(&state); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty()) - ); -} - -/// The worked example without its `FILTER` clause: the rename carries -/// `region = 'us'` into the selection as `r`, and the expression condition -/// stays in the definition's scan. -#[tokio::test] -async fn sql_renamed_column_moves_and_expression_stays() { - let state = sql_state( - "SELECT job, approx_percentile_cont(latency, 0.99) \ - FROM (SELECT job, region AS r, latency FROM t \ - WHERE region = 'us' AND latency * 2 > 10) \ - GROUP BY job", - ) - .await; - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [(column(None, "r"), Constraint::In(vec![utf8("us")]))].into(), - None - )] - ); - assert_definition_unfiltered(&state); - let project = definition_child(&state); - assert!(matches!( - project.non_asap(), - Some(NonASAPOp::Project { .. }) - )); - let scan = project.children()[0].clone(); - let Some(NonASAPOp::Scan { predicates, .. }) = scan.non_asap() else { - panic!("expected a scan, got {scan:#?}"); - }; - assert_eq!(predicates.len(), 1, "{predicates:#?}"); - assert!( - matches!(&predicates[0], Predicate(ScalarExpr::Compare { left, .. }) - if matches!(**left, ScalarExpr::Arithmetic { .. })), - "the expression condition should stay: {predicates:#?}" - ); -} - -#[test] -fn promql_offset_window_is_relative_time() { - let state = promql_state("quantile_over_time(0.99, m[1m] offset 2m)"); - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(-180_000), Bound::Included(-120_000))) - )] - ); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::Scan { .. }) - )); -} - -/// The label matcher is below `rate`, which the walk does not pass: the -/// definition keeps all of it, the matcher included. -#[test] -fn promql_matcher_below_rate_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(m{job=\"api\"}[5m]))"); - assert_nothing_selected(&state); - let scans: Vec<_> = OperatorNode::reachable(&definition_child(&state)) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { predicates, .. }) => Some(predicates.len()), - _ => None, - }) - .collect(); - assert_eq!(scans, vec![1], "the matcher stays in the scan"); -} - -/// A PromQL comparison lowers to a `Filter` above `rate`, so it picks the -/// rate outputs the state reads; the definition keeps the `rate` below it. -#[test] -fn promql_comparison_above_rate_moves_to_selection() { - let state = promql_state("quantile(0.99, rate(m[5m]) > 0)"); - let filter = state.children()[0]; - assert!(matches!(filter.non_asap(), Some(NonASAPOp::Filter { .. }))); - assert!(Rc::ptr_eq(&definition_child(&state), filter.children()[0])); - assert_eq!( - state.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column(None, "value"), - Constraint::Interval { - lower: Bound::Excluded(ScalarValue::Float64(0.0)), - upper: Bound::Unbounded, - }, - )] - .into(), - None - )] - ); -} - -/// A vector/vector comparison is a `BinaryOp`, which the walk does not enter. -#[test] -fn promql_vector_comparison_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(a[5m]) > rate(b[5m]))"); - assert_nothing_selected(&state); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::BinaryOp { .. }) - )); -} - -/// A `bool` comparison lowers to a `Project` (values become 0 or 1), which -/// drops no row, so it stays in the definition. -#[test] -fn promql_bool_comparison_stays_in_definition() { - let state = promql_state("quantile(0.99, rate(m[5m]) > bool 0)"); - assert_nothing_selected(&state); - assert!(matches!( - definition_child(&state).non_asap(), - Some(NonASAPOp::Project { .. }) - )); -} diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index f5a81d896..96c7f469d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -659,6 +659,14 @@ fn placement(below: bool) -> HavingPlacement { } } +/// Each memo group's expression as an operator-node tree, by group id: the +/// trees target discovery still walks. `Memo::logical_node_trees` is +/// deleted; discovery reads the groups in #1207 PR 3. +fn logical_node_trees(memo: &Memo) -> Vec> { + let _ = memo; + todo!("#1207 PR 3: discover targets over the memo groups, not node trees") +} + /// Discover one target per measure of each eligible aggregate memo group of /// `dag`, including operator plans read by scalar roots: Pass 1 walks the /// groups, so an aggregate several queries read is one target per measure. @@ -673,11 +681,7 @@ fn placement(below: bool) -> HavingPlacement { pub fn generate_per_computation_candidates( memo: &Memo, ) -> Result { - let logical_nodes: Rc<[Rc]> = memo - .groups() - .iter() - .map(|group| Rc::clone(&group.logical_node)) - .collect(); + let logical_nodes: Rc<[Rc]> = logical_node_trees(memo).into(); let group_of: HashMap<*const OperatorNode, MemoGroupId> = logical_nodes .iter() .enumerate() @@ -1975,7 +1979,7 @@ fn key_items(keys: usize) -> Vec { /// included, reads its input column (the sample value when it names none). fn measure_input( intent: &AggIntent, - input: &OperatorNode, + input: &Rc, filter: Option<&Predicate>, ) -> Result { let schema = &input.schema; diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs index 15a95ad23..bcc2a625d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs @@ -820,20 +820,10 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { let QueryRoot::Operator(estimate) = &hll[0] else { panic!("operator root expected") }; - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - panic!("summary evaluation expected") - }; - // The state takes every row of its source: one unrestricted selection box. - let coverage = summary_input.coverage().unwrap(); - assert!(OperatorNode::reachable(coverage.definition()).iter().any(|node| matches!( - node.non_asap(), - Some(NonASAPOp::Scan { source: Source::Table { table_ref }, .. }) if table_ref == "flows" - ))); - assert_eq!(coverage.selection().len(), 1); - assert!( - coverage.selection()[0].columns().is_empty() - && coverage.selection()[0].relative_time().is_none() - ); + assert!(matches!( + estimate.asap(), + Some(ASAPOp::SummaryEstimate { .. }) + )); hll[0].validate_structure().unwrap(); assert!(compose_logical_candidate(&inventory, &[99]).is_err()); } diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/logical-asap-optimization/src/pass1/realization.rs index 285a814bb..f510bdf65 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/logical-asap-optimization/src/pass1/realization.rs @@ -83,7 +83,9 @@ pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> Co /// [`ranked_column`] as a reference: the sample value when it is that /// column, else the column by name. -pub(crate) fn ranked_value(child: &OperatorNode) -> Result<(ColumnId, ColumnRef), ResolveError> { +pub(crate) fn ranked_value( + child: &Rc, +) -> Result<(ColumnId, ColumnRef), ResolveError> { let ranked = ranked_column(child)?; let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { Ok(value) if value == ranked => ColumnRef::SampleValue, diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index 9a8b45ffc..d9971be0c 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -1087,79 +1087,15 @@ impl<'a> ViewIndex<'a> { } } -/// The coverage of view `id` (IR `ir`), refreshed from `inputs`. A merge's -/// is the union of its inputs' selections under their shared definition -/// ([`SummaryCoverage::derive`]). A summary built from exactly one view and -/// nothing else, with no selection of its own (a rollup to a coarser -/// grouping, or across series), takes that input's selection: its -/// definition is its own sub-DAG with the input replaced by the input's -/// definition, so the rollups of equal inputs share one definition. -/// Anything else keeps its own derived coverage. +/// The coverage of view `id` (IR `ir`), refreshed from `inputs`: the +/// coverage its memo group stores, which the realized plan will carry. +/// Coverage is derived only in the memo, never on node trees. fn coverage_of( id: PhysicalASAPNodeId, inputs: &MaterializedViewInputs, ir: &HashMap>, coverage_by_node: &HashMap, ) -> Result { - let node = &ir[&id]; - let own = SummaryCoverage::derive(node)?; - let [input] = inputs.materialized_views.as_slice() else { - return Ok(own); - }; - if !inputs.raw_data_windows.is_empty() - || matches!(node.asap(), Some(ASAPOp::SummaryMerge { .. })) - || own.selection != [SelectionBox::default()] - { - return Ok(own); - } - let input_coverage = &coverage_by_node[input]; - let over_definition = with_replaced(node, &ir[input], &input_coverage.definition); - Ok(SummaryCoverage { - definition: SummaryCoverage::derive(&over_definition)?.definition, - selection: input_coverage.selection.clone(), - }) -} - -/// `node` with every use of `old` (by address) replaced by `new`; nodes -/// that do not reach `old` are shared, not copied. -fn with_replaced( - node: &Rc, - old: &Rc, - new: &Rc, -) -> Rc { - fn replace( - node: &Rc, - old: &Rc, - new: &Rc, - cache: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if Rc::ptr_eq(node, old) { - return Rc::clone(new); - } - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let children = node.children(); - let replaced: Vec> = children - .iter() - .map(|child| replace(child, old, new, cache)) - .collect(); - let result = if children - .iter() - .zip(&replaced) - .all(|(a, b)| Rc::ptr_eq(a, b)) - { - Rc::clone(node) - } else { - let mut next = replaced.into_iter(); - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|_| next.next().expect("one per child")); - Rc::new(copy) - }; - cache.insert(Rc::as_ptr(node), Rc::clone(&result)); - result - } - replace(node, old, new, &mut HashMap::new()) + let _ = (id, inputs, ir, coverage_by_node); + todo!("#1207 PR 6: the view's coverage from its memo group's GroupProperties::summary_coverage") } diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 707ab4add..3c35f8118 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -5,7 +5,8 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; -use super::node::{OperatorNode, OperatorResultKind}; +use super::input::{BuiltOrInput, OperatorInput}; +use super::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; use crate::ir::operator::operator_properties::Reduction; use crate::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; @@ -263,7 +264,7 @@ impl ASAPOp { } } -impl ASAPOp { +impl ASAPOp { fn unimplemented() -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(UNIMPLEMENTED_ASAP_OP.into()) } @@ -276,7 +277,7 @@ impl ASAPOp { | ASAPOp::SummaryJoin { family, .. } => Some(family), ASAPOp::SummaryMerge { children, .. } => children.first().and_then(|child| { child - .schema + .schema() .fields .iter() .find(|field| !field.is_plain()) @@ -304,29 +305,12 @@ impl ASAPOp { family, reduction, .. - } => { - let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( - &child.schema, - reduction, - &[crate::ir::operator::AggIntent::Sum { col: None }], - &[], - )?; - let index = match reduction { - Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( - &crate::ir::scalar::ColumnRef::SampleValue, - &schema, - ) - .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, - Reduction::Reduce(_) => schema.fields.len() - 1, - }; - schema.fields[index] = Field::new("state", family.clone(), false); - schema - } + } => summary_agg_schema(child.schema(), family, reduction)?, SummaryEstimate { summary_input, query, } => { - let input = &summary_input.schema; + let input = summary_input.schema(); let (name, mut dtype) = match query { SketchStatistic::Quantile { .. } => ("quantile", DataType::Float64), SketchStatistic::Cardinality => ("cardinality", DataType::Int64), @@ -336,7 +320,7 @@ impl ASAPOp { SketchStatistic::TopK { .. } => ("topk", DataType::Utf8), }; if matches!( - summary_input.summary_build(), + summary_build(summary_input), Some( SummaryAgg { reduction: Reduction::PerEntity, @@ -367,107 +351,31 @@ impl ASAPOp { } FinalizeExactAccumulator { child } => { // A rollup finalizes as the coarse build it stands for. - if let Some(build) = rolled_up_build(child) { - return FinalizeExactAccumulator { child: build }.output_schema(); + if let Some((build, schema)) = rolled_up_build(child) { + return finalized_schema(Some(&build), &schema); } // Merged tumbling windows finalize as one build over their window. if super::tumbling_window_chain::merged_chain(child).is_some() { - return FinalizeExactAccumulator { - child: Rc::new(super::tumbling_window_chain::merged_window_build(child)?), - } - .output_schema(); - } - let value_result = if let Some(ASAPOp::SummaryAgg { - child: source, - family: FieldDataType::ExactAggregate(kind, _), - input, - reduction, - filter, - .. - }) = child.summary_build() - { - { - use crate::ir::operator::AggIntent; - use crate::ir::schema::{ExactKind, SummaryInputExpr}; - let column = match &input.weight { - SummaryInputExpr::Column(col) => Some( - crate::ir::scalar::column_resolution::resolve_column_ref( - col, - &source.schema, - ) - .map_err(|e| { - SchemaDerivationError::InvalidScalarSignature(e.to_string()) - })?, - ), - _ => None, - }; - let measure = match kind { - ExactKind::Sum => Some(AggIntent::Sum { col: column }), - ExactKind::Min => Some(AggIntent::Min { col: column }), - ExactKind::Max => Some(AggIntent::Max { col: column }), - ExactKind::Count => Some(AggIntent::Count { - accuracy: crate::ir::properties::AccuracyTarget::exact(), - }), - ExactKind::Rate => Some(AggIntent::Rate), - ExactKind::IRate => Some(AggIntent::IRate), - ExactKind::Increase => Some(AggIntent::Increase), - }; - measure - .map(|measure| { - crate::ir::NonASAPOp::Aggregate { - child: Rc::clone(source), - reduction: reduction.clone(), - measures: vec![measure], - output_names: vec![], - // A filtered SUM/MIN/MAX is NULL for a - // group without a matching row. - filters: vec![filter.clone()], - having: None, - } - .output_schema() - }) - .transpose()? - .and_then(|s| match reduction { - Reduction::PerEntity => { - s.column_id("value").and_then(|i| s.fields.get(i).cloned()) - } - Reduction::Reduce(_) => s.fields.last().cloned(), - }) - } - } else { - None - }; - let mut out = child.schema.clone(); - for f in &mut out.fields { - if let FieldDataType::ExactAggregate(kind, _) = &f.dtype { - if let Some(result) = &value_result { - // The finalized value replaces the aggregate it - // realizes, so it takes that aggregate's column. - f.name = result.name.clone(); - f.dtype = result.dtype.clone(); - f.nullable = result.nullable; - } else { - f.dtype = FieldDataType::Plain(finalized_data_type(kind)); - } - } + let (build, schema) = super::tumbling_window_chain::merged_window_build(child)?; + return finalized_schema(Some(&build), &schema); } - out + finalized_schema(summary_build(child).as_ref(), child.schema())? } - MaintainPopulation { child, .. } => child.schema.clone(), + MaintainPopulation { child, .. } => child.schema().clone(), EvaluatePopulation { child, evaluation } => { use crate::ir::operator::maintained_population::PopulationInput; use crate::ir::operator::{AggIntent, GroupKeys}; - let Some(MaintainPopulation { + let Operator::ASAP(MaintainPopulation { child: source, population, - }) = child.asap() + }) = child.operator() else { return Err(SchemaDerivationError::InvalidScalarSignature( "population evaluation requires maintained membership".into(), )); }; if matches!(evaluation, PopulationStatistic::TopK { .. }) { - source.schema.clone() + source.schema().clone() } else { let (keys, column) = match &population.input { PopulationInput::Rows { @@ -480,7 +388,7 @@ impl ASAPOp { .grouping .iter() .map(|name| { - source.schema.column_id(name).ok_or_else(|| { + source.schema().column_id(name).ok_or_else(|| { SchemaDerivationError::InvalidScalarSignature( "population grouping column is absent".into(), ) @@ -524,7 +432,7 @@ impl ASAPOp { children, group_by, .. } => { self.validate_inputs()?; - let input = &children[0].schema; + let input = children[0].schema(); match group_by { None => input.clone(), Some(keys) => regrouped_schema(input, keys)?, @@ -558,11 +466,11 @@ impl ASAPOp { /// Local input-contract checks. pub(crate) fn validate_inputs(&self) -> Result<(), SchemaDerivationError> { use ASAPOp::*; - let needs_state = |node: &OperatorNode, what: &str| { - if node.result_kind != OperatorResultKind::State { + let needs_state = |node: &I, what: &str| { + if node.result_kind() != OperatorResultKind::State { Err(SchemaDerivationError::InvalidScalarSignature(format!( "{what} requires summary state as input, got {:?}", - node.result_kind + node.result_kind() ))) } else { Ok(()) @@ -581,11 +489,11 @@ impl ASAPOp { } if let Some(n) = newest_tumbling_windows { let tumbling_window_count = match children.as_slice() { - [chain] => match chain.asap() { - Some(SummaryTumblingWindowChain { + [chain] => match chain.operator() { + Operator::ASAP(SummaryTumblingWindowChain { tumbling_window_count, .. - }) => Some(*tumbling_window_count), + }) => Some(tumbling_window_count), _ => None, }, _ => None, @@ -606,7 +514,7 @@ impl ASAPOp { // Matching state parameters and grouping positions are necessary; // matching names alone cannot prove two states compatible. if first - .schema + .schema() .fields .iter() .filter(|field| !field.is_plain()) @@ -619,7 +527,7 @@ impl ASAPOp { } for child in children { needs_state(child, "SummaryMerge")?; - if child.schema != first.schema { + if child.schema() != first.schema() { return Err(SchemaDerivationError::InvalidScalarSignature( "summary merge inputs must have identical state and grouping schemas" .into(), @@ -639,7 +547,7 @@ impl ASAPOp { // The family-capability table is the one source of which // statistic a state's family answers. let states: Vec<_> = summary_input - .schema + .schema() .fields .iter() .filter(|f| !f.is_plain()) @@ -663,7 +571,7 @@ impl ASAPOp { FinalizeExactAccumulator { child } => { needs_state(child, "FinalizeExactAccumulator")?; if child - .schema + .schema() .fields .iter() .all(|f| !matches!(f.dtype, FieldDataType::ExactAggregate(..))) @@ -676,7 +584,7 @@ impl ASAPOp { } EvaluatePopulation { child, evaluation } => { needs_state(child, "EvaluatePopulation")?; - if !matches!(child.asap(), Some(MaintainPopulation { population, .. }) if population.supports(evaluation)) + if !matches!(child.operator(), Operator::ASAP(MaintainPopulation { population, .. }) if population.supports(evaluation)) { return Err(SchemaDerivationError::InvalidScalarSignature( "population evaluation requires compatible maintained membership".into(), @@ -698,12 +606,12 @@ impl ASAPOp { // The chain is the range selector: it reads the series the // selector would, or a SQL table's rows for a trailing window. if !matches!( - child.result_kind, + child.result_kind(), OperatorResultKind::InstantVector | OperatorResultKind::Relation ) { return Err(SchemaDerivationError::InvalidScalarSignature(format!( "a tumbling-window chain reads an instant vector or a relation, got {:?}", - child.result_kind + child.result_kind() ))); } self.summary_agg_inputs() @@ -740,7 +648,7 @@ impl ASAPOp { else { unreachable!("called on a summary build"); }; - if family.is_plain() || child.result_kind == OperatorResultKind::State { + if family.is_plain() || child.result_kind() == OperatorResultKind::State { return Err(SchemaDerivationError::InvalidScalarSignature( "summary aggregation requires values and produces a state family".into(), )); @@ -765,12 +673,12 @@ impl ASAPOp { } Ok(()) } - check(&input.weight, &child.schema)?; + check(&input.weight, child.schema())?; if let Some(item) = &input.item { - check(item, &child.schema)?; + check(item, child.schema())?; } if let Some(filter) = filter { - if filter.0.scalar_type(&child.schema)?.0 != DataType::Bool { + if filter.0.scalar_type(child.schema())?.0 != DataType::Bool { return Err(SchemaDerivationError::InvalidScalarSignature( "summary filter must be boolean".into(), )); @@ -781,26 +689,30 @@ impl ASAPOp { } /// The build a rollup (a `SummaryMerge` with `group_by` of one input) -/// stands for: the input's build (one build over a merged chain's window) -/// grouped by the input columns the rollup's keys name. `None` for any -/// other node, or a rollup whose build does not derive. -fn rolled_up_build(state: &OperatorNode) -> Option> { - let ASAPOp::SummaryMerge { +/// stands for, with its schema: the input's build (one build over a merged +/// chain's window) grouped by the input columns the rollup's keys name. +/// `None` for any other input, or a rollup whose build does not derive. +fn rolled_up_build(state: &I) -> Option<(ASAPOp>, Schema)> { + let Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: Some(keys), .. - } = state.asap()? + }) = state.operator() else { return None; }; let [input] = children.as_slice() else { return None; }; - let input = selected_from(input).unwrap_or(input); - let build = match input.asap()? { - ASAPOp::SummaryAgg { .. } => (**input).clone(), - _ if super::tumbling_window_chain::merged_chain(input).is_some() => { - super::tumbling_window_chain::merged_window_build(input).ok()? + let input = selected_from(input).unwrap_or_else(|| input.clone()); + let build = match input.operator() { + Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => { + op.map_children(|child| BuiltOrInput::Input(child.clone())) + } + _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { + super::tumbling_window_chain::merged_window_build(&input) + .ok()? + .0 } _ => return None, }; @@ -811,27 +723,138 @@ fn rolled_up_build(state: &OperatorNode) -> Option> { reduction, grouping, filter, - } = build.asap()? + } = build else { return None; }; let mapped = keys .iter() - .map(|&key| match reduction { + .map(|&key| match &reduction { Reduction::Reduce(by) if !by.is_without() => by.keys().get(key).copied(), Reduction::PerEntity => Some(key), Reduction::Reduce(_) => None, }) .collect::>>()?; - OperatorNode::new_shared(super::node::Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: family.clone(), - input: update.clone(), - reduction: Reduction::by(mapped), - grouping: grouping.clone(), - filter: filter.clone(), - })) - .ok() + let reduction = Reduction::by(mapped); + let schema = summary_agg_schema(child.schema(), &family, &reduction).ok()?; + Some(( + ASAPOp::SummaryAgg { + child, + family, + input: update, + reduction, + grouping, + filter, + }, + schema, + )) +} + +/// The schema of a `SummaryAgg` (or `SummaryTumblingWindowChain`) of +/// `family` by `reduction` over rows of schema `child`: the reduction's +/// output columns, its value column replaced by the state. +pub(crate) fn summary_agg_schema( + child: &Schema, + family: &FieldDataType, + reduction: &Reduction, +) -> Result { + let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( + child, + reduction, + &[crate::ir::operator::AggIntent::Sum { col: None }], + &[], + )?; + let index = match reduction { + Reduction::PerEntity => crate::ir::scalar::resolve_column_ref( + &crate::ir::scalar::ColumnRef::SampleValue, + &schema, + ) + .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, + Reduction::Reduce(_) => schema.fields.len() - 1, + }; + schema.fields[index] = Field::new("state", family.clone(), false); + Ok(schema) +} + +/// The schema `FinalizeExactAccumulator` derives over state of +/// `state_schema` that `build` (the state's `SummaryAgg`, if any) builds: +/// each exact aggregate replaced by the value it finalizes to. +fn finalized_schema( + build: Option<&ASAPOp>, + state_schema: &Schema, +) -> Result { + let value_result = if let Some(ASAPOp::SummaryAgg { + child: source, + family: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + }) = build + { + { + use crate::ir::operator::AggIntent; + use crate::ir::schema::{ExactKind, SummaryInputExpr}; + let column = match &input.weight { + SummaryInputExpr::Column(col) => Some( + crate::ir::scalar::column_resolution::resolve_column_ref(col, source.schema()) + .map_err(|e| { + SchemaDerivationError::InvalidScalarSignature(e.to_string()) + })?, + ), + _ => None, + }; + let measure = match kind { + ExactKind::Sum => Some(AggIntent::Sum { col: column }), + ExactKind::Min => Some(AggIntent::Min { col: column }), + ExactKind::Max => Some(AggIntent::Max { col: column }), + ExactKind::Count => Some(AggIntent::Count { + accuracy: crate::ir::properties::AccuracyTarget::exact(), + }), + ExactKind::Rate => Some(AggIntent::Rate), + ExactKind::IRate => Some(AggIntent::IRate), + ExactKind::Increase => Some(AggIntent::Increase), + }; + measure + .map(|measure| { + crate::ir::NonASAPOp::Aggregate { + child: source.clone(), + reduction: reduction.clone(), + measures: vec![measure], + output_names: vec![], + // A filtered SUM/MIN/MAX is NULL for a + // group without a matching row. + filters: vec![filter.clone()], + having: None, + } + .output_schema() + }) + .transpose()? + .and_then(|s| match reduction { + Reduction::PerEntity => { + s.column_id("value").and_then(|i| s.fields.get(i).cloned()) + } + Reduction::Reduce(_) => s.fields.last().cloned(), + }) + } + } else { + None + }; + let mut out = state_schema.clone(); + for f in &mut out.fields { + if let FieldDataType::ExactAggregate(kind, _) = &f.dtype { + if let Some(result) = &value_result { + // The finalized value replaces the aggregate it + // realizes, so it takes that aggregate's column. + f.name = result.name.clone(); + f.dtype = result.dtype.clone(); + f.nullable = result.nullable; + } else { + f.dtype = FieldDataType::Plain(finalized_data_type(kind)); + } + } + } + Ok(out) } /// The schema of a regrouping `SummaryMerge` over states of schema `input`: @@ -877,11 +900,11 @@ fn finalized_data_type(kind: &crate::ir::schema::sketch::ExactKind) -> DataType /// The states `node` selects from when it is a `Filter` that keeps whole /// summary states by their plain columns (its output is state); `None` for -/// any other node. -pub fn selected_from(node: &OperatorNode) -> Option<&Rc> { - match node.non_asap() { - Some(crate::ir::NonASAPOp::Filter { child, .. }) - if node.result_kind == OperatorResultKind::State => +/// any other input. +pub fn selected_from(node: &I) -> Option { + match node.operator() { + Operator::NonASAP(crate::ir::NonASAPOp::Filter { child, .. }) + if node.result_kind() == OperatorResultKind::State => { Some(child) } @@ -889,18 +912,38 @@ pub fn selected_from(node: &OperatorNode) -> Option<&Rc> { } } +/// The build whose state this summary state carries: a `SummaryAgg` or +/// `SummaryTumblingWindowChain` itself, or the first input's build under +/// `SummaryMerge`s (merged inputs have one definition, which summary +/// coverage checks). `None` under a regrouping merge, whose states no build +/// carries (its coverage's definition is the build it stands for), and for +/// any other input. +pub fn summary_build(state: &I) -> Option> { + match state.operator() { + Operator::ASAP( + op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }), + ) => Some(op), + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + .. + }) => summary_build(children.first()?), + _ => None, + } +} + /// The category of the values a evaluation of `state` produces: the category /// of the relational input the state was built from. -fn source_kind(node: &OperatorNode) -> OperatorResultKind { +fn source_kind(node: &I) -> OperatorResultKind { if let Some(states) = selected_from(node) { - return source_kind(states); + return source_kind(&states); } - match &node.operator { - super::node::Operator::ASAP(op) => match op.children().first() { - Some(child) => source_kind(child), + match node.operator() { + Operator::ASAP(op) => match op.children().first() { + Some(child) => source_kind(*child), None => OperatorResultKind::Relation, }, - super::node::Operator::NonASAP(_) => match node.result_kind { + Operator::NonASAP(_) => match node.result_kind() { OperatorResultKind::RangeVector => OperatorResultKind::InstantVector, OperatorResultKind::State => OperatorResultKind::Relation, other => other, @@ -914,21 +957,28 @@ fn source_kind(node: &OperatorNode) -> OperatorResultKind { /// series identity), then the estimated weight, `topk` (Float64). `None` /// when `state` is not a keyed build whose item columns resolve in its /// input. -fn topk_estimate_schema(state: &OperatorNode) -> Option { - use crate::ir::schema::SummaryInputExpr; +fn topk_estimate_schema(state: &I) -> Option { // A rollup reads items as the coarse build it stands for. - let rolled_up = rolled_up_build(state); - let build = match &rolled_up { - Some(build) => build.asap(), - None => state.summary_build(), - }; - let (Some(ASAPOp::SummaryAgg { child, input, .. }) - | Some(ASAPOp::SummaryTumblingWindowChain { child, input, .. })) = build + match rolled_up_build(state) { + Some((build, _)) => topk_schema_of_build(&build, state.schema()), + None => topk_schema_of_build(&summary_build(state)?, state.schema()), + } +} + +/// [`topk_estimate_schema`] of state of `state_schema` built by `build`. +fn topk_schema_of_build( + build: &ASAPOp, + state_schema: &Schema, +) -> Option { + use crate::ir::schema::SummaryInputExpr; + let (ASAPOp::SummaryAgg { child, input, .. } + | ASAPOp::SummaryTumblingWindowChain { child, input, .. }) = build else { return None; }; + let child_schema = child.schema(); let resolve = |column: &crate::ir::scalar::ColumnRef| { - crate::ir::scalar::resolve_column_ref(column, &child.schema).ok() + crate::ir::scalar::resolve_column_ref(column, child_schema).ok() }; let columns = match input.item.as_ref()? { SummaryInputExpr::Column(column) => vec![resolve(column)?], @@ -944,9 +994,9 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { }) => { let excluded: Vec<_> = excluding.iter().filter_map(resolve).collect(); let value = resolve(&crate::ir::scalar::ColumnRef::SampleValue); - (0..child.schema.fields.len()) + (0..child_schema.fields.len()) .filter(|index| { - Some(*index) != child.schema.time_index + Some(*index) != child_schema.time_index && Some(*index) != value && !excluded.contains(index) }) @@ -960,12 +1010,11 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { let complete_item_identity = matches!( input.item.as_ref()?, SummaryInputExpr::Column(_) | SummaryInputExpr::Tuple(_) - ) || child.schema.closed + ) || child_schema.closed || columns.iter().any(|&index| { - child.schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY + child_schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY }); - let mut fields: Vec = state - .schema + let mut fields: Vec = state_schema .fields .iter() .filter(|field| field.is_plain()) @@ -974,9 +1023,9 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { for index in columns { if !fields .iter() - .any(|field| field.name == child.schema.fields[index].name) + .any(|field| field.name == child_schema.fields[index].name) { - fields.push(child.schema.fields[index].clone()); + fields.push(child_schema.fields[index].clone()); } } // One row per (partition, item), not one row per partition. Preserve this @@ -990,7 +1039,7 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { Some(Schema { fields, unique_keys, - ..state.schema.clone() + ..state_schema.clone() }) } @@ -999,18 +1048,18 @@ fn topk_estimate_schema(state: &OperatorNode) -> Option { /// through `SummaryMerge`) is filtered, or, over a relation, is global (zero /// rows give one empty state) or summarizes a nullable column (a group of /// NULLs gives an empty state). -fn quantile_may_be_empty(state: &OperatorNode) -> bool { - match state.asap() { - Some(ASAPOp::SummaryMerge { +fn quantile_may_be_empty(state: &I) -> bool { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { // A global rollup must also represent the empty input population, // even when every existing grouped state is nonempty. (source_kind(state) == OperatorResultKind::Relation && group_by.as_ref().is_some_and(Vec::is_empty)) - || children.iter().any(|child| quantile_may_be_empty(child)) + || children.iter().any(quantile_may_be_empty) } - Some( + Operator::ASAP( ASAPOp::SummaryAgg { child, input, @@ -1029,15 +1078,15 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { if filter.is_some() { return true; } - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { return false; } let global = matches!(reduction, Reduction::Reduce(by) if by.keys().is_empty()); let nullable = match &input.weight { crate::ir::schema::sketch::SummaryInputExpr::Column(column) => { - crate::ir::scalar::resolve_column_ref(column, &child.schema) + crate::ir::scalar::resolve_column_ref(column, child.schema()) .ok() - .and_then(|index| child.schema.fields.get(index)) + .and_then(|index| child.schema().fields.get(index)) .is_some_and(|field| field.nullable) } _ => false, @@ -1045,6 +1094,6 @@ fn quantile_may_be_empty(state: &OperatorNode) -> bool { global || nullable } // Selecting whole states empties none of them. - _ => selected_from(state).is_some_and(|states| quantile_may_be_empty(states)), + _ => selected_from(state).is_some_and(|states| quantile_may_be_empty(&states)), } } diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs new file mode 100644 index 000000000..82a7127bb --- /dev/null +++ b/crates/types/src/ir/operator/input.rs @@ -0,0 +1,137 @@ +//! How the derivation rules read an operator's inputs: one trait, so the +//! same rules (output schema and kind, input validation, sharing, summary +//! coverage) derive a front-end node's schema and a memo group's +//! `GroupProperties` (concepts doc `memo-groups-properties-requirements.md`, +//! section 3). + +use std::rc::Rc; + +use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::non_asap::NonASAPOp; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::Schema; +use crate::ir::SchemaDerivationError; + +/// An input of an operator, as the rules read it. +pub trait OperatorInput: Sized + Clone { + fn schema(&self) -> &Schema; + fn result_kind(&self) -> OperatorResultKind; + /// The input's own operator, its children as inputs again. + fn operator(&self) -> Operator; + /// What the input covers when it is summary state in the memo; `None` + /// for any other input. + fn summary_coverage(&self) -> Option<&SummaryCoverage>; +} + +/// A front-end node tree. Node trees carry no summary coverage. +impl OperatorInput for Rc { + fn schema(&self) -> &Schema { + &self.schema + } + + fn result_kind(&self) -> OperatorResultKind { + self.result_kind + } + + fn operator(&self) -> Operator { + self.operator.clone() + } + + fn summary_coverage(&self) -> Option<&SummaryCoverage> { + None + } +} + +/// An input a rule builds while it derives (one build over a chain's +/// window, the build a rollup stands for, the build a selection of states +/// stands for): an operator over `I`s that is neither a node of a tree nor +/// a memo group. A rule never builds one over another rule's built input, +/// so the type does not nest. +#[derive(Clone)] +pub(crate) enum BuiltOrInput { + Input(I), + Built(Rc>), +} + +pub(crate) struct BuiltInput { + operator: Operator>, + schema: Schema, + result_kind: OperatorResultKind, +} + +impl BuiltOrInput { + /// A non-ASAP `operator` with its derived schema and kind. Only non-ASAP + /// rules derive it, which build no input themselves. + pub(crate) fn build( + operator: NonASAPOp>, + ) -> Result { + let schema = operator.output_schema()?; + Ok(Self::with_schema(Operator::NonASAP(operator), schema)) + } + + /// `operator` with `schema` and its derived kind. + pub(crate) fn with_schema(operator: Operator>, schema: Schema) -> Self { + let result_kind = operator.output_kind(); + Self::Built(Rc::new(BuiltInput { + operator, + schema, + result_kind, + })) + } +} + +impl OperatorInput for BuiltOrInput { + fn schema(&self) -> &Schema { + match self { + Self::Input(input) => input.schema(), + Self::Built(built) => &built.schema, + } + } + + fn result_kind(&self) -> OperatorResultKind { + match self { + Self::Input(input) => input.result_kind(), + Self::Built(built) => built.result_kind, + } + } + + fn operator(&self) -> Operator { + match self { + Self::Input(input) => input + .operator() + .map_children(|child| Self::Input(child.clone())), + Self::Built(built) => built.operator.clone(), + } + } + + fn summary_coverage(&self) -> Option<&SummaryCoverage> { + match self { + Self::Input(input) => input.summary_coverage(), + Self::Built(_) => None, + } + } +} + +impl BuiltOrInput> { + /// The node tree of this input: a built input as a new node over its + /// inputs' trees. + pub(crate) fn to_node(&self) -> Rc { + match self { + Self::Input(node) => Rc::clone(node), + Self::Built(built) => Rc::new(OperatorNode::with_schema( + built.operator.map_children(Self::to_node), + built.schema.clone(), + )), + } + } +} + +/// A build over node trees (`tumbling_window_chain`'s builds) as a node. +pub(crate) fn node_of_build( + (build, schema): (super::asap::ASAPOp>>, Schema), +) -> OperatorNode { + OperatorNode::with_schema( + Operator::ASAP(build.map_children(BuiltOrInput::to_node)), + schema, + ) +} diff --git a/crates/types/src/ir/operator/maintained_population.rs b/crates/types/src/ir/operator/maintained_population.rs index 570bebfa6..f11ab8682 100644 --- a/crates/types/src/ir/operator/maintained_population.rs +++ b/crates/types/src/ir/operator/maintained_population.rs @@ -2,6 +2,8 @@ //! Resource limits, ingestion placement and data structures belong to the executor. use serde::{Deserialize, Serialize}; +use crate::ir::operator::input::OperatorInput; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CurrentSeriesInput { pub metric: String, @@ -56,35 +58,35 @@ pub struct MaintainedPopulation { impl CurrentSeriesInput { /// Verify the named contract against the canonical maintenance input. - pub(crate) fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + pub(crate) fn matches_node(&self, input: &I) -> bool { use crate::ir::operator::Source; use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator, ScalarExpr, TimeRangeKind}; - let input = match &input.operator { + let input = match input.operator() { Operator::NonASAP(NonASAPOp::TimeRange { range, kind: TimeRangeKind::Instant, child, }) if self.lookback_ms > 0 - && *range == std::time::Duration::from_millis(self.lookback_ms) => + && range == std::time::Duration::from_millis(self.lookback_ms) => { - child.as_ref() + child } Operator::NonASAP(NonASAPOp::TimeRange { .. }) => return false, - _ if self.lookback_ms == 300_000 => input, + _ if self.lookback_ms == 300_000 => input.clone(), _ => return false, }; let Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { metric }, predicates, schema, - }) = &input.operator + }) = input.operator() else { return false; }; if self.metric.is_empty() - || *metric != self.metric + || metric != self.metric || (schema.closed && !schema.has_promql_series_identity()) || schema.time_index.is_none() { @@ -99,7 +101,7 @@ impl CurrentSeriesInput { return false; } let mut matchers = Vec::new(); - for predicate in predicates { + for predicate in &predicates { let ScalarExpr::Compare { left, op, right, .. } = &predicate.0 @@ -138,7 +140,7 @@ impl CurrentSeriesInput { impl MaintainedPopulation { /// Whether `input` is the maintenance input this population declares. - pub(crate) fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { + pub(crate) fn matches_node(&self, input: &I) -> bool { use crate::ir::operator::Source; use crate::ir::schema::DataType; use crate::ir::{NonASAPOp, Operator}; @@ -149,18 +151,21 @@ impl MaintainedPopulation { value_column, grouping, } => { + let operator = input.operator(); let Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { .. }, schema, .. - }) = &input.operator + }) = &operator else { return false; }; // The same computation, whatever accuracy or timing has - // been attached to the node since. - let same_source = - expected.operator == input.operator && expected.schema == input.schema; + // been attached to the node since. A `Scan` has no input, + // so its operator without inputs is all of it. + let same_source = expected.operator.map_children(|_| ()) + == operator.map_children(|_| ()) + && expected.schema == *input.schema(); same_source && schema.closed && schema diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index f7baca7a9..8b0d5ce2e 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -4,6 +4,8 @@ //! - [`node`] — [`OperatorNode`](node::OperatorNode): an operator plus the //! planning properties every traversal needs. //! - [`non_asap`] / [`asap`] — ordinary query operators and ASAP operators. +//! - [`input`] — how the derivation rules read an operator's inputs +//! ([`OperatorInput`](input::OperatorInput)). //! - [`operator_properties`] — operator parameter types ([`GroupKeys`], //! [`Reduction`], [`Source`], …). //! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). @@ -12,6 +14,7 @@ //! - [`query`] — a query root: an operator result or a standalone scalar. pub mod agg_intent; pub mod asap; +pub mod input; pub mod maintained_population; pub mod node; pub mod non_asap; diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index fbadb664d..956a85dc6 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -11,9 +11,9 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; +use super::input::OperatorInput; use super::non_asap::NonASAPOp; use crate::ir::properties::guarantee::AccuracyGuarantee; -use crate::ir::properties::summary_coverage::{NodeToSummaryCoverageCache, SummaryCoverage}; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -62,7 +62,9 @@ impl Operator { } } -impl Operator { +/// The derivation rules, one set for every kind of input +/// ([`OperatorInput`]): front-end node trees and memo groups. +impl Operator { pub(crate) fn output_schema(&self) -> Result { match self { Operator::NonASAP(op) => op.output_schema(), @@ -83,6 +85,16 @@ impl Operator { Operator::ASAP(op) => op.validate_inputs(), } } + + /// Whether two equal copies of this operator over the same inputs return + /// the same rows, so they may be one group: an ASAP operator, or a + /// non-ASAP one [`NonASAPOp::is_shareable_when_equal`]. + pub(crate) fn is_shareable_when_equal(&self) -> bool { + match self { + Operator::NonASAP(op) => op.is_shareable_when_equal(), + Operator::ASAP(_) => true, + } + } } /// A node of the logical DAG. Nodes are immutable and shared through `Rc`; @@ -106,9 +118,7 @@ impl OperatorNode { /// ASAP operator, ...). pub fn new(operator: Operator) -> Result { let schema = operator.output_schema()?; - let node = Self::with_schema(operator, schema); - node.check_merge(&mut NodeToSummaryCoverageCache::new())?; - Ok(node) + Ok(Self::with_schema(operator, schema)) } /// Build a node with caller-supplied output names and qualifiers. For @@ -136,45 +146,10 @@ impl OperatorNode { self } - /// What this summary state covers; `None` for a node that is not a - /// `SummaryAgg` or a valid `SummaryMerge`. Derived on each call, since - /// the public fields can be edited in place. - #[cfg(any(test, feature = "test-support"))] - pub fn coverage(&self) -> Option { - SummaryCoverage::derive(self).ok() - } - - /// The build whose state this summary-state node carries: a - /// `SummaryAgg` or `SummaryTumblingWindowChain` itself, or the first input's build under - /// `SummaryMerge`s (merged inputs have one definition, which - /// `coverage` checks). `None` under a regrouping - /// merge, whose states no build in the DAG carries (its coverage's - /// `definition` is the build it stands for), and for any other node. - pub fn summary_build(&self) -> Option<&ASAPOp> { - match self.asap()? { - op @ (ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) => { - Some(op) - } - ASAPOp::SummaryMerge { - children, - group_by: None, - .. - } => children.first()?.summary_build(), - _ => None, - } - } - - /// A `SummaryMerge` is valid only over inputs with the same definition - /// and disjoint selections. `cache` holds the coverage already derived - /// for nodes of the DAG being validated. - fn check_merge( - &self, - cache: &mut NodeToSummaryCoverageCache, - ) -> Result<(), SchemaDerivationError> { - if matches!(self.asap(), Some(ASAPOp::SummaryMerge { .. })) { - SummaryCoverage::derive_cached(self, cache)?; - } - Ok(()) + /// The build whose state this summary-state node carries + /// ([`summary_build`](super::asap::summary_build)). + pub fn summary_build(self: &Rc) -> Option { + super::asap::summary_build(self) } pub fn non_asap(&self) -> Option<&NonASAPOp> { @@ -289,9 +264,6 @@ impl OperatorNode { /// derived from the operator. Both operator categories may override field /// names and qualifiers; all structural metadata must match derivation. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { - // Each summary state's coverage is derived once, not once per merge - // above it; `reachable` keeps every node alive while it is used. - let mut coverage = NodeToSummaryCoverageCache::new(); for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { node.schema.fields.get(i).is_none_or(|f| { @@ -309,7 +281,6 @@ impl OperatorNode { )); } node.operator.validate_inputs()?; - node.check_merge(&mut coverage)?; if node.result_kind != node.operator.output_kind() { return Err(SchemaDerivationError::InvalidScalarSignature( "retained result kind disagrees with operation".into(), diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 7eed22fda..dd1755456 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -6,7 +6,8 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; -use super::node::{OperatorNode, OperatorResultKind}; +use super::input::OperatorInput; +use super::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::agg_intent::AggIntent; use crate::ir::operator::operator_properties::{ BinaryOpKind, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, @@ -491,7 +492,7 @@ impl NonASAPOp { } } -impl NonASAPOp { +impl NonASAPOp { /// Whether two structurally equal copies of this node may be one shared /// node: every execution over the same input rows returns the same /// output rows, so one copy over their one input can be computed (or @@ -514,7 +515,7 @@ impl NonASAPOp { /// function the IR accepts is deterministic (no `random()`; unregistered /// SQL functions are rejected) and that a deployment's /// `AggIntent::Extension` returns the same rows for the same input. - pub fn is_shareable_when_equal(&self) -> bool { + pub(crate) fn is_shareable_when_equal(&self) -> bool { use NonASAPOp::*; match self { Limit { @@ -524,14 +525,14 @@ impl NonASAPOp { child, } => { (n.is_none() && *offset == 0) - || matches!(child.non_asap(), Some(Sort { partition_by: sorted_by, .. }) - if sorted_by == partition_by) + || matches!(child.operator(), Operator::NonASAP(Sort { partition_by: sorted_by, .. }) + if sorted_by == *partition_by) } // Empty `cols` means every column. Dedup { cols, child } => { cols.is_empty() || cols.iter().collect::>().len() - == child.schema.fields.len() + == child.schema().fields.len() } SQLWindowFunc { func, @@ -610,7 +611,7 @@ impl NonASAPOp { .output_schema(); } let mut output = - aggregate_output_schema(&child.schema, reduction, measures, output_names)?; + aggregate_output_schema(child.schema(), reduction, measures, output_names)?; let offset = reduction.group_keys().map_or(0, |keys| keys.len()); // A filtered quantile is NULL for a group without a matching // row, as its summary estimate is. @@ -625,14 +626,14 @@ impl NonASAPOp { } // Over a relation, a quantile is also NULL for a global // aggregate of no row and a group whose values are all NULL. - if child.result_kind == OperatorResultKind::Relation { + if child.result_kind() == OperatorResultKind::Relation { for (index, measure) in measures.iter().enumerate() { if matches!(measure, AggIntent::Quantile { .. }) && (offset == 0 || measure .input_cols() .iter() - .any(|i| child.schema.fields[*i].nullable)) + .any(|i| child.schema().fields[*i].nullable)) { if let Some(field) = output.fields.get_mut(offset + index) { field.nullable = true; @@ -640,7 +641,7 @@ impl NonASAPOp { } } } - if child.result_kind == OperatorResultKind::Relation { + if child.result_kind() == OperatorResultKind::Relation { for (index, measure) in measures.iter().enumerate() { if matches!( measure, @@ -654,7 +655,7 @@ impl NonASAPOp { || measure .input_cols() .iter() - .any(|i| child.schema.fields[*i].nullable); + .any(|i| child.schema().fields[*i].nullable); if let Some(field) = output.fields.get_mut(offset + index) { field.nullable = nullable; } @@ -671,13 +672,13 @@ impl NonASAPOp { | PromqlSeriesSample { child, .. } | PromqlInfoEnrich { child, .. } | TimeRange { child, .. } - | TimeShift { child, .. } => child.schema.clone(), + | TimeShift { child, .. } => child.schema().clone(), // ρ — relabel preserves every input column and writes one label // `dst` (Utf8): overwritten in place if it already exists, else // appended (nullable). Row-uniqueness is no longer provable. PromqlRelabel { dst, child, .. } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); if let Some(existing) = out.fields.iter_mut().find(|c| c.name == *dst) { existing.dtype = FieldDataType::Plain(DataType::Utf8); existing.nullable = true; @@ -698,7 +699,7 @@ impl NonASAPOp { qualifier, child, } => { - let in_schema = &child.schema; + let in_schema = child.schema(); let fields: Vec = cols .iter() .enumerate() @@ -747,7 +748,7 @@ impl NonASAPOp { } Dedup { cols, child } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); if !cols.is_empty() { out.add_unique_key(cols.clone()); } @@ -758,12 +759,12 @@ impl NonASAPOp { let mut s = children .first() .ok_or(SchemaDerivationError::EmptyConcat)? - .schema + .schema() .clone(); // A column is nullable if any child's is: a rolled-up // grouping key is NULL only in the coarser levels. for child in &children[1..] { - for (field, other) in s.fields.iter_mut().zip(&child.schema.fields) { + for (field, other) in s.fields.iter_mut().zip(&child.schema().fields) { field.nullable |= other.nullable; } } @@ -771,7 +772,7 @@ impl NonASAPOp { s } SetOp { left, .. } => { - let mut s = left.schema.clone(); + let mut s = left.schema().clone(); s.unique_keys.clear(); s } @@ -781,8 +782,8 @@ impl NonASAPOp { left, right, } => { - let l = &left.schema; - let r = &right.schema; + let l = left.schema(); + let r = right.schema(); if matches!(kind, JoinKind::Semi | JoinKind::Anti) { return Ok(Schema { unique_keys: Vec::new(), @@ -825,7 +826,7 @@ impl NonASAPOp { child, .. } => { - let mut out = child.schema.clone(); + let mut out = child.schema().clone(); let arg = args.first().map(|a| a.scalar_type(&out)).transpose()?; let arg_dtype = || { arg.as_ref() @@ -871,7 +872,7 @@ impl NonASAPOp { BinaryOp { lhs, rhs, operator, .. } => { - let mut output = lhs.schema.clone(); + let mut output = lhs.schema().clone(); let grouping = operator .vector_match .as_ref() @@ -883,7 +884,7 @@ impl NonASAPOp { let mut additions = Vec::new(); if right_rows { additions.extend( - rhs.schema + rhs.schema() .fields .iter() .filter(|c| c.plain_dtype() == Some(&DataType::Utf8)) @@ -917,7 +918,7 @@ impl NonASAPOp { Source::Table { .. } => OperatorResultKind::Relation, }, Values { .. } | SQLWindowFunc { .. } => OperatorResultKind::Relation, - TimeRange { child, .. } if child.result_kind == OperatorResultKind::Relation => { + TimeRange { child, .. } if child.result_kind() == OperatorResultKind::Relation => { OperatorResultKind::Relation } TimeRange { kind, .. } => match kind { @@ -931,12 +932,12 @@ impl NonASAPOp { // category (a SQL GROUP BY stays a relation). Aggregate { reduction, child, .. - } => match (reduction, child.result_kind) { + } => match (reduction, child.result_kind()) { (Reduction::PerEntity, OperatorResultKind::RangeVector) | (_, OperatorResultKind::State) => OperatorResultKind::InstantVector, (_, kind) => kind, }, - Project { child, cols, .. } if cols.iter().any(|item| matches!(item.expr, ScalarExpr::Column(i) if child.schema.fields.get(i).is_some_and(|f| !f.is_plain()))) => OperatorResultKind::State, + Project { child, cols, .. } if cols.iter().any(|item| matches!(item.expr, ScalarExpr::Column(i) if child.schema().fields.get(i).is_some_and(|f| !f.is_plain()))) => OperatorResultKind::State, // Selecting whole states by their plain (key) columns keeps them states. Filter { child, pred } if selects_states(child, pred) => OperatorResultKind::State, Filter { child, .. } @@ -947,12 +948,12 @@ impl NonASAPOp { | TimeShift { child, .. } | PromqlRelabel { child, .. } | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } => readable(child.result_kind), - Join { left, .. } | SetOp { left, .. } => readable(left.result_kind), + | PromqlSeriesSample { child, .. } => readable(child.result_kind()), + Join { left, .. } | SetOp { left, .. } => readable(left.result_kind()), Concat { children, .. } => children .first() - .map_or(OperatorResultKind::Relation, |c| readable(c.result_kind)), - BinaryOp { lhs, .. } => readable(lhs.result_kind), + .map_or(OperatorResultKind::Relation, |c| readable(c.result_kind())), + BinaryOp { lhs, .. } => readable(lhs.result_kind()), } } @@ -960,8 +961,8 @@ impl NonASAPOp { /// and its children's output categories. pub(crate) fn validate_inputs(&self) -> Result<(), SchemaDerivationError> { use NonASAPOp::*; - let no_state = |node: &OperatorNode, what: &str| { - if node.result_kind == OperatorResultKind::State { + let no_state = |node: &I, what: &str| { + if node.result_kind() == OperatorResultKind::State { Err(SchemaDerivationError::InvalidScalarSignature(format!( "{what} consumes summary state; read it out first" ))) @@ -970,18 +971,19 @@ impl NonASAPOp { } }; let invalid = |message: &str| SchemaDerivationError::InvalidScalarSignature(message.into()); - let predicate = |pred: &Predicate, scope: &Schema| -> Result<(), SchemaDerivationError> { - if matches!( - pred.0.scalar_type(scope)?.0, - DataType::Bool | DataType::Null - ) { - Ok(()) - } else { - Err(invalid("predicate must be boolean")) - } - }; - let instant = |child: &OperatorNode| -> Result<(), SchemaDerivationError> { - if child.result_kind == OperatorResultKind::InstantVector { + let predicate = + |pred: &Predicate, scope: &Schema| -> Result<(), SchemaDerivationError> { + if matches!( + pred.0.scalar_type(scope)?.0, + DataType::Bool | DataType::Null + ) { + Ok(()) + } else { + Err(invalid("predicate must be boolean")) + } + }; + let instant = |child: &I| -> Result<(), SchemaDerivationError> { + if child.result_kind() == OperatorResultKind::InstantVector { Ok(()) } else { Err(invalid("operation requires an instant vector")) @@ -1025,14 +1027,14 @@ impl NonASAPOp { if !selects_states(child, pred) { no_state(child, "Filter")?; } - predicate(pred, &child.schema)?; + predicate(pred, child.schema())?; } Project { child, cols, .. } => { for col in cols { if let ScalarExpr::Column(index) = col.expr { - columns(&[index], &child.schema)?; + columns(&[index], child.schema())?; } else { - col.expr.scalar_type(&child.schema)?; + col.expr.scalar_type(child.schema())?; } } } @@ -1044,8 +1046,8 @@ impl NonASAPOp { } => { no_state(lhs, "BinaryOp")?; no_state(rhs, "BinaryOp")?; - if lhs.result_kind != rhs.result_kind - || lhs.result_kind == OperatorResultKind::RangeVector + if lhs.result_kind() != rhs.result_kind() + || lhs.result_kind() == OperatorResultKind::RangeVector { return Err(invalid("binary operands have incompatible result kinds")); } @@ -1058,16 +1060,16 @@ impl NonASAPOp { } => { no_state(left, "Join")?; no_state(right, "Join")?; - if left.result_kind != OperatorResultKind::Relation - || right.result_kind != OperatorResultKind::Relation + if left.result_kind() != OperatorResultKind::Relation + || right.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL join requires relations")); } let scope = Schema::new( - left.schema + left.schema() .fields .iter() - .chain(&right.schema.fields) + .chain(&right.schema().fields) .cloned() .collect(), ); @@ -1076,8 +1078,8 @@ impl NonASAPOp { SetOp { left, right, .. } => { no_state(left, "SetOp")?; no_state(right, "SetOp")?; - if left.result_kind != OperatorResultKind::Relation - || right.result_kind != OperatorResultKind::Relation + if left.result_kind() != OperatorResultKind::Relation + || right.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL set operation requires relations")); } @@ -1097,16 +1099,16 @@ impl NonASAPOp { } => { no_state(child, "Aggregate")?; if let Reduction::Reduce(keys) = reduction { - columns(keys.keys(), &child.schema)?; + columns(keys.keys(), child.schema())?; } for measure in measures { - columns(&measure.input_cols(), &child.schema)?; + columns(&measure.input_cols(), child.schema())?; } if !filters.is_empty() && filters.len() != measures.len() { return Err(invalid("aggregate filter count differs from measure count")); } for pred in filters.iter().flatten() { - predicate(pred, &child.schema)?; + predicate(pred, child.schema())?; } if let Some(pred) = having { predicate(pred, &self.output_schema()?)?; @@ -1114,23 +1116,23 @@ impl NonASAPOp { } Dedup { child, cols } => { no_state(child, "Dedup")?; - columns(cols, &child.schema)?; + columns(cols, child.schema())?; } Sort { child, keys, partition_by, } => { - columns(partition_by.keys(), &child.schema)?; + columns(partition_by.keys(), child.schema())?; for key in keys { - key.expr.scalar_type(&child.schema)?; + key.expr.scalar_type(child.schema())?; } } Limit { child, partition_by, .. - } => columns(partition_by.keys(), &child.schema)?, + } => columns(partition_by.keys(), child.schema())?, SQLWindowFunc { child, args, @@ -1138,12 +1140,12 @@ impl NonASAPOp { partition_by, .. } => { - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { return Err(invalid("SQL window requires a relation")); } - columns(partition_by.keys(), &child.schema)?; + columns(partition_by.keys(), child.schema())?; for expr in args.iter().chain(order_by.iter().map(|k| &k.expr)) { - expr.scalar_type(&child.schema)?; + expr.scalar_type(child.schema())?; } } PromqlVectorFromScalar(expr) => { @@ -1156,12 +1158,12 @@ impl NonASAPOp { | PromqlSeriesSample { child, .. } => instant(child)?, PromqlRelabel { child, value, .. } => { instant(child)?; - if value.scalar_type(&child.schema)?.0 != DataType::Utf8 { + if value.scalar_type(child.schema())?.0 != DataType::Utf8 { return Err(invalid("label expression requires a string")); } } TimeRange { child, .. } => { - if child.result_kind != OperatorResultKind::Relation { + if child.result_kind() != OperatorResultKind::Relation { instant(child)?; } } @@ -1169,7 +1171,7 @@ impl NonASAPOp { // A relation is a SQL table's rows, shifted under a trailing // window's tumbling window. if !matches!( - child.result_kind, + child.result_kind(), OperatorResultKind::InstantVector | OperatorResultKind::RangeVector | OperatorResultKind::Relation @@ -1187,13 +1189,13 @@ impl NonASAPOp { /// Each input retains its keys only when the equality predicates constrain /// the other input to at most one matching row. In particular this preserves /// grouped sum/count keys through the SQL-average implementation. -fn inner_join_unique_keys( - predicate: &ScalarExpr, +fn inner_join_unique_keys( + predicate: &ScalarExpr, left: &Schema, right: &Schema, ) -> Vec> { use crate::ir::scalar::CompareOpKind; - fn columns(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { + fn columns(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { if let ScalarExpr::Compare { left, op: CompareOpKind::Eq, @@ -1208,7 +1210,7 @@ fn inner_join_unique_keys( } None } - fn equality(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { + fn equality(expr: &ScalarExpr) -> Option<(ColumnId, ColumnId)> { if let Some(pair) = columns(expr) { return Some(pair); } @@ -1241,7 +1243,7 @@ fn inner_join_unique_keys( } None } - fn collect(expr: &ScalarExpr, pairs: &mut Vec<(ColumnId, ColumnId)>) { + fn collect(expr: &ScalarExpr, pairs: &mut Vec<(ColumnId, ColumnId)>) { if let ScalarExpr::BoolAnd(terms) = expr { for term in terms { collect(term, pairs); @@ -1289,11 +1291,11 @@ fn inner_join_unique_keys( /// Whether a `Filter` of `pred` over `child` selects whole summary states: /// `child` is state and `pred` reads only its plain columns, such as group keys. -fn selects_states(child: &OperatorNode, pred: &Predicate) -> bool { - child.result_kind == OperatorResultKind::State +fn selects_states(child: &I, pred: &Predicate) -> bool { + child.result_kind() == OperatorResultKind::State && pred.0.columns_referenced().iter().all(|&column| { child - .schema + .schema() .fields .get(column) .is_some_and(|field| field.is_plain()) @@ -1309,7 +1311,11 @@ fn readable(kind: OperatorResultKind) -> OperatorResultKind { /// Default output-column name for a projection item with no explicit alias: /// a bare column keeps its (schema) name; anything else gets `col_{i}`. -fn default_proj_name(expr: &ScalarExpr, idx: usize, schema: &Schema) -> String { +fn default_proj_name( + expr: &ScalarExpr, + idx: usize, + schema: &Schema, +) -> String { match expr { ScalarExpr::Column(id) => schema .fields @@ -1793,7 +1799,7 @@ mod tests { unique_keys: vec![vec![0, 1]], closed: true, }; - let equal = |left, right| ScalarExpr::Compare { + let equal = |left, right| ScalarExpr::>::Compare { left: Box::new(ScalarExpr::Column(left)), op: CompareOpKind::Eq, right: Box::new(ScalarExpr::Column(right)), diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index a0974bf3e..d4a611915 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -8,7 +8,7 @@ //! the family merges (the family-capability table), and the executor runs //! the chain as its tumbling windows (`expand_tumbling_window_chains`). A merge of only the //! newest `m < n` tumbling windows (`SummaryMerge::newest_tumbling_windows`) is the state of one -//! build over `TimeRange(m·w)` ([`merged_window_build`]): one chain materialized for +//! build over `TimeRange(m·w)` (`merged_window_build`): one chain materialized for //! the longest window serves every query whose window, on the same tumbling windows, //! ends at the same time and is shorter. #[cfg(any(test, feature = "test-support"))] @@ -16,36 +16,39 @@ use std::collections::HashMap; use std::rc::Rc; use std::time::Duration; -use super::asap::ASAPOp; +use super::asap::{summary_agg_schema, ASAPOp}; +use super::input::node_of_build; +use super::input::{BuiltOrInput, OperatorInput}; use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; #[cfg(any(test, feature = "test-support"))] use super::operator_properties::TimeShift; +use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; fn invalid(message: &str) -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(message.into()) } -/// The `SummaryTumblingWindowChain` a summary-state node carries merged: the chain itself, +/// The `SummaryTumblingWindowChain` a summary state carries merged: the chain itself, /// or the one chain a `SummaryMerge` that keeps its groups reads, in full -/// or its newest tumbling windows (`merged_tumbling_window_count`). `None` for any other node, +/// or its newest tumbling windows (`merged_tumbling_window_count`). `None` for any other input, /// a rollup (regrouping merge) included. -pub fn merged_chain(state: &OperatorNode) -> Option<&OperatorNode> { - match state.asap()? { - ASAPOp::SummaryTumblingWindowChain { .. } => Some(state), - ASAPOp::SummaryMerge { +pub fn merged_chain(state: &I) -> Option { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: None, .. - } => match children.as_slice() { + }) => match children.as_slice() { [child] if matches!( - child.asap(), - Some(ASAPOp::SummaryTumblingWindowChain { .. }) + child.operator(), + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ) => { - Some(child) + Some(child.clone()) } _ => None, }, @@ -53,45 +56,51 @@ pub fn merged_chain(state: &OperatorNode) -> Option<&OperatorNode> { } } -/// How many of its chain's tumbling windows, newest first, a summary-state node with +/// How many of its chain's tumbling windows, newest first, a summary state with /// a [`merged_chain`] merges: the merge's `newest_tumbling_windows`, otherwise the -/// chain's `tumbling_window_count`. `None` for any other node. -fn merged_tumbling_window_count(state: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryTumblingWindowChain { +/// chain's `tumbling_window_count`. `None` for any other input. +fn merged_tumbling_window_count(state: &I) -> Option { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_count, .. - }) = merged_chain(state)?.asap() + }) = merged_chain(state)?.operator() else { return None; }; - match state.asap()? { - ASAPOp::SummaryMerge { + match state.operator() { + Operator::ASAP(ASAPOp::SummaryMerge { newest_tumbling_windows: Some(newest), .. - } => Some(*newest), - _ => Some(*tumbling_window_count), + }) => Some(newest), + _ => Some(tumbling_window_count), } } +/// A build over the inputs of a chain over `I`, with its schema. +pub(crate) type BuildWithSchema = (ASAPOp>, Schema); + /// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the /// chain's input whose state the merged tumbling windows of `chain` equal. -pub(crate) fn whole_window_build( - chain: &OperatorNode, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +#[cfg(any(test, feature = "test-support"))] +pub(crate) fn whole_window_build( + chain: &I, +) -> Result, SchemaDerivationError> { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_count, .. - }) = chain.asap() + }) = chain.operator() else { return Err(invalid("expected a tumbling-window chain")); }; - newest_tumbling_windows_build(chain, *tumbling_window_count) + newest_tumbling_windows_build(chain, tumbling_window_count) } -/// The `SummaryAgg` whose state a summary-state node with a +/// The `SummaryAgg` whose state a summary state with a /// [`merged_chain`] carries: one build over the window of the tumbling windows it /// merges (`merged_tumbling_window_count`). -pub fn merged_window_build(state: &OperatorNode) -> Result { +pub(crate) fn merged_window_build( + state: &I, +) -> Result, SchemaDerivationError> { let (Some(chain), Some(tumbling_window_count)) = (merged_chain(state), merged_tumbling_window_count(state)) else { @@ -99,65 +108,94 @@ pub fn merged_window_build(state: &OperatorNode) -> Result, +) -> Result { + merged_window_build(state).map(node_of_build) } /// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the /// chain's input: the state of the chain's newest `tumbling_window_count` tumbling windows. -fn newest_tumbling_windows_build( - chain: &OperatorNode, +fn newest_tumbling_windows_build( + chain: &I, tumbling_window_count: u32, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +) -> Result, SchemaDerivationError> { + let Operator::ASAP(chain_op) = chain.operator() else { + return Err(invalid("expected a tumbling-window chain")); + }; + named_as_chain( + window_build(&chain_op, tumbling_window_count)?, + chain.schema(), + ) +} + +/// [`newest_tumbling_windows_build`] of the chain operator `chain`, without +/// its schema. +pub(crate) fn window_build( + chain: &ASAPOp, + tumbling_window_count: u32, +) -> Result>, SchemaDerivationError> { + let ASAPOp::SummaryTumblingWindowChain { child, tumbling_window_width_ms, .. - }) = chain.asap() + } = chain else { return Err(invalid("expected a tumbling-window chain")); }; let window = tumbling_window_width_ms .checked_mul(u64::from(tumbling_window_count)) .ok_or_else(|| invalid("tumbling-window chain window overflows"))?; - build_over(chain, range(Rc::clone(child), window)?) + build_over(chain, range(BuiltOrInput::Input(child.clone()), window)?) } #[cfg(any(test, feature = "test-support"))] /// Tumbling window `index` (0 = newest) of `chain`: its `SummaryAgg` over the range /// `TimeRange(tumbling_window_width_ms)` ending `index · tumbling_window_width_ms` before the /// chain's input. -pub(crate) fn tumbling_window( - chain: &OperatorNode, +pub(crate) fn tumbling_window( + chain: &I, index: u32, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +) -> Result, SchemaDerivationError> { + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child, tumbling_window_width_ms, .. - }) = chain.asap() + }) = chain.operator() else { return Err(invalid("expected a tumbling-window chain")); }; let back = i64::try_from(tumbling_window_width_ms.saturating_mul(u64::from(index))) .map_err(|_| invalid("tumbling window offset overflows"))?; - build_over( - chain, - range(shifted(child, back)?, *tumbling_window_width_ms)?, + let Operator::ASAP(chain_op) = chain.operator() else { + return Err(invalid("expected a tumbling-window chain")); + }; + named_as_chain( + build_over( + &chain_op, + range(shifted(child, back)?, tumbling_window_width_ms)?, + )?, + chain.schema(), ) } #[cfg(any(test, feature = "test-support"))] /// `child` evaluated `back_ms` earlier: one `TimeShift`, folded into a /// `TimeShift` directly below. -fn shifted( - child: &Rc, +fn shifted( + child: I, back_ms: i64, -) -> Result, SchemaDerivationError> { +) -> Result, SchemaDerivationError> { if back_ms == 0 { - return Ok(Rc::clone(child)); + return Ok(BuiltOrInput::Input(child)); } - let (shift, input) = match child.non_asap() { - Some(NonASAPOp::TimeShift { + let (shift, input) = match child.operator() { + Operator::NonASAP(NonASAPOp::TimeShift { shift, child: input, }) => ( @@ -178,52 +216,70 @@ fn shifted( child, ), }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { + BuiltOrInput::build(NonASAPOp::TimeShift { shift, - child: Rc::clone(input), - })) + child: BuiltOrInput::Input(input), + }) } -fn range( - child: Rc, +/// A range `TimeRange(width_ms)` over `child`. +fn range( + child: BuiltOrInput, width_ms: u64, -) -> Result, SchemaDerivationError> { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { +) -> Result, SchemaDerivationError> { + BuiltOrInput::build(NonASAPOp::TimeRange { range: Duration::from_millis(width_ms), kind: TimeRangeKind::Range, child, - })) + }) } -/// The chain's `SummaryAgg` over `input`, with the chain's column names. -fn build_over( - chain: &OperatorNode, - input: Rc, -) -> Result { - let Some(ASAPOp::SummaryTumblingWindowChain { +/// The chain's `SummaryAgg` over `input`. +fn build_over( + chain: &ASAPOp, + input: BuiltOrInput, +) -> Result>, SchemaDerivationError> { + let ASAPOp::SummaryTumblingWindowChain { family, input: update, reduction, grouping, filter, .. - }) = chain.asap() + } = chain.map_children(|child| BuiltOrInput::Input(child.clone())) else { return Err(invalid("expected a tumbling-window chain")); }; - let mut build = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { + Ok(ASAPOp::SummaryAgg { child: input, - family: family.clone(), - input: update.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }))?; - for (field, kept) in build.schema.fields.iter_mut().zip(&chain.schema.fields) { + family, + input: update, + reduction, + grouping, + filter, + }) +} + +/// `build` with its schema, named as the chain of schema `chain`. +fn named_as_chain( + build: ASAPOp>, + chain: &Schema, +) -> Result, SchemaDerivationError> { + let ASAPOp::SummaryAgg { + child, + family, + reduction, + .. + } = &build + else { + return Err(invalid("expected a summary build")); + }; + let mut schema = summary_agg_schema(child.schema(), family, reduction)?; + for (field, kept) in schema.fields.iter_mut().zip(&chain.fields) { field.name = kept.name.clone(); field.table = kept.table.clone(); } - Ok(build) + Ok((build, schema)) } /// `root` with every `SummaryTumblingWindowChain` expanded into its tumbling windows: a `SummaryMerge` @@ -284,7 +340,8 @@ fn expand( inputs.push(Rc::clone(done)); continue; } - let tumbling_window = Rc::new(tumbling_window(child, index)?); + let tumbling_window = + Rc::new(node_of_build(tumbling_window(child, index)?)); if index == 0 { cache.chains.push(Rc::clone(child)); } diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index f371a9931..a6d195844 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,47 +3,67 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived from the node, never declared. Walking down from a -//! `SummaryAgg`, a predicate conjunct moves into `selection` when it can be -//! lifted to the `SummaryAgg` (through `Filter`, a range `TimeRange`, a -//! `TimeShift` without `@` and direct-column `Project` items) and it is a -//! value set or an interval on one column. Everything else stays in -//! `definition` as a residual, so two states are merged only when they -//! compute the same thing over disjoint rows. +//! Coverage is derived once for each summary-state memo group, from its +//! input groups, never declared, and stored in its `GroupProperties`. +//! Walking down from a `SummaryAgg` through the input groups' expressions, a +//! predicate conjunct moves into `selection` when it can be lifted to the +//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without +//! `@` and direct-column `Project` items) and it is a value set or an +//! interval on one column. Everything else stays in `definition` as a +//! residual, so two states are merged only when they compute the same thing +//! over disjoint rows. Node trees carry no coverage. use std::cmp::Ordering; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::BTreeMap; use std::ops::Bound; -use std::rc::Rc; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::ir::operator::asap::ASAPOp; -use crate::ir::operator::node::{Operator, OperatorNode}; +use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; +use crate::ir::operator::input::{BuiltOrInput, OperatorInput}; +use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::operator::operator_properties::Reduction; +use crate::ir::operator::tumbling_window_chain::window_build; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; use crate::ir::schema::sketch::GroupingStrategy; use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::SchemaDerivationError; +use crate::stages::{MemoGroupAsInput, MemoGroupId}; + +/// What a summary state computes, with every lifted selection removed. +#[derive(Debug, Clone, PartialEq)] +pub struct SummaryStateDefinition { + /// The group the unlifted operators read: the first input under the + /// SummaryAgg that is not Filter / range TimeRange / TimeShift without @ + /// / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator, + /// so states over one source with different matchers may merge. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments; `()` marks its + /// single input (the next one, or input_group). Two states may merge only + /// if input_group and these operators are equal. + pub unlifted_operators: Vec>, +} #[derive(Debug, Clone, PartialEq)] pub struct SummaryCoverage { - /// The `SummaryAgg` with the selection removed from its sub-DAG. - pub(crate) definition: Rc, - /// Union of boxes over the output rows of the definition's child. + pub(crate) definition: SummaryStateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. pub(crate) selection: Vec, } #[cfg(any(test, feature = "test-support"))] impl SummaryCoverage { - /// The `SummaryAgg` with the selection removed from its sub-DAG. - pub fn definition(&self) -> &Rc { + pub fn definition(&self) -> &SummaryStateDefinition { &self.definition } - /// Union of boxes over the output rows of the definition's child. + /// Union of boxes over the output rows of the unlifted operators' input. pub fn selection(&self) -> &[SelectionBox] { &self.selection } @@ -256,121 +276,134 @@ pub enum CoverageError { /// nothing, and a Hydra's groups share one state. Bad input. #[error("the inputs' family cannot merge different groups")] FamilyCannotRegroup, + /// A `SummaryAgg` whose filter reads a scalar subquery: its definition + /// cannot name the group the subquery reads, so it merges with nothing. + #[error("a summary filter with a scalar subquery has no comparable definition")] + SubqueryInDefinition, /// The build a regrouping `SummaryMerge` stands for does not derive: /// bad input. #[error("the regrouped build does not derive: {reason}")] RegroupedBuild { reason: String }, } -/// Coverage already derived during one validation, by node address. The -/// nodes must stay alive while it is in use. -pub(crate) type NodeToSummaryCoverageCache = HashMap<*const OperatorNode, SummaryCoverage>; +/// A memo group read as an input, and the inputs a rule builds over such +/// groups. +type GroupInput<'m> = BuiltOrInput>; impl SummaryCoverage { - /// Coverage of a `SummaryAgg`, `SummaryTumblingWindowChain` or `SummaryMerge`. A chain - /// covers what one build over its whole window does, and a merge of its - /// newest tumbling windows what one build over their window does. A merge fails unless - /// every input has the same definition and their selections are + /// Coverage of a summary state a memo group computes with `operator` + /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, + /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped + /// `SummaryAgg`. A chain covers what one build over its whole window + /// does, and a merge of its newest tumbling windows what one build over + /// their window does. A merge reads its inputs' stored coverage and fails + /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent - /// ([`MergeInputRule::MayOverlap`] in the family-capability - /// table) may also merge overlapping selections, and a counter family + /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may + /// also merge overlapping selections, and a counter family /// ([`MergeInputRule::DisjointInTime`]) needs disjoint time windows. - pub(crate) fn derive(node: &OperatorNode) -> Result { - Self::derive_cached(node, &mut NodeToSummaryCoverageCache::new()) - } - - /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, - /// of its sub-DAG's operators, output kinds and schemas, children - /// first, without planning metadata (`timing`, `guarantee`, - /// `lifetime`). Two equal definitions (the equality a merge - /// requires) get equal ids on any node and in any process, so a summary - /// store can key states by this id and a concrete selection. - pub(crate) fn definition_id(&self) -> String { - format!( - "{:032x}", - definition_hash(&self.definition, &mut HashMap::new()) - ) - } - - /// [`derive`](Self::derive), reusing and recording the coverage of - /// every summary state in `cache`, so validating a DAG derives each - /// state once. - pub(crate) fn derive_cached( - node: &OperatorNode, - cache: &mut NodeToSummaryCoverageCache, + pub(crate) fn derive_from_inputs<'m>( + operator: &Operator>, ) -> Result { - let key = node as *const OperatorNode; - if let Some(coverage) = cache.get(&key) { - return Ok(coverage.clone()); - } - if let Some(build) = selected_build(node)? { - let coverage = of_summary_agg(&build); - cache.insert(key, coverage.clone()); - return Ok(coverage); - } - let coverage = match node.asap() { - Some(ASAPOp::SummaryAgg { .. }) => of_summary_agg(node), - // Back-to-back tumbling windows cover the whole window one build over it - // would. - Some(ASAPOp::SummaryTumblingWindowChain { .. }) => of_summary_agg( - &crate::ir::operator::tumbling_window_chain::whole_window_build(node).map_err( - |error| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }, - )?, - ), - // The newest tumbling windows of one chain cover what one build over their - // window does. - Some(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(_), + let expansion = + |error: SchemaDerivationError| CoverageError::TumblingWindowChainExpansion { + reason: error.to_string(), + }; + let as_built = |op: &ASAPOp>| -> ASAPOp> { + op.map_children(|input| BuiltOrInput::Input(*input)) + }; + match operator { + Operator::NonASAP(_) => match selected_build(operator)? { + Some(build) => of_summary_agg(&build), + None => Err(CoverageError::NotSummary), + }, + Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), + // Back-to-back tumbling windows cover the whole window one build + // over it would. + Operator::ASAP( + op @ ASAPOp::SummaryTumblingWindowChain { + tumbling_window_count, + .. + }, + ) => of_summary_agg(&window_build(op, *tumbling_window_count).map_err(expansion)?), + // The newest tumbling windows of one chain cover what one build + // over their window does. + Operator::ASAP(ASAPOp::SummaryMerge { + children, + newest_tumbling_windows: Some(newest), group_by: None, - .. - }) => of_summary_agg( - &crate::ir::operator::tumbling_window_chain::merged_window_build(node).map_err( - |error| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }, - )?, - ), - Some(ASAPOp::SummaryMerge { + }) => { + let [chain] = children.as_slice() else { + return Err(CoverageError::NotSummary); + }; + let Operator::ASAP(chain) = chain.operator() else { + return Err(CoverageError::NotSummary); + }; + of_summary_agg(&window_build(&chain, *newest).map_err(expansion)?) + } + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { let inputs = children .iter() - .map(|child| Self::derive_cached(child, cache)) + .map(|child| { + child + .summary_coverage() + .cloned() + .ok_or(CoverageError::NotSummary) + }) .collect::, _>>()?; let merged = of_merge(&inputs)?; match group_by { - None => merged, - Some(keys) => regrouped(merged, keys)?, + None => Ok(merged), + Some(keys) => regrouped(merged, &children[0], keys), } } - _ => return Err(CoverageError::NotSummary), - }; - cache.insert(key, coverage.clone()); - Ok(coverage) + Operator::ASAP(_) => Err(CoverageError::NotSummary), + } + } + + /// A stable identity of `definition`: the 128-bit FNV-1a hash, in hex, + /// of its input group and unlifted operators. Two equal definitions + /// (the equality a merge requires) of one memo get equal ids, so a + /// summary store can key states by this id and a concrete selection. + pub(crate) fn definition_id(&self) -> String { + const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; + const PRIME: u128 = 0x0000000001000000000000000000013b; + let mut hash = OFFSET; + for byte in format!("{:?}", self.definition).bytes() { + hash ^= u128::from(byte); + hash = hash.wrapping_mul(PRIME); + } + format!("{hash:032x}") } } /// For a `Filter` that keeps whole states of a grouped `SummaryAgg` by /// their key columns, the build that computes exactly the kept states: the /// same build over its input rows filtered by the same predicate on the -/// input columns those keys name. `None` for any other node. -fn selected_build(node: &OperatorNode) -> Result>, CoverageError> { - let Some(states) = crate::ir::operator::asap::selected_from(node) else { +/// input columns those keys name. `None` for any other operator. +fn selected_build<'m>( + operator: &Operator>, +) -> Result>>, CoverageError> { + let Operator::NonASAP(NonASAPOp::Filter { + pred, + child: states, + }) = operator + else { return Ok(None); }; - let Some(NonASAPOp::Filter { pred, .. }) = node.non_asap() else { + if operator.output_kind() != OperatorResultKind::State { return Ok(None); - }; - let Some(ASAPOp::SummaryAgg { + } + let Operator::ASAP(ASAPOp::SummaryAgg { child, family, input, reduction: Reduction::Reduce(by), grouping, filter, - }) = states.asap() + }) = states.operator() else { return Err(CoverageError::NotSummary); }; @@ -381,7 +414,7 @@ fn selected_build(node: &OperatorNode) -> Result>, Cover let mut unmapped = None; let rows_pred = pred.0.map_columns(&mut |column| { let mapped = by.keys().get(column).copied().filter(|&input_column| { - matches!((states.schema.fields.get(column), child.schema.fields.get(input_column)), + matches!((states.schema().fields.get(column), child.schema().fields.get(input_column)), (Some(at_state), Some(at_input)) if at_state.name == at_input.name) }); mapped.unwrap_or_else(|| { @@ -392,41 +425,42 @@ fn selected_build(node: &OperatorNode) -> Result>, Cover if let Some(key) = unmapped { return Err(CoverageError::NotAGroupKey { key }); } - let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: crate::ir::scalar::Predicate(rows_pred), - child: Rc::clone(child), - })) - .map_err(|error| CoverageError::RegroupedBuild { + let mut built = |input: &MemoGroupAsInput<'m>| BuiltOrInput::Input(*input); + let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { reason: error.to_string(), - })?; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + }; + let rows = BuiltOrInput::build(NonASAPOp::Filter { + pred: Predicate(rows_pred.map_operator_refs(&mut built)), + child: BuiltOrInput::Input(child), + }) + .map_err(regrouped_build)?; + let reduction = Reduction::Reduce(by); + summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; + Ok(Some(ASAPOp::SummaryAgg { child: rows, - family: family.clone(), - input: input.clone(), - reduction: Reduction::Reduce(by.clone()), - grouping: grouping.clone(), - filter: filter.clone(), + family, + input, + reduction, + grouping, + filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), })) - .map(Some) - .map_err(|error| CoverageError::RegroupedBuild { - reason: error.to_string(), - }) } /// The coverage of a merge of states covering `inputs`. fn of_merge(inputs: &[SummaryCoverage]) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; - let mut proven = HashSet::new(); - if !inputs + if inputs .iter() - .all(|input| same_definition(&first.definition, &input.definition, &mut proven)) + .any(|input| input.definition != first.definition) { return Err(CoverageError::DefinitionMismatch); } // A family without a merge is checked as the strictest rule; the // executor rejects the merge itself. - let required = match first.definition.asap() { - Some(ASAPOp::SummaryAgg { family, .. }) => FamilyCapability::of(family).merge_inputs, + let required = match first.definition.unlifted_operators.first() { + Some(Operator::ASAP(ASAPOp::SummaryAgg { family, .. })) => { + FamilyCapability::of(family).merge_inputs + } _ => None, } .unwrap_or(MergeInputRule::Disjoint); @@ -452,26 +486,30 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result Result { - let Some(ASAPOp::SummaryAgg { - child, +/// The coverage of a regrouping merge, keyed by `keys`, of states whose +/// merge covers `merged` (`first` is its first input): the same rows +/// (`selection`), computed by the input's build grouped by the columns +/// `keys` name. The input states of one output group are different groups +/// of one build, so their rows are disjoint. A counter family merges only +/// time-disjoint windows, never different groups at one time. +fn regrouped( + merged: SummaryCoverage, + first: &MemoGroupAsInput<'_>, + keys: &[ColumnId], +) -> Result { + let Some(Operator::ASAP(ASAPOp::SummaryAgg { family, input, reduction, grouping, filter, - }) = merged.definition.asap() + .. + })) = merged.definition.unlifted_operators.first() else { return Err(CoverageError::NotSummary); }; @@ -480,7 +518,8 @@ fn regrouped(merged: SummaryCoverage, keys: &[ColumnId]) -> Result Result Result, _>>()?; - // Derived as a direct build's schema, so its definition id is the same. - let definition = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), + let reduction = Reduction::by(mapped); + summary_agg_schema(&rows, family, &reduction).map_err(|error| { + CoverageError::RegroupedBuild { + reason: error.to_string(), + } + })?; + // A direct build's definition, so its definition id is the same. + let mut definition = merged.definition.clone(); + definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { + child: (), family: family.clone(), input: input.clone(), - reduction: Reduction::by(mapped), + reduction, grouping: grouping.clone(), filter: filter.clone(), - })) - .map_err(|error| CoverageError::RegroupedBuild { - reason: error.to_string(), - })?; + }); Ok(SummaryCoverage { definition, selection: merged.selection, }) } +/// The schema of the rows the build of summary state `state` reads (its +/// `SummaryAgg`'s input; one build over a chain's window reads its range of +/// the chain's input, which has the input's schema). `None` when `state` +/// is no summary state. +fn build_input_schema(state: &I) -> Option { + if let Some(states) = selected_from(state) { + return build_input_schema(&states); + } + match state.operator() { + Operator::ASAP( + ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, + ) => Some(child.schema().clone()), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { + build_input_schema(children.first()?) + } + _ => None, + } +} + /// Whether the time windows of every box of `inputs` are pairwise apart: /// sorted by start, each ends before the next starts. `false` when a box /// has no time window or two windows overlap, boxes of one input included. @@ -548,216 +610,132 @@ fn windows_ordered_apart(inputs: &[SummaryCoverage]) -> bool { }) } -/// The hash [`SummaryCoverage::definition_id`] reports: each node's -/// operator without its children, output kind and schema (in their `Debug` -/// form, which every IR type derives and which is fixed for a given IR), -/// then its children's hashes in order. `cache` hashes a shared node once. -fn definition_hash( - node: &Rc, - cache: &mut HashMap<*const OperatorNode, u128>, -) -> u128 { - if let Some(hash) = cache.get(&Rc::as_ptr(node)) { - return *hash; - } - const OFFSET: u128 = 0x6c62272e07bb014262b821756295c58d; - const PRIME: u128 = 0x0000000001000000000000000000013b; - let mut hash = OFFSET; - let mut write = |bytes: &[u8]| { - for byte in bytes { - hash ^= u128::from(*byte); - hash = hash.wrapping_mul(PRIME); - } - }; - let own = format!( - "{:?}|{:?}|{:?}", - node.operator.map_children(|_| ()), - node.result_kind, - node.schema - ); - write(own.as_bytes()); - for child in node.children() { - let child_hash = definition_hash(child, cache); - write(&child_hash.to_le_bytes()); - } - cache.insert(Rc::as_ptr(node), hash); - hash -} - -/// Structural equality ignoring planning metadata (`timing`, `guarantee`). -/// `proven` memoizes node pairs already found equal. -fn same_definition( - a: &Rc, - b: &Rc, - proven: &mut HashSet<(*const OperatorNode, *const OperatorNode)>, -) -> bool { - let key = (Rc::as_ptr(a), Rc::as_ptr(b)); - if Rc::ptr_eq(a, b) || proven.contains(&key) { - return true; - } - let (ac, bc) = (a.children(), b.children()); - let equal = a.operator.map_children(|_| ()) == b.operator.map_children(|_| ()) - && a.result_kind == b.result_kind - && a.schema == b.schema - && ac.len() == bc.len() - && ac - .iter() - .zip(&bc) - .all(|(x, y)| same_definition(x, y, proven)); - if equal { - proven.insert(key); - } - equal -} - -fn of_summary_agg(node: &OperatorNode) -> SummaryCoverage { - let Some(ASAPOp::SummaryAgg { +/// The coverage of `build`, a `SummaryAgg` over memo groups (or over an +/// input a rule built over them): its operators walked down through the +/// liftable operators of its input groups' expressions. +fn of_summary_agg(build: &ASAPOp>) -> Result { + let ASAPOp::SummaryAgg { child, family, input, reduction, grouping, filter, - }) = node.asap() + } = build else { unreachable!("called on a SummaryAgg"); }; + // Operator<()> cannot say which group a scalar subquery reads. + if build.children().len() > 1 { + return Err(CoverageError::SubqueryInDefinition); + } - // The chain a predicate can be lifted through, top to bottom, and the - // first node below it. - let mut chain: Vec<&Rc> = Vec::new(); - let mut base = child; - while let Some(op) = base.non_asap() { - let passes = match op { - // A rename outside a `Project` would be dropped with the node - // when everything is lifted out of it, so such a node stays. - NonASAPOp::Project { .. } => true, - _ if base.operator.output_schema().ok().as_ref() != Some(&base.schema) => false, - NonASAPOp::Filter { .. } => true, - NonASAPOp::TimeRange { kind, .. } => *kind == TimeRangeKind::Range, - NonASAPOp::TimeShift { shift, .. } => shift.at.is_none(), - _ => false, - }; + // The operators a predicate can be lifted through, top to bottom, and + // the first input below them. One with a scalar subquery stays below. + let mut chain: Vec>> = Vec::new(); + let mut base = child.clone(); + loop { + let operator = base.operator(); + let passes = operator.children().len() == 1 + && match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => { + *kind == TimeRangeKind::Range + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), + _ => false, + }; if !passes { break; } - chain.push(base); - base = base.children()[0]; + base = operator.children()[0].clone(); + chain.push(operator); } + let BuiltOrInput::Input(input_group) = &base else { + unreachable!("a rule builds only liftable operators over memo groups"); + }; // Time is lifted only from exactly one range window. let lift_time = chain .iter() - .filter(|link| matches!(link.non_asap(), Some(NonASAPOp::TimeRange { .. }))) + .filter(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. }))) .count() == 1; - let top = &child.schema; + let top = child.schema(); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. let mut lifted = BTreeMap::new(); let agg_filter = filter .as_ref() .and_then(|pred| residual(pred, top, &[], &mut lifted)); - let kept: Vec> = chain + let kept: Vec>> = chain .iter() .enumerate() - .map(|(depth, link)| match link.non_asap() { - Some(NonASAPOp::Filter { pred, .. }) => { + .map(|(depth, link)| match link { + Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { residual(pred, top, &chain[..depth], &mut lifted) } _ => None, }) .collect(); - let scan_kept = match base.non_asap() { - Some(NonASAPOp::Scan { predicates, .. }) => Some( - predicates + // A `Scan`'s predicates (PromQL label matchers) are lifted too; when + // one is, the `Scan` with the rest of its predicates is the last + // unlifted operator, and there is no input group. + let scan = match base.operator() { + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates, + schema, + }) => { + let lifted_above = lifted.clone(); + let kept = predicates .iter() .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) - .collect::>(), - ), + .collect(); + (lifted != lifted_above).then_some(NonASAPOp::Scan { + source, + predicates: kept, + schema, + }) + } _ => None, }; - // Bottom-up: rebuild without what was lifted. A node whose input and - // operator are unchanged keeps its `Rc`. - let mut rebuilt = Rc::clone(base); - if let ( - Some(kept), - Some(NonASAPOp::Scan { - source, - predicates, - schema, - }), - ) = (scan_kept, base.non_asap()) - { - if kept != *predicates { - rebuilt = rebuild( - base, - NonASAPOp::Scan { - source: source.clone(), - predicates: kept, - schema: schema.clone(), - }, - ); - } - } + let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { + child: (), + family: family.clone(), + input: input.clone(), + reduction: reduction.clone(), + grouping: grouping.clone(), + filter: agg_filter, + })]; let (mut window_ms, mut shift_ms) = (0i64, 0i64); - for (depth, link) in chain.iter().enumerate().rev() { - let child = Rc::clone(&rebuilt); - let op = match link.non_asap().expect("chain nodes are non-ASAP") { - NonASAPOp::Filter { pred, .. } => match &kept[depth] { - None => continue, - Some(rest) if rest == pred && Rc::ptr_eq(&child, link.children()[0]) => { - rebuilt = Rc::clone(link); - continue; + for (depth, link) in chain.iter().enumerate() { + match link { + Operator::NonASAP(NonASAPOp::Filter { .. }) => { + if let Some(rest) = kept[depth].clone() { + unlifted_operators.push(Operator::NonASAP(NonASAPOp::Filter { + pred: rest, + child: (), + })); } - Some(rest) => NonASAPOp::Filter { - pred: rest.clone(), - child, - }, - }, - NonASAPOp::TimeRange { range, .. } if lift_time => { + } + Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) if lift_time => { window_ms = range.as_millis() as i64; - continue; } - NonASAPOp::TimeShift { shift, .. } if lift_time => { + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if lift_time => { shift_ms += shift.offset_ms; - continue; - } - _ if Rc::ptr_eq(&child, link.children()[0]) => { - rebuilt = Rc::clone(link); - continue; } - NonASAPOp::TimeRange { range, kind, .. } => NonASAPOp::TimeRange { - range: *range, - kind: *kind, - child, - }, - NonASAPOp::TimeShift { shift, .. } => NonASAPOp::TimeShift { - shift: *shift, - child, - }, - NonASAPOp::Project { - cols, qualifier, .. - } => NonASAPOp::Project { - cols: cols.clone(), - qualifier: qualifier.clone(), - child, - }, - _ => unreachable!("only liftable operators are on the chain"), - }; - rebuilt = rebuild(link, op); + other => unlifted_operators.push(other.map_children(|_| ())), + } } - let definition = rebuild_asap( - node, - ASAPOp::SummaryAgg { - child: rebuilt, - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: agg_filter, - }, - ); + let input_group = match scan { + Some(scan) => { + unlifted_operators.push(Operator::NonASAP(scan)); + None + } + None => Some(input_group.group_id), + }; let selection = SelectionBox { columns: lifted @@ -776,40 +754,28 @@ fn of_summary_agg(node: &OperatorNode) -> SummaryCoverage { Bound::Included(-shift_ms), )), }; - SummaryCoverage { - definition, + Ok(SummaryCoverage { + definition: SummaryStateDefinition { + input_group, + unlifted_operators, + }, selection: vec![selection], - } -} - -/// A node with the same output schema over a new operator. The definition -/// describes what was computed; it is not re-validated as a plan. -fn rebuild(original: &OperatorNode, op: NonASAPOp) -> Rc { - Rc::new(OperatorNode::with_schema( - Operator::NonASAP(op), - original.schema.clone(), - )) -} - -fn rebuild_asap(original: &OperatorNode, op: ASAPOp) -> Rc { - Rc::new(OperatorNode::with_schema( - Operator::ASAP(op), - original.schema.clone(), - )) + }) } /// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest. `above` are the chain -/// nodes between the predicate and the `SummaryAgg`, top to bottom. A column -/// whose `(table, name)` is not unique in `top` cannot be named in a -/// selection, so its conjuncts stay. -fn residual( - pred: &Predicate, +/// `top`, the agg child's schema) and return the rest, without input refs +/// (the caller keeps only predicates without scalar subqueries). `above` +/// are the operators between the predicate and the `SummaryAgg`, top to +/// bottom. A column whose `(table, name)` is not unique in `top` cannot be +/// named in a selection, so its conjuncts stay. +fn residual( + pred: &Predicate, top: &Schema, - above: &[&Rc], + above: &[Operator], lifted: &mut BTreeMap, -) -> Option { - let mut rest: Vec = Vec::new(); +) -> Option> { + let mut rest: Vec> = Vec::new(); for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; @@ -832,7 +798,7 @@ fn residual( Some((column, constraint)) => { lifted.insert(column, constraint); } - None => rest.push(conjunct.clone()), + None => rest.push(conjunct.map_operator_refs(&mut |_| ())), } } match rest.len() { @@ -844,12 +810,12 @@ fn residual( /// Map a column of the input of `above.last()` up to the agg child's output: /// a `Project` passes it only as a direct column item. -fn column_at_top(mut column: ColumnId, above: &[&Rc]) -> Option { +fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { for link in above.iter().rev() { - if let Some(NonASAPOp::Project { cols, .. }) = link.non_asap() { + if let Operator::NonASAP(NonASAPOp::Project { cols, .. }) = link { column = cols .iter() - .position(|item| item.expr == ScalarExpr::Column(column))?; + .position(|item| matches!(item.expr, ScalarExpr::Column(c) if c == column))?; } } Some(column) @@ -857,8 +823,8 @@ fn column_at_top(mut column: ColumnId, above: &[&Rc]) -> Option`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. -fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { - let literal = |e: &ScalarExpr| match e { +fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { + let literal = |e: &ScalarExpr| match e { ScalarExpr::Literal(v) if *v != ScalarValue::Null => Some(v.clone()), _ => None, }; diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 42cada55a..4a312b0b9 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -19,6 +19,7 @@ use std::rc::Rc; use serde::{Deserialize, Serialize}; +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::node::OperatorNode; use crate::ir::scalar::scalar_type_rules::MapScalarFunction; use crate::ir::schema::{ColumnId, DataType, Schema}; @@ -353,7 +354,7 @@ impl ScalarExpr { } } -impl ScalarExpr { +impl ScalarExpr { /// Infer the `(DataType, nullable)` this expression produces against the /// input schema its owner evaluates it in. Unregistered functions are /// rejected. A reference to a field carrying summary state is an error: @@ -579,7 +580,7 @@ impl ScalarExpr { // `scalar(v)` is one float sample (NaN when the vector is not // exactly one series); a scalar subquery is its single column. ScalarExpr::PromqlScalarFromVector(node) => { - if node.result_kind != super::OperatorResultKind::InstantVector { + if node.result_kind() != super::OperatorResultKind::InstantVector { return Err(signature("scalar() requires an instant vector")); } (DataType::Float64, false) @@ -622,25 +623,25 @@ fn common_scalar_type(a: &DataType, b: &DataType) -> Result Result<(), SchemaDerivationError> { - if node.result_kind == super::OperatorResultKind::Relation { +fn relation(node: &C) -> Result<(), SchemaDerivationError> { + if node.result_kind() == super::OperatorResultKind::Relation { Ok(()) } else { Err(signature("SQL subquery requires a relation")) } } -fn scalar_subquery_field( - node: &OperatorNode, +fn scalar_subquery_field( + node: &C, ) -> Result<&crate::ir::schema::Field, SchemaDerivationError> { relation(node)?; - match node.schema.fields.as_slice() { + match node.schema().fields.as_slice() { [field] => Ok(field), _ => Err(signature("scalar subquery requires exactly one column")), } } -fn sql_function_type( +fn sql_function_type( name: &str, - args: &[ScalarExpr], + args: &[ScalarExpr], schema: &Schema, ) -> Result<(DataType, bool), SchemaDerivationError> { let types = args @@ -720,7 +721,10 @@ fn sql_function_type( /// Resolve the bounded canonical `asap_struct_field(struct, selector)` operation. /// Selectors are positive 1-based literal ordinals or exact literal field names. -pub fn struct_field_type(args: &[ScalarExpr], schema: &Schema) -> Result<(DataType, bool), String> { +pub fn struct_field_type( + args: &[ScalarExpr], + schema: &Schema, +) -> Result<(DataType, bool), String> { let [input, selector] = args else { return Err("struct field access requires a struct and constant selector".into()); }; @@ -757,8 +761,8 @@ pub fn struct_field_type(args: &[ScalarExpr], schema: &Schema) -> Result<(DataTy /// Resolve `asap_element_access(collection, index)`: Map access through the /// map function contract, List access with integer indices. -pub fn element_access_type( - args: &[ScalarExpr], +pub fn element_access_type( + args: &[ScalarExpr], schema: &Schema, ) -> Result<(DataType, bool), String> { let [input, index] = args else { diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 1ad5c2186..6600fd1ca 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,9 +1,10 @@ //! Ranked-row output contract shared by logical TopK and its implementations. +use crate::ir::operator::input::OperatorInput; use crate::ir::operator::{Reduction, Source}; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; -use crate::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; +use crate::ir::{ASAPOp, NonASAPOp, Operator, ScalarExpr}; /// The column a top-k ranks `child` by. Over a one-measure grouped /// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it @@ -12,17 +13,17 @@ use crate::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; /// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` /// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over /// per-series rows (PromQL `topk by (..) (k, rows)`) it is the sample value. -pub fn ranked_column(child: &OperatorNode) -> Result { - let mut node = child; +pub fn ranked_column(child: &I) -> Result { + let mut node = child.clone(); let reduction = loop { - match &node.operator { + match node.operator() { // Only a renaming keeps the measure last. Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols.len() == child.schema.fields.len() + if cols.len() == child.schema().fields.len() && cols .iter() .enumerate() - .all(|(i, item)| item.expr == ScalarExpr::Column(i)) => + .all(|(i, item)| matches!(item.expr, ScalarExpr::Column(c) if c == i)) => { node = child } @@ -49,34 +50,34 @@ pub fn ranked_column(child: &OperatorNode) -> Result { } }; match reduction { - Some(Reduction::Reduce(_)) => Ok(child.schema.fields.len() - 1), - _ => resolve_column_ref(&ColumnRef::SampleValue, &child.schema), + Some(Reduction::Reduce(_)) => Ok(child.schema().fields.len() - 1), + _ => resolve_column_ref(&ColumnRef::SampleValue, child.schema()), } } /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. -pub fn ranked_row_projection( - child: &OperatorNode, +pub(crate) fn ranked_row_projection( + child: &I, reduction: &Reduction, output_name: &str, -) -> Result, crate::ir::SchemaDerivationError> { +) -> Result>, crate::ir::SchemaDerivationError> { use crate::ir::SchemaDerivationError; let value = ranked_column(child) .map_err(|error| SchemaDerivationError::InvalidScalarSignature(error.to_string()))?; let keys = reduction.group_keys().map_or(&[][..], |keys| keys.keys()); for &index in keys { - if index >= child.schema.fields.len() { + if index >= child.schema().fields.len() { return Err(SchemaDerivationError::InvalidGroupByColumn( index, - child.schema.fields.len(), + child.schema().fields.len(), )); } } - let mut columns: Vec = keys + let mut columns: Vec> = keys .iter() .map(|&index| ProjectItem { - alias: Some(child.schema.fields[index].name.clone()), + alias: Some(child.schema().fields[index].name.clone()), expr: ScalarExpr::Column(index), }) .collect(); @@ -85,7 +86,7 @@ pub fn ranked_row_projection( expr: ScalarExpr::Column(value), }); let promql = reads_only_time_series(child); - for (index, field) in child.schema.fields.iter().enumerate() { + for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item && index != value @@ -102,7 +103,7 @@ pub fn ranked_row_projection( // so consumers' already-bound column positions remain valid. if let Ok(identity) = resolve_column_ref( &ColumnRef::Named(PROMQL_SERIES_IDENTITY.into()), - &child.schema, + child.schema(), ) { if !keys.contains(&identity) { columns.push(ProjectItem { @@ -142,12 +143,11 @@ pub fn series_identity_item( } } -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node - .children() - .iter() - .all(|child| reads_only_time_series(child)), +fn reads_only_time_series(node: &I) -> bool { + match node.operator() { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { + matches!(source, Source::TimeSeries { .. }) + } + operator => operator.children().into_iter().all(reads_only_time_series), } } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 540d02d2c..04c78b7d5 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -5,8 +5,10 @@ use std::collections::HashSet; -use super::{MemoGroup, MemoGroupId}; +use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; +use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::schema::{SketchAlgorithm, SketchParams}; +use crate::ir::ASAPOp; use crate::ir::Operator; use crate::ir::QueryRoot; use crate::ir::SchemaDerivationError; @@ -45,25 +47,29 @@ impl Memo { todo!("#1207 PR 2") } - /// Hash-consing: the group with this expression, added if new. Fails - /// when the expression breaks the node rules (schema derivation, input + /// Hash-consing: the group with this expression, added if new, with the + /// `GroupProperties` [`derive_group_properties`] derives. Fails when the + /// expression breaks the node rules (schema derivation, input /// validation, merge check). pub fn find_or_add_group( &mut self, expression: Operator, ) -> Result { - let _ = expression; + let _group_properties = derive_group_properties(self, &expression)?; todo!("#1207 PR 2: hash-consing over groups") } /// Adds `candidate` to `group`'s candidates. Fails when its expression - /// breaks the node rules or its output differs from the group's. + /// breaks the node rules ([`derive_group_properties`]) or its output + /// schema or kind differs from the group's. pub fn add_candidate( &mut self, group: MemoGroupId, candidate: CandidateForMemoGroup, ) -> Result { - let _ = (group, candidate); + let _output = derive_group_properties(self, &candidate.expression.operator) + .map_err(CandidateError::Schema)?; + let _ = group; todo!("#1207 PR 2") } @@ -81,7 +87,7 @@ impl Memo { let count = self.groups.len(); let mut reads: Vec> = vec![HashSet::new(); count]; for (g, group) in self.groups.iter().enumerate() { - for child in group.child_groups.iter().chain( + for child in group.expression.children().into_iter().chain( group .candidates .iter() @@ -116,6 +122,34 @@ impl Memo { } } +/// The properties of a group, or a candidate, computing `expression` over +/// its input groups: the rules that check operator nodes, reading each +/// input group's `group_properties` and `expression` +/// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression +/// breaks them (schema derivation, input validation, merge check). +fn derive_group_properties( + memo: &Memo, + expression: &Operator, +) -> Result { + let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); + let schema = operator.output_schema()?; + operator.validate_inputs()?; + let summary_coverage = match SummaryCoverage::derive_from_inputs(&operator) { + Ok(coverage) => Some(coverage), + // A merge is valid only over states of one definition it may merge. + Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { + return Err(error.into()) + } + Err(_) => None, + }; + Ok(GroupProperties { + schema, + result_kind: operator.output_kind(), + summary_coverage, + shareable_when_equal: operator.is_shareable_when_equal(), + }) +} + /// A candidate [`Memo::add_candidate`] rejects; the memo is unchanged. #[derive(Debug, thiserror::Error)] pub enum CandidateError { diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 67df23b50..153346093 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -2,36 +2,87 @@ //! with its candidates. use std::collections::BTreeSet; -use std::rc::Rc; -use super::CandidateForMemoGroup; -use crate::ir::{OperatorNode, QueryRoot}; +use super::{CandidateForMemoGroup, Memo}; +use crate::ir::operator::input::OperatorInput; +use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::schema::Schema; +use crate::ir::{Operator, OperatorResultKind, QueryRoot}; /// A group, by index in the memo's groups. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct MemoGroupId(pub u32); -/// One distinct logical sub-expression of the workload. +/// One distinct logical sub-expression of the workload: what it computes +/// and the output every candidate of it must produce. #[derive(Debug, Clone)] pub struct MemoGroup { - /// Its logical node as the frontends (and Pass 1 targets) see it; its - /// children are other groups' nodes. - pub logical_node: Rc, - /// Its child groups, in the node's child order. - pub child_groups: Vec, - /// The groups that read it ([`Self::parent_groups`]). Public only so - /// Stage 1, which builds and extends the memo, can maintain it; later - /// stages read it through [`Self::parent_groups`]. + /// What it computes: one operator, with its arguments. Each + /// `MemoGroupId` in it is an input group, never this group's own id. θ + /// is no argument of an operator, so it is not part of a group. + pub expression: Operator, + /// Derived once when the group is created, from its input groups; never + /// changes. + pub group_properties: GroupProperties, + /// The groups that read it ([`Self::parent_groups`]) through their + /// expression or a candidate's. pub parents: Vec, - /// The queries whose roots reach it ([`Self::reached_by_queries`]). - /// Public only so Stage 1, which builds and extends the memo, can - /// maintain it; later stages read it through - /// [`Self::reached_by_queries`]. + /// The queries whose roots reach it ([`Self::reached_by_queries`]), + /// ascending. pub queries: Vec, /// Its candidates: empty after Stage 0, filled by Stage 1. pub candidates: Vec, } +/// What every candidate of a group produces, derived from the group's +/// expression and its input groups' properties by the rules that check +/// operator nodes. +#[derive(Debug, Clone)] +pub struct GroupProperties { + pub schema: Schema, + pub result_kind: OperatorResultKind, + /// What a summary-state group covers; `None` for any other group. + pub summary_coverage: Option, + /// Whether two equal expressions return the same rows for the same + /// input rows, so they may be one group (not an unordered `LIMIT`). + pub shareable_when_equal: bool, +} + +/// A memo group read as an operator's input: its `group_properties` and +/// its `expression`, whose inputs are read the same way. +#[derive(Clone, Copy)] +pub(crate) struct MemoGroupAsInput<'m> { + pub(crate) memo: &'m Memo, + pub(crate) group_id: MemoGroupId, +} + +impl<'m> MemoGroupAsInput<'m> { + fn group(&self) -> &'m MemoGroup { + &self.memo.groups()[self.group_id.0 as usize] + } +} + +impl OperatorInput for MemoGroupAsInput<'_> { + fn schema(&self) -> &Schema { + &self.group().group_properties.schema + } + + fn result_kind(&self) -> OperatorResultKind { + self.group().group_properties.result_kind + } + + fn operator(&self) -> Operator { + self.group().expression.map_children(|&group_id| Self { + memo: self.memo, + group_id, + }) + } + + fn summary_coverage(&self) -> Option<&SummaryCoverage> { + self.group().group_properties.summary_coverage.as_ref() + } +} + impl MemoGroup { /// The groups that read it. pub fn parent_groups(&self) -> &[MemoGroupId] { diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 13cb000d3..596aee4f4 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -21,7 +21,8 @@ pub use candidates::{ LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, WindowLayout, }; pub use logical_memo_candidates::{LogicalASAPMemoCandidates, LogicalEdge}; -pub use memo::{queries_rooted_at, root_groups, MemoGroup, MemoGroupId}; +pub(crate) use memo::MemoGroupAsInput; +pub use memo::{queries_rooted_at, root_groups, GroupProperties, MemoGroup, MemoGroupId}; pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, }; diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 8e138c2f2..71879fdc5 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -4,6 +4,7 @@ //! to the crate-private function of the same name. use crate::ir::materialized_view::RawDataWindow; +use crate::ir::operator::input::node_of_build; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; use crate::ir::properties::summary_coverage::SummaryCoverage; @@ -18,17 +19,17 @@ pub fn definition_id(coverage: &SummaryCoverage) -> String { coverage.definition_id() } -/// `tumbling_window_chain::whole_window_build`. -pub fn whole_window_build(chain: &OperatorNode) -> Result { - crate::ir::operator::tumbling_window_chain::whole_window_build(chain) +/// `tumbling_window_chain::whole_window_build`, as a node. +pub fn whole_window_build(chain: &Rc) -> Result { + crate::ir::operator::tumbling_window_chain::whole_window_build(chain).map(node_of_build) } -/// `tumbling_window_chain::tumbling_window`. +/// `tumbling_window_chain::tumbling_window`, as a node. pub fn tumbling_window( - chain: &OperatorNode, + chain: &Rc, index: u32, ) -> Result { - crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index) + crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index).map(node_of_build) } /// `Operator::output_schema`. @@ -47,7 +48,7 @@ pub fn validate_asap_inputs(op: &ASAPOp) -> Result<(), SchemaDerivationError> { } /// `CurrentSeriesInput::matches_node`. -pub fn matches_node(input: &CurrentSeriesInput, node: &OperatorNode) -> bool { +pub fn matches_node(input: &CurrentSeriesInput, node: &Rc) -> bool { input.matches_node(node) } diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 71f33a0e6..85b33b75c 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -208,7 +208,11 @@ fn declared_group_bound(node: &OperatorNode, data: &DataWorkload) -> Option ) => grouped(reduction, child), // A merge's children are states of one definition: its first // build's grouping stands for all of them. - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => match node.summary_build() { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + .. + }) => match children.first().and_then(|first| first.summary_build()) { Some( ASAPOp::SummaryAgg { reduction, child, .. @@ -216,7 +220,7 @@ fn declared_group_bound(node: &OperatorNode, data: &DataWorkload) -> Option | ASAPOp::SummaryTumblingWindowChain { reduction, child, .. }, - ) => grouped(reduction, child), + ) => grouped(&reduction, &child), _ => None, }, Operator::ASAP(ASAPOp::SummaryEstimate { @@ -365,11 +369,11 @@ fn at_most_one_row(node: &OperatorNode) -> bool { .. }) => keys.is_empty(), // A merge gives one state per group, whatever the tumbling windows it merges. - Operator::ASAP(ASAPOp::SummaryMerge { .. }) => matches!( - node.summary_build(), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => matches!( + children.first().and_then(|first| first.summary_build()), Some( ASAPOp::SummaryAgg { reduction, .. } | ASAPOp::SummaryTumblingWindowChain { reduction, .. } - ) if global(reduction) + ) if global(&reduction) ), Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index fa1718e0e..bf56559c0 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -6,7 +6,6 @@ mod physical_export; mod planner_vocabulary; mod schema_rebuilding; mod structure_contract; -mod summary_coverage; mod summary_merge_structure; mod tumbling_window_chain; mod workload_compatibility; diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 6382be875..47ce849bc 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -117,7 +117,9 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { .unwrap(); pass.validate_structure().unwrap(); assert_eq!(pass.result_kind, OperatorResultKind::State); - assert!(ScalarExpr::Column(0).scalar_type(&pass.schema).is_err()); + assert!(ScalarExpr::>::Column(0) + .scalar_type(&pass.schema) + .is_err()); assert!(validate_asap_inputs(&ASAPOp::SummaryEstimate { summary_input: state.clone(), query: SketchStatistic::Cardinality diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs deleted file mode 100644 index 52b414922..000000000 --- a/crates/types/tests/summary_coverage.rs +++ /dev/null @@ -1,1049 +0,0 @@ -//! Summary coverage is `definition + selection`, derived from the sub-DAG -//! (`docs/design_docs/proposals/asap-primitive-schema.md` §4.2). Merges need -//! the same definition and disjoint selections. -use std::ops::Bound; -use std::rc::Rc; -use std::time::Duration; - -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::operator::TimeShift; -use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, CoverageError, SelectionBox, -}; -use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, GroupingStrategy, MergeInputRule, SketchAlgorithm, SummaryUpdate, -}; -use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, Predicate, - ProjectItem, ScalarExpr, SchemaDerivationError, TimeRangeKind, -}; -use asap_types::test_support::definition_id; - -const JOB: usize = 0; -const REGION: usize = 1; -const LATENCY: usize = 2; - -fn table() -> Rc { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("size", DataType::Float64, false), - ]) -} - -fn table_with(fields: Vec) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::new(fields), - })) - .unwrap() -} - -/// PromQL leaf `m{predicates}` with columns `[ts, value, job]`. -fn series(predicates: Vec) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates, - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -fn compare(column: usize, op: CompareOpKind, value: ScalarValue) -> ScalarExpr { - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op, - right: Box::new(ScalarExpr::Literal(value)), - semantics: ExprSemantics::Sql, - } -} - -fn eq(column: usize, value: &str) -> ScalarExpr { - compare(column, CompareOpKind::Eq, ScalarValue::Utf8(value.into())) -} - -fn filter(child: Rc, pred: ScalarExpr) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - })) - .unwrap() -} - -fn kll(child: Rc, column: &str) -> Rc { - summary( - child, - column, - FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), - ) -} - -fn summary(child: Rc, column: &str, family: FieldDataType) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input: SummaryUpdate::column(ColumnRef::Named(column.into())), - reduction: Reduction::by(vec![JOB]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -/// KLL of latency by job over `t` restricted by `pred`. -fn latency_where(pred: ScalarExpr) -> Rc { - kll(filter(table(), pred), "latency") -} - -/// Tumbling window: `TimeRange(1m)` over `TimeShift(shift)` over `m`. -fn tumbling_window(shift_ms: i64, predicates: Vec) -> Rc { - kll(tumbling_window_rows(shift_ms, predicates), "value") -} - -/// Exact counter-increase tumbling window over the same rows as [`tumbling_window`]. -fn increase_tumbling_window(shift_ms: i64, predicates: Vec) -> Rc { - summary( - tumbling_window_rows(shift_ms, predicates), - "value", - FieldDataType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - ) -} - -/// KLL over `TimeRange(width_ms)` over `TimeShift(shift_ms)` over `m`: the -/// window `(-(shift + width), -shift]`. -fn window(shift_ms: i64, width_ms: u64) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: series(vec![]), - })) - .unwrap(); - let range = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap(); - kll(range, "value") -} - -fn tumbling_window_rows(shift_ms: i64, predicates: Vec) -> Rc { - let shifted = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms: shift_ms, - at: None, - }, - child: series(predicates), - })) - .unwrap(); - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(60_000), - kind: TimeRangeKind::Range, - child: shifted, - })) - .unwrap() -} - -fn merge(children: Vec>) -> Result, SchemaDerivationError> { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) -} - -fn rejected(children: Vec>) -> CoverageError { - match merge(children) { - Err(SchemaDerivationError::Coverage(error)) => error, - other => panic!("expected a coverage error, got {other:?}"), - } -} - -fn column(name: &str) -> ColumnIdentity { - ColumnIdentity::new(None, name) -} - -fn utf8(values: &[&str]) -> Constraint { - Constraint::In( - values - .iter() - .map(|v| ScalarValue::Utf8((*v).into())) - .collect(), - ) -} - -fn relative(from_ms: i64, to_ms: i64) -> Option<(Bound, Bound)> { - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))) -} - -/// A lifted filter leaves the definition over the bare scan. -#[test] -fn filters_lift_into_the_selection() { - let scan = table(); - let us = kll(filter(Rc::clone(&scan), eq(REGION, "us")), "latency"); - let coverage = us.coverage().unwrap(); - assert!(Rc::ptr_eq(coverage.definition().children()[0], &scan)); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [(column("region"), utf8(&["us"]))].into(), - None - )] - ); -} - -/// Different populations merge into one box; the same population twice -/// would count every row twice. -#[test] -fn populations_merge_when_disjoint() { - let merged = merge(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "eu")), - ]) - .unwrap(); - assert_eq!( - merged.coverage().unwrap().selection()[0].columns()[&column("region")], - utf8(&["us", "eu"]) - ); - assert_eq!( - rejected(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "us")), - ]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Equal schemas do not make a KLL over latency and one over size mergeable. -#[test] -fn different_inputs_do_not_merge() { - assert_eq!( - rejected(vec![ - kll(filter(table(), eq(REGION, "us")), "latency"), - kll(filter(table(), eq(REGION, "eu")), "size"), - ]), - CoverageError::DefinitionMismatch - ); -} - -/// The same update over scans of different columns is a different computation. -#[test] -fn the_same_update_over_different_scan_schemas_does_not_merge() { - let scan = |measure: &str| { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain(measure, DataType::Float64, false), - Field::plain("value", DataType::Float64, false), - ]) - }; - assert_eq!( - rejected(vec![ - kll(filter(scan("latency"), eq(REGION, "us")), "value"), - kll(filter(scan("size"), eq(REGION, "eu")), "value"), - ]), - CoverageError::DefinitionMismatch - ); -} - -/// `shipping.region` and `billing.region` are different columns, so -/// restricting each says nothing about overlap. -#[test] -fn qualified_columns_stay_distinct() { - let scan = || { - let qualified = |table: &str| Field { - table: Some(table.into()), - ..Field::plain("region", DataType::Utf8, false) - }; - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - qualified("shipping"), - qualified("billing"), - Field::plain("latency", DataType::Float64, false), - ]) - }; - assert_eq!( - rejected(vec![ - kll(filter(scan(), eq(1, "us")), "latency"), - kll(filter(scan(), eq(2, "eu")), "latency"), - ]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Tumbling windows take their time from `TimeRange` over `TimeShift`: adjacent tumbling windows -/// join, a gap stays two boxes, and the same tumbling window twice overlaps. -#[test] -fn time_tumbling_windows_merge_and_keep_gaps() { - let first = tumbling_window(0, vec![]); - assert_eq!( - first.coverage().unwrap().selection()[0].relative_time(), - relative(-60_000, 0) - ); - assert!(Rc::ptr_eq( - first.coverage().unwrap().definition().children()[0], - first.children()[0].children()[0].children()[0] - )); - let joined = merge(vec![first.clone(), tumbling_window(60_000, vec![])]).unwrap(); - assert_eq!( - joined.coverage().unwrap().selection(), - vec![SelectionBox::new(Default::default(), relative(-120_000, 0))] - ); - let gapped = merge(vec![first.clone(), tumbling_window(120_000, vec![])]).unwrap(); - assert_eq!(gapped.coverage().unwrap().selection().len(), 2); - assert_eq!( - rejected(vec![first.clone(), first]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Counter tumbling-window merge only when their time windows are disjoint: states -/// disjoint by a label but over the same window would interleave in time. -#[test] -fn counter_tumbling_windows_merge_only_disjoint_in_time() { - let joined = merge(vec![ - increase_tumbling_window(0, vec![]), - increase_tumbling_window(60_000, vec![]), - ]) - .unwrap(); - assert_eq!( - joined.coverage().unwrap().selection()[0].relative_time(), - relative(-120_000, 0) - ); - let by_job = |job| increase_tumbling_window(0, vec![Predicate(eq(2, job))]); - assert_eq!( - rejected(vec![by_job("api"), by_job("web")]), - CoverageError::InputsOverlap { - required: MergeInputRule::DisjointInTime, - } - ); -} - -/// A day of one-minute tumbling windows (1,440 inputs) merges into one window, in -/// either order: the inputs' windows are sorted and only neighbours are -/// compared, so the check is O(n log n), not all pairs. A tumbling window listed -/// twice still overlaps. -#[test] -fn a_day_of_minute_tumbling_windows_merges_into_one_window() { - const TUMBLING_WINDOW_COUNT: i64 = 1_440; - let day = relative(-TUMBLING_WINDOW_COUNT * 60_000, 0); - let newest_first: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| tumbling_window(i * 60_000, vec![])) - .collect(); - let oldest_first: Vec<_> = newest_first.iter().rev().cloned().collect(); - for tumbling_windows in [newest_first.clone(), oldest_first] { - let merged = merge(tumbling_windows).unwrap(); - merged.validate_structure().unwrap(); - let coverage = merged.coverage().unwrap(); - let selection = coverage.selection(); - assert_eq!(selection.len(), 1); - assert_eq!(selection[0].relative_time(), day); - } - let counters: Vec<_> = (0..TUMBLING_WINDOW_COUNT) - .map(|i| increase_tumbling_window(i * 60_000, vec![])) - .collect(); - assert_eq!( - merge(counters).unwrap().coverage().unwrap().selection()[0].relative_time(), - day - ); - let mut repeated = newest_first; - repeated.push(tumbling_window(30 * 60_000, vec![])); - assert_eq!( - rejected(repeated), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// A long window that spans two short ones overlaps both, though the -/// short ones are apart: sorted by start, the long window's neighbour -/// overlaps it, so the all-pairs check runs and rejects the merge. The -/// short windows alone merge as two boxes. -#[test] -fn a_long_window_spanning_others_is_rejected() { - let long = window(0, 100_000); - let (near, far) = (window(10_000, 10_000), window(50_000, 10_000)); - assert_eq!( - rejected(vec![long.clone(), near.clone(), far.clone()]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); - assert_eq!( - rejected(vec![near.clone(), far.clone(), long]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); - let apart = merge(vec![far, near]).unwrap(); - assert_eq!( - apart - .coverage() - .unwrap() - .selection() - .iter() - .map(|b| b.relative_time()) - .collect::>(), - [relative(-60_000, -50_000), relative(-20_000, -10_000)] - ); -} - -/// Label matchers on the scan lift like any filter. -#[test] -fn scan_predicates_lift() { - let api = tumbling_window(0, vec![Predicate(eq(2, "api"))]); - let coverage = api.coverage().unwrap(); - assert_eq!( - coverage.selection()[0].columns()[&column("job")], - utf8(&["api"]) - ); - let scan = coverage.definition().children()[0]; - assert!(matches!( - scan.non_asap(), - Some(NonASAPOp::Scan { predicates, .. }) if predicates.is_empty() - )); - merge(vec![api, tumbling_window(0, vec![Predicate(eq(2, "web"))])]).unwrap(); -} - -/// Value ranges are selections: `latency < 100` and `latency >= 100` are -/// disjoint, `latency <= 100` overlaps `latency >= 100`. -#[test] -fn value_ranges_merge_when_disjoint() { - let latency = |op| latency_where(compare(LATENCY, op, ScalarValue::Float64(100.0))); - merge(vec![latency(CompareOpKind::Lt), latency(CompareOpKind::Ge)]).unwrap(); - assert_eq!( - rejected(vec![latency(CompareOpKind::Le), latency(CompareOpKind::Ge)]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// Touching value ranges join like touching time windows; the joined range -/// stays explicit, since neither input took NULL rows. A gap stays apart. -#[test] -fn touching_value_ranges_join() { - let latency = |op, v| latency_where(compare(LATENCY, op, ScalarValue::Float64(v))); - let joined = merge(vec![ - latency(CompareOpKind::Lt, 100.0), - latency(CompareOpKind::Ge, 100.0), - ]) - .unwrap(); - assert_eq!( - joined.coverage().unwrap().selection(), - vec![SelectionBox::new( - [( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Unbounded, - }, - )] - .into(), - None - )] - ); - let gap = merge(vec![ - latency(CompareOpKind::Lt, 100.0), - latency(CompareOpKind::Gt, 100.0), - ]) - .unwrap(); - assert_eq!(gap.coverage().unwrap().selection().len(), 2); -} - -/// A predicate that is not a value set or interval on one column stays in -/// the definition, so states with different residuals do not merge. -#[test] -fn residual_predicates_stay_in_the_definition() { - let doubled_above = |threshold: f64, region: &str| { - let doubled = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(LATENCY)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), - semantics: ExprSemantics::Sql, - }; - latency_where(ScalarExpr::BoolAnd(vec![ - ScalarExpr::Compare { - left: Box::new(doubled), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(threshold))), - semantics: ExprSemantics::Sql, - }, - eq(REGION, region), - ])) - }; - let state = doubled_above(10.0, "us"); - let coverage = state.coverage().unwrap(); - let definition = coverage.definition(); - assert!(matches!( - definition.children()[0].non_asap(), - Some(NonASAPOp::Filter { .. }) - )); - merge(vec![state.clone(), doubled_above(10.0, "eu")]).unwrap(); - assert_eq!( - rejected(vec![state, doubled_above(20.0, "eu")]), - CoverageError::DefinitionMismatch - ); -} - -/// An instant selector picks the latest sample per series, which is not a -/// selection of rows, so it stays in the definition. -#[test] -fn instant_selectors_stay_in_the_definition() { - let instant = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: Duration::from_millis(300_000), - kind: TimeRangeKind::Instant, - child: series(vec![]), - })) - .unwrap(); - let state = kll(Rc::clone(&instant), "value"); - let coverage = state.coverage().unwrap(); - assert_eq!(coverage.selection()[0].relative_time(), None); - assert!(Rc::ptr_eq(coverage.definition().children()[0], &instant)); -} - -/// A merge is a state with coverage, so merges nest. -#[test] -fn merges_nest() { - let us_eu = merge(vec![ - latency_where(eq(REGION, "us")), - latency_where(eq(REGION, "eu")), - ]) - .unwrap(); - merge(vec![us_eu.clone(), latency_where(eq(REGION, "apac"))]).unwrap(); - assert_eq!( - rejected(vec![us_eu, latency_where(eq(REGION, "eu"))]), - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - ); -} - -/// A merge built around `new` is still checked by `validate_structure`. -#[test] -fn forged_merges_fail_validation() { - let us = latency_where(eq(REGION, "us")); - let schema = us.schema.clone(); - let forged = Rc::new(OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![us.clone(), us], - newest_tumbling_windows: None, - group_by: None, - }), - schema, - )); - assert!(forged.coverage().is_none()); - assert!(matches!( - forged.validate_structure(), - Err(SchemaDerivationError::Coverage( - CoverageError::InputsOverlap { - required: MergeInputRule::Disjoint, - } - )) - )); -} - -/// Only summary states have coverage. -#[test] -fn non_summary_nodes_have_no_coverage() { - assert!(table().coverage().is_none()); - assert!(filter(table(), eq(REGION, "us")).coverage().is_none()); -} - -/// Two output columns with the same `(table, name)` cannot be told apart in -/// a selection, so restrictions on them stay in the definition. -#[test] -fn ambiguous_column_names_do_not_lift() { - let renamed = |restricted: usize, value: &str| { - let item = |column: usize, alias: Option<&str>| ProjectItem { - alias: alias.map(Into::into), - expr: ScalarExpr::Column(column), - }; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ - item(JOB, None), - item(REGION, Some("k")), - item(LATENCY, Some("k")), - item(LATENCY, None), - ], - qualifier: None, - child: table(), - })) - .unwrap(); - kll(filter(project, eq(restricted, value)), "latency") - }; - assert_eq!( - rejected(vec![renamed(1, "us"), renamed(2, "eu")]), - CoverageError::DefinitionMismatch - ); -} - -/// A rename outside a `Project` stays in the definition: here B's `Filter` -/// swaps the names of `region` and `tier`, so B's `tier = 'eu'` must not -/// read as `region = 'eu'` and look disjoint from A's `region = 'us'`. -#[test] -fn renaming_filters_stay_in_the_definition() { - let scan = || { - table_with(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("tier", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]) - }; - let us = kll(filter(scan(), eq(1, "us")), "latency"); - let swapped = filter(scan(), eq(2, "eu")); - let mut schema = swapped.schema.clone(); - schema.fields[1].name = "tier".into(); - schema.fields[2].name = "region".into(); - let swapped = OperatorNode::with_schema(swapped.operator.clone(), schema); - let eu_tier = kll(Rc::new(swapped), "latency"); - assert_eq!( - rejected(vec![us, eu_tier]), - CoverageError::DefinitionMismatch - ); -} - -/// `latency = 1` and `latency = 1.0` select the same rows of a Float64 column. -#[test] -fn literals_of_different_types_are_not_proven_different() { - let one = |value| latency_where(compare(LATENCY, CompareOpKind::Eq, value)); - assert!(merge(vec![ - one(ScalarValue::Int64(1)), - one(ScalarValue::Float64(1.0)) - ]) - .is_err()); -} - -/// A scan predicate whose conjuncts only partly lift keeps just the rest. -#[test] -fn partly_lifted_scan_predicates_keep_only_the_rest() { - let doubled_positive = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(1)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), - semantics: ExprSemantics::Promql, - }), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.0))), - semantics: ExprSemantics::Promql, - }; - let job = |name: &str| { - tumbling_window( - 0, - vec![Predicate(ScalarExpr::BoolAnd(vec![ - eq(2, name), - doubled_positive.clone(), - ]))], - ) - }; - merge(vec![job("api"), job("web")]).unwrap(); -} - -/// The worked example of the design doc (§4.2.2), as the SQL frontend -/// lowers it: the `FILTER` condition and `region = 'us'` (in -/// `Scan.predicates`, through a renaming `Project`) move into the -/// selection; the condition on an expression stays. -#[test] -fn design_doc_worked_example() { - let doubled_above_10 = ScalarExpr::Compare { - left: Box::new(ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Mul, - left: Box::new(ScalarExpr::Column(LATENCY)), - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(2))), - semantics: ExprSemantics::Sql, - }), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(10))), - semantics: ExprSemantics::Sql, - }; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![Predicate(ScalarExpr::BoolAnd(vec![ - eq(REGION, "us"), - doubled_above_10.clone(), - ]))], - schema: Schema::new(vec![ - Field::plain("job", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - ]), - })) - .unwrap(); - let item = |column: usize, alias: Option<&str>| ProjectItem { - alias: alias.map(Into::into), - expr: ScalarExpr::Column(column), - }; - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ - item(JOB, None), - item(REGION, Some("r")), - item(LATENCY, None), - ], - qualifier: None, - child: scan, - })) - .unwrap(); - let below_100 = compare(2, CompareOpKind::Lt, ScalarValue::Int64(100)); - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: project, - family: FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), - input: SummaryUpdate::column(ColumnRef::Named("latency".into())), - reduction: Reduction::by(vec![0]), - grouping: GroupingStrategy::default(), - filter: Some(Predicate(below_100)), - })) - .unwrap(); - - let coverage = state.coverage().unwrap(); - assert_eq!( - coverage.selection(), - vec![SelectionBox::new( - [ - (column("r"), utf8(&["us"])), - ( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Int64(100)), - }, - ), - ] - .into(), - None - )] - ); - // definition: SummaryAgg (no filter) over Project over - // Scan t with only `latency * 2 > 10` left in its predicates. - let Some(ASAPOp::SummaryAgg { filter, child, .. }) = coverage.definition().asap() else { - panic!("definition is a SummaryAgg"); - }; - assert!(filter.is_none()); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Project { .. }))); - assert!(matches!( - child.children()[0].non_asap(), - Some(NonASAPOp::Scan { predicates, .. }) if *predicates == vec![Predicate(doubled_above_10)] - )); -} - -/// The coverage of `node`, which must have one. -fn coverage_of( - node: &OperatorNode, -) -> asap_types::ir::properties::summary_coverage::SummaryCoverage { - node.coverage().expect("a summary state") -} - -/// Equal definitions get equal ids whatever the node, its selection, timing -/// or guarantee; a different computation (another metric, another column, -/// another sketch) gets another id. -#[test] -fn equal_definitions_get_equal_ids() { - let id = |node: &OperatorNode| definition_id(&coverage_of(node)); - let us = latency_where(eq(REGION, "us")); - let eu = latency_where(eq(REGION, "eu")); - assert_eq!(id(&us), id(&eu)); - let newest = tumbling_window(0, vec![]); - assert_eq!(id(&newest), id(&tumbling_window(60_000, vec![]))); - assert_eq!(id(&newest).len(), 32); - let size = kll(filter(table(), eq(REGION, "eu")), "size"); - assert_ne!(id(&size), id(&eu)); - assert_ne!( - id(&tumbling_window(0, vec![])), - id(&increase_tumbling_window(0, vec![])) - ); - assert_ne!(id(&tumbling_window(0, vec![])), id(&us)); -} - -/// Six 10-minute tumbling windows at offsets 0 to 50 minutes share one definition id, -/// each with its own time selection `(-(i + 1) · 10m, -i · 10m]`; their -/// merge has the same id and the hour as its selection. -#[test] -fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { - let tumbling_windows: Vec> = - (0..6).map(|i| window(i * 600_000, 600_000)).collect(); - let coverages: Vec<_> = tumbling_windows - .iter() - .map(|tumbling_window| coverage_of(tumbling_window)) - .collect(); - for (i, coverage) in coverages.iter().enumerate() { - let i = i as i64; - assert_eq!(definition_id(coverage), definition_id(&coverages[0])); - assert_eq!( - coverage.selection(), - [SelectionBox::new( - Default::default(), - relative(-(i + 1) * 600_000, -i * 600_000) - )] - ); - } - let hour = coverage_of(&merge(tumbling_windows).unwrap()); - assert_eq!(definition_id(&hour), definition_id(&coverages[0])); - assert_eq!( - hour.selection(), - [SelectionBox::new( - Default::default(), - relative(-3_600_000, 0) - )] - ); -} - -/// A selection round-trips through JSON; columns are a list and bounds -/// name their kind. -#[test] -fn selections_round_trip_through_json() { - let selection = SelectionBox::new( - [ - (column("region"), utf8(&["eu", "us"])), - ( - column("latency"), - Constraint::Interval { - lower: Bound::Unbounded, - upper: Bound::Excluded(ScalarValue::Float64(100.0)), - }, - ), - ] - .into(), - relative(-120_000, -60_000), - ); - let json = serde_json::to_value(&selection).unwrap(); - assert_eq!( - json["relative_time"], - serde_json::json!([{"excluded": -120000}, {"included": -60000}]) - ); - assert_eq!(json["columns"].as_array().unwrap().len(), 2); - assert_eq!( - serde_json::from_value::(json).unwrap(), - selection - ); -} - -/// A `family` build over `child`'s `latency` (a table) or `value` (a -/// series), grouped by `reduction`. -fn grouped( - child: Rc, - family: FieldDataType, - reduction: Reduction, -) -> Rc { - let column = match child.schema.column_id("latency") { - Some(_) => "latency", - None => "value", - }; - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input: SummaryUpdate::column(ColumnRef::Named(column.into())), - reduction, - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap() -} - -fn kll_family() -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) -} - -fn rollup(state: Rc, keys: Vec) -> Result, String> { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![state], - group_by: Some(keys), - newest_tumbling_windows: None, - })) - .map_err(|error| error.to_string()) -} - -/// States by (job, region) over `region ∈ {us, eu}`, rolled up to region, -/// cover what a direct build by region over those rows does: the same -/// definition (and id) and the same selection. -#[test] -fn a_rollup_covers_the_coarser_build() { - let rows = filter( - table(), - ScalarExpr::InList { - expr: Box::new(ScalarExpr::Column(REGION)), - list: vec![ - ScalarExpr::Literal(ScalarValue::Utf8("us".into())), - ScalarExpr::Literal(ScalarValue::Utf8("eu".into())), - ], - negated: false, - }, - ); - let finer = grouped(rows.clone(), kll_family(), Reduction::by(vec![JOB, REGION])); - let coarse = grouped(rows, kll_family(), Reduction::by(vec![REGION])); - let rolled = rollup(finer, vec![1]).unwrap(); - assert_eq!(rolled.schema, coarse.schema); - let (rolled, coarse) = (rolled.coverage().unwrap(), coarse.coverage().unwrap()); - assert_eq!(definition_id(&rolled), definition_id(&coarse)); - assert_eq!(rolled.selection(), coarse.selection()); - assert_eq!( - rolled.selection()[0].columns()[&column("region")], - utf8(&["us", "eu"]) - ); -} - -/// Per-series states keep their input's columns in place, so a rollup by -/// the `job` label covers a build by `job` over the same series. -#[test] -fn a_rollup_of_per_series_states_covers_the_build_by_its_labels() { - let rows = tumbling_window_rows(0, vec![]); - let job = rows.schema.column_id("job").unwrap(); - let per_series = grouped(rows.clone(), kll_family(), Reduction::PerEntity); - let state_job = per_series.schema.column_id("job").unwrap(); - let rolled = rollup(per_series, vec![state_job]).unwrap(); - let by_job = grouped(rows, kll_family(), Reduction::by(vec![job])); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&by_job.coverage().unwrap()) - ); -} - -/// A rollup is rejected when a key is no group key of the input's build, -/// and when its family cannot merge different groups (counter states merge -/// consecutive windows of one series, IRate has no merge). -#[test] -fn invalid_rollups_fail_construction() { - let without = grouped( - table(), - kll_family(), - Reduction::Reduce( - asap_types::ir::operator::operator_properties::GroupKeys::without(vec![REGION]), - ), - ); - assert!(rollup(without, vec![0]) - .unwrap_err() - .contains(&CoverageError::NotAGroupKey { key: 0 }.to_string())); - for (kind, params) in [ - (ExactKind::Increase, ExactParams::Increase), - (ExactKind::Rate, ExactParams::Rate), - (ExactKind::IRate, ExactParams::IRate), - ] { - let state = grouped( - tumbling_window_rows(0, vec![]), - FieldDataType::ExactAggregate(kind, params), - Reduction::by(vec![2]), - ); - assert!( - rollup(state, vec![]) - .unwrap_err() - .contains(&CoverageError::FamilyCannotRegroup.to_string()), - "{kind:?}" - ); - } -} - -/// A rollup to no keys of a SQL relation's states covers the global build -/// over the same rows: a global SQL aggregate, one row even over no input -/// rows, as the rollup's one state. -#[test] -fn a_rollup_to_no_keys_of_a_relation_covers_the_global_build() { - let by_job = grouped(table(), kll_family(), Reduction::by(vec![JOB])); - let global = grouped(table(), kll_family(), Reduction::by(vec![])); - let rolled = rollup(by_job, vec![]).unwrap(); - assert_eq!(rolled.schema, global.schema); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&global.coverage().unwrap()) - ); -} - -/// `group_by` is left out of JSON when unset, so a merge that keeps its -/// groups serializes as before; a rollup's keys round-trip. -#[test] -fn group_by_is_serialized_only_for_a_rollup() { - let state = grouped(table(), kll_family(), Reduction::by(vec![JOB, REGION])); - let merge = ASAPOp::SummaryMerge { - children: vec![0u32], - group_by: None, - newest_tumbling_windows: None, - }; - assert_eq!( - serde_json::to_value(&merge).unwrap(), - serde_json::json!({"SummaryMerge": {"children": [0]}}) - ); - let rollup_op: ASAPOp = ASAPOp::SummaryMerge { - children: vec![0], - group_by: Some(vec![1]), - newest_tumbling_windows: None, - }; - let json = serde_json::to_string(&rollup_op).unwrap(); - assert_eq!( - serde_json::from_str::>(&json).unwrap(), - rollup_op - ); - assert!(rollup(state, vec![1]).is_ok()); -} - -/// An exact sum rolled up from (job, region) to region finalizes to the -/// direct build's column, type and nullability. -#[test] -fn a_finalized_rollup_has_the_direct_builds_schema() { - let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let finer = grouped(table(), sum.clone(), Reduction::by(vec![JOB, REGION])); - let coarse = grouped(table(), sum, Reduction::by(vec![REGION])); - let finalize = |state: Rc| { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: state, - })) - .unwrap() - }; - assert_eq!( - finalize(rollup(finer, vec![1]).unwrap()).schema, - finalize(coarse).schema - ); -} - -/// A filter on the group keys of per-(job, region) states keeps whole -/// states, so it is still state: merging the kept states covers the global -/// build over the filtered rows. A filter reading the state column is not a -/// selection of states and is rejected. -#[test] -fn a_key_filter_over_states_selects_states() { - let states = grouped(table(), kll_family(), Reduction::by(vec![JOB, REGION])); - let kept = filter(Rc::clone(&states), eq(JOB, "a")); - assert_eq!(kept.result_kind, OperatorResultKind::State); - let rolled = rollup(kept, vec![]).unwrap(); - let direct = grouped( - filter(table(), eq(JOB, "a")), - kll_family(), - Reduction::by(vec![]), - ); - assert_eq!(rolled.schema, direct.schema); - assert_eq!( - definition_id(&rolled.coverage().unwrap()), - definition_id(&direct.coverage().unwrap()) - ); - let on_state = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::IsNull(Box::new(ScalarExpr::Column(2)))), - child: states, - })) - .unwrap(); - assert_eq!(on_state.result_kind, OperatorResultKind::Relation); - assert!(on_state.validate_structure().is_err()); -} diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index b9c73d9ee..afb075fa9 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -653,28 +653,3 @@ fn summary_merge_is_permissive_about_the_family() { merged.validate_structure().unwrap(); assert_eq!(FamilyCapability::of(&irate()).merge_inputs, None); } - -/// Idempotent families (HLL, exact Min/Max) merge states that may share -/// rows, and the union of identical selections is that selection; counting -/// families (exact Sum, KLL) still need disjoint selections. -#[test] -fn idempotent_families_merge_overlapping_states() { - let same_region = |family: FieldDataType| { - merge(vec![ - family_state(family.clone(), "us"), - family_state(family, "us"), - ]) - }; - for family in [hll(), exact(ExactKind::Max, ExactParams::Max)] { - let merged = same_region(family.clone()).unwrap_or_else(|e| panic!("{family:?}: {e}")); - merged.validate_structure().unwrap(); - assert_eq!( - merged.coverage().unwrap().selection(), - family_state(family, "us").coverage().unwrap().selection() - ); - } - for family in [exact(ExactKind::Sum, ExactParams::Sum), kll()] { - let error = same_region(family).unwrap_err(); - assert!(error.contains("not proven disjoint"), "{error}"); - } -} diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index d5242da20..82df7f48f 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -1,7 +1,6 @@ //! A `SummaryTumblingWindowChain` of tumbling windows stands for its tumbling windows: it covers what one //! build over its whole window does, and expands into the `SummaryAgg` //! tumbling windows the executor runs (#509 Pass 2, window-composition rule). -use std::ops::Bound; use std::rc::Rc; use std::time::Duration; @@ -72,22 +71,14 @@ fn merge(children: Vec>) -> Rc { .unwrap() } -fn relative_time(node: &OperatorNode) -> (Bound, Bound) { - let coverage = node.coverage().expect("coverage"); - let [selection] = coverage.selection() else { - panic!("one box: {coverage:?}") - }; - selection.relative_time().expect("a time window") -} - /// The merged chain is a state of the build over the whole 5-min window: -/// the same schema and coverage `(-5m, 0]`, and it validates. +/// the same schema, and it validates. #[test] fn merged_tumbling_windows_cover_the_whole_window() { let chain = chain(scan()); let merged = merge(vec![Rc::clone(&chain)]); merged.validate_structure().unwrap(); - assert!(merged_chain(&merged).is_some_and(|c| std::ptr::eq(c, &*chain))); + assert!(merged_chain(&merged).is_some_and(|c| Rc::ptr_eq(&c, &chain))); let whole = whole_window_build(&chain).unwrap(); let Some(NonASAPOp::TimeRange { range, kind, .. }) = whole.children()[0].non_asap() else { panic!("{:?}", whole.operator) @@ -97,11 +88,6 @@ fn merged_tumbling_windows_cover_the_whole_window() { (Duration::from_millis(5 * MINUTE), TimeRangeKind::Range) ); assert_eq!(merged.schema, whole.schema); - assert_eq!(merged.coverage(), whole.coverage()); - assert_eq!( - relative_time(&merged), - (Bound::Excluded(-5 * MINUTE as i64), Bound::Included(0)) - ); } /// Tumbling window `i` is the build over `(-(i+1)·1m, -i·1m]`, the query's own offset @@ -113,14 +99,6 @@ fn each_tumbling_window_is_one_shifted_minute() { for i in 0..5 { let tumbling_window = tumbling_window(&chain, i).unwrap(); let back = offset + i64::from(i) * MINUTE as i64; - assert_eq!( - relative_time(&tumbling_window), - ( - Bound::Excluded(-(back + MINUTE as i64)), - Bound::Included(-back) - ), - "tumbling window {i}" - ); let range = tumbling_window.children()[0]; let Some(NonASAPOp::TimeShift { shift, child }) = range.children()[0].non_asap() else { panic!("tumbling window {i} reads a shift") @@ -131,7 +109,7 @@ fn each_tumbling_window_is_one_shifted_minute() { } /// Expansion gives the executor a merge over the five tumbling windows, newest first, -/// which covers what the merged chain does; a chain read without a merge +/// newest first; a chain read without a merge /// does not expand. #[test] fn expansion_merges_the_tumbling_windows() { @@ -159,7 +137,6 @@ fn expansion_merges_the_tumbling_windows() { "tumbling window {i}" ); } - assert_eq!(summary_input.coverage(), merged.coverage()); assert!(expand_tumbling_window_chains(&chain).is_err()); } @@ -201,24 +178,18 @@ fn newest(chain: Rc, newest_tumbling_windows: u32) -> Rc Date: Fri, 6 Nov 2026 21:57:39 +0000 Subject: [PATCH 12/30] refactor(accuracy): delete size_for_all_readers and its shared-size helpers (#1207 PR 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One state's θ is no longer sized for several consumers: θ comes from parameter_list per candidate. size_for_all_readers had no non-test caller; min_recorded, kll::shared_k and univmon::shared_heap served only it. --- crates/accuracy/src/estimators/kll.rs | 28 -- crates/accuracy/src/estimators/mod.rs | 332 ------------------ crates/accuracy/src/estimators/univmon.rs | 39 -- .../end-to-end-accuracy-guarantees.md | 4 +- 4 files changed, 1 insertion(+), 402 deletions(-) diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index 11815e994..3225994e4 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -180,34 +180,6 @@ pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> Ske } } -/// The `k` of one KLL shared by two consumers sized alone at `(k, δ)` and -/// `(k2, δ2)`: each `k` resized to the smaller failure probability at no -/// larger rank error ([`with_failure_probability`]), then the larger. The -/// shared build states its guarantee at that smaller δ, where a `k` sized -/// for a larger δ has a larger rank error, so the larger `k` alone could -/// miss its consumer's ε. -pub(crate) fn shared_k( - k: u32, - failure_probability: Option, - k2: u32, - failure_probability2: Option, -) -> u32 { - let delta = failure_probability - .unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY) - .min(failure_probability2.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY)); - let resized = |k, failure_probability| match with_failure_probability( - &SketchParams::Kll { - k, - failure_probability, - }, - delta, - ) { - SketchParams::Kll { k, .. } => k, - _ => k, - }; - resized(k, failure_probability).max(resized(k2, failure_probability2)) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 67fca98d2..69a14f3a1 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -247,162 +247,6 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { (tail * (1.0 + 1e-12)).min(1.0) } -/// The parameters of one state shared by consumers of the same algorithm, -/// each sized alone with one of `params` (Pass 2's summary-capability -/// rule): each parameter's largest value, except where a smaller value is -/// stricter, where it takes the smallest (a KLL's failure probability, -/// DDSketch's α). Not every bound is monotone in its parameters (a -/// UnivMon's L2 collision bound is tighter at a power-of-two width than one -/// column wider), so the shared state is not assumed to meet each -/// consumer's target: the summary-capability rule composes every guarantee -/// above it again, and plan selection checks each consumer against the -/// shared state. `None` for no parameters, -/// parameters of different algorithms, or a shared UnivMon the library -/// would not build (`univmon::hash_independent`). -/// -/// A KLL's `k` and a distinct-count UnivMon's heap are the exceptions to -/// the largest value: their error grows as their failure probability -/// shrinks, so each consumer's `k` or heap is first resized to the shared -/// (smallest) failure probability at no larger error (`kll::shared_k`, -/// `univmon::shared_heap`), and the shared build meets every consumer. -pub fn size_for_all_readers(params: &[&SketchParams]) -> Option { - use SketchParams as P; - let (first, rest) = params.split_first()?; - let shared = rest.iter().try_fold((*first).clone(), |a, b| { - Some(match (a, *b) { - ( - P::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability, - }, - P::UnivMon { - heap_size: h, - sketch_rows: r, - sketch_cols: c, - layers: l, - distinct_failure_probability: f, - }, - ) => P::UnivMon { - // Not the largest heap: a distinct count's error grows as - // its failure probability shrinks, so each consumer's heap - // first grows to the shared one at no larger error. - heap_size: univmon::shared_heap(heap_size, distinct_failure_probability, *h, *f), - sketch_rows: sketch_rows.max(*r), - sketch_cols: sketch_cols.max(*c), - layers: layers.max(*l), - // `None`: not sized for a distinct count. - distinct_failure_probability: min_recorded(distinct_failure_probability, *f), - }, - ( - P::Kll { - k, - failure_probability, - }, - P::Kll { - k: k2, - failure_probability: f2, - }, - ) => P::Kll { - k: kll::shared_k(k, failure_probability, *k2, *f2), - // `None` is the calibration's δ, above any recorded one. - failure_probability: min_recorded(failure_probability, *f2), - }, - (P::Cms { width, depth }, P::Cms { width: w, depth: d }) => P::Cms { - width: width.max(*w), - depth: depth.max(*d), - }, - (P::CountSketch { width, depth }, P::CountSketch { width: w, depth: d }) => { - P::CountSketch { - width: width.max(*w), - depth: depth.max(*d), - } - } - ( - P::CmsWithHeap { - width, - depth, - heap_size, - }, - P::CmsWithHeap { - width: w, - depth: d, - heap_size: h, - }, - ) => P::CmsWithHeap { - width: width.max(*w), - depth: depth.max(*d), - heap_size: heap_size.max(*h), - }, - ( - P::CountSketchWithHeap { - width, - depth, - heap_size, - }, - P::CountSketchWithHeap { - width: w, - depth: d, - heap_size: h, - }, - ) => P::CountSketchWithHeap { - width: width.max(*w), - depth: depth.max(*d), - heap_size: heap_size.max(*h), - }, - ( - P::Hll { - precision, - relative_error, - }, - P::Hll { - precision: p, - relative_error: e, - }, - ) => P::Hll { - precision: precision.max(*p), - relative_error: min_recorded(relative_error, *e), - }, - (P::DDSketch { alpha }, P::DDSketch { alpha: a }) => P::DDSketch { - alpha: alpha.min(*a), - }, - ( - P::Kmv { k, relative_error }, - P::Kmv { - k: k2, - relative_error: e, - }, - ) => P::Kmv { - k: k.max(*k2), - relative_error: min_recorded(relative_error, *e), - }, - _ => return None, - }) - })?; - // The largest rows of one UnivMon with the largest columns of another - // can pass the hash budget `asap_sketchlib` builds within. - match shared { - P::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } if !univmon::hash_independent(sketch_rows, sketch_cols, layers) => None, - shared => Some(shared), - } -} - -/// The smaller of two recorded failure probabilities; an unrecorded one -/// (`None`) yields to a recorded one. -fn min_recorded(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - } -} - /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { @@ -420,182 +264,6 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { (x.ceil() as u32).clamp(lo, hi) } -#[cfg(test)] -mod shared_size_tests { - use super::*; - - /// The shared size takes each parameter's largest value, and the - /// smallest where smaller is stricter (KLL δ, DDSketch α); sizes of - /// different algorithms have none. - #[test] - fn shared_size_takes_each_parameters_strictest_value() { - assert_eq!( - size_for_all_readers(&[&univmon(256, 5, 1024, 16), &univmon(1, 7, 4096, 1)]), - Some(univmon(256, 7, 4096, 16)) - ); - // The distinct count's failure probability, where one was sized - // for, takes the smallest. - let distinct = |heap_size, delta| SketchParams::UnivMon { - heap_size, - sketch_rows: 1, - sketch_cols: 1, - layers: 19, - distinct_failure_probability: Some(delta), - }; - let Some(SketchParams::UnivMon { - heap_size, - distinct_failure_probability, - .. - }) = size_for_all_readers(&[ - &univmon(256, 5, 1024, 16), - &distinct(4096, 0.01), - &distinct(2048, 0.001), - ]) - else { - unreachable!() - }; - // The heap of 4096 sized at δ = 0.01 grows to the shared δ = 0.001 - // at its own relative error. - assert_eq!( - (heap_size, distinct_failure_probability), - (40_961, Some(0.001)) - ); - let kll = |k, failure_probability| SketchParams::Kll { - k, - failure_probability, - }; - let SketchParams::Kll { - k, - failure_probability, - } = size_for_all_readers(&[ - &kll(200, None), - &kll(100, Some(0.001)), - &kll(150, Some(0.005)), - ]) - .unwrap() - else { - unreachable!() - }; - // The `k = 200` consumer, sized at the calibration's δ = 0.01, is - // resized to the shared δ = 0.001 at its own rank error. - assert_eq!((k, failure_probability), (258, Some(0.001))); - assert_eq!( - size_for_all_readers(&[ - &SketchParams::DDSketch { alpha: 0.01 }, - &SketchParams::DDSketch { alpha: 0.001 } - ]), - Some(SketchParams::DDSketch { alpha: 0.001 }) - ); - assert_eq!( - size_for_all_readers(&[ - &kll(200, None), - &SketchParams::Hll { - precision: 12, - relative_error: None - } - ]), - None - ); - assert_eq!(size_for_all_readers(&[]), None); - } - - /// A KLL shared by a consumer at rank (0.01, δ = 0.01) and one at - /// (0.05, δ = 1e-4) meets both: each consumer's `k` is resized to the - /// shared δ = 1e-4 at no larger rank error (A's 0.00997) before the - /// larger is taken (k = 412), not 269 stated at 1e-4 (rank error 0.0151). - #[test] - fn a_kll_shared_across_failure_probabilities_meets_each_reader() { - let consumers = [(0.01, 0.01), (0.05, 1e-4)]; - let params = consumers.map(|(eps, delta)| kll::size_params(eps, delta)); - let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); - let quantile = SketchStatistic::Quantile { q: 0.99 }; - let guarantee = sketch_guarantee(&SketchAlgorithm::Kll, &shared, &quantile).unwrap(); - for (eps, delta) in consumers { - assert!( - satisfies( - &guarantee, - &AccuracyTarget::max_error(eps) - .failure_probability(delta) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - ), - "{shared:?} misses ({eps}, {delta})" - ); - } - assert_eq!( - shared, - SketchParams::Kll { - k: 412, - failure_probability: Some(1e-4) - } - ); - } - - fn univmon(heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8) -> SketchParams { - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability: None, - } - } - - /// Each UnivMon fits `asap_sketchlib`'s hash budget, but the largest - /// rows of one with the largest columns of the other do not - /// (`12 · (⌈log2 2^16⌉ + 1) = 204 > 128`): the library would not build - /// the shared shape, so there is none. - #[test] - fn no_shared_univmon_past_the_hash_budget() { - assert!(univmon::hash_independent(12, 512, 1)); - assert!(univmon::hash_independent(5, 1 << 16, 1)); - assert_eq!( - size_for_all_readers(&[&univmon(1, 12, 512, 1), &univmon(1, 5, 1 << 16, 1)]), - None - ); - } - - /// A distinct-count UnivMon shared by a consumer at relative (0.05, - /// δ = 0.01) and one at (0.2, δ = 1e-4) meets both: each consumer's heap - /// grows to the shared δ = 1e-4 at no larger error before the larger is - /// taken (5 400 000), not B's 337 500 stated at 1e-4 (error 0.2). - #[test] - fn a_distinct_univmon_shared_across_failure_probabilities_meets_each_reader() { - use asap_types::ir::operator::agg_intent::default_cardinality; - let consumers = [(0.05, 0.01), (0.2, 1e-4)]; - let params = consumers.map(|(eps, delta)| { - size_params(SketchAlgorithm::UnivMon, &default_cardinality(), eps, delta) - }); - let shared = size_for_all_readers(&[¶ms[0], ¶ms[1]]).unwrap(); - let guarantee = univmon::distinct_guarantee(&shared, "m").unwrap(); - let error = guarantee.spec.error.evaluate().unwrap(); - // The top-layer term stays symbolic in the source's distinct bound; - // the Chebyshev term is the share of δ the heap was sized for. - let ProbabilityExpr::UnionBound { terms } = &guarantee.spec.failure_probability else { - panic!("{guarantee:?}") - }; - let chebyshev = terms[0].evaluate().unwrap(); - for (eps, delta) in consumers { - assert!( - error <= eps && chebyshev <= delta, - "{shared:?} misses ({eps}, {delta}): error {error}" - ); - } - let SketchParams::UnivMon { - heap_size, - distinct_failure_probability, - .. - } = shared - else { - unreachable!() - }; - assert_eq!( - (heap_size, distinct_failure_probability), - (5_400_000, Some(1e-4)) - ); - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index aa4dcacf0..41b694d1a 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -370,45 +370,6 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar } } -/// The heap of one UnivMon shared by two consumers with heaps `heap_size` -/// and `heap_size2`, sized for a distinct count at the failure -/// probabilities they record (`None`: not sized for one): each distinct -/// consumer's heap grown to the smaller failure probability at no larger -/// relative error ([`with_failure_probability`]), then the larger. The -/// shared build states its distinct guarantee at that smaller δ, where a -/// heap sized for a larger δ has a larger error, so the larger heap alone -/// could miss its consumer's ε. A consumer not sized for a distinct count keeps -/// its heap. -pub(crate) fn shared_heap( - heap_size: u32, - distinct_failure_probability: Option, - heap_size2: u32, - distinct_failure_probability2: Option, -) -> u32 { - let delta = match (distinct_failure_probability, distinct_failure_probability2) { - (Some(a), Some(b)) => a.min(b), - (a, b) => match a.or(b) { - Some(delta) => delta, - None => return heap_size.max(heap_size2), - }, - }; - let resized = |heap_size, distinct_failure_probability| match with_failure_probability( - &SketchParams::UnivMon { - heap_size, - sketch_rows: 1, - sketch_cols: 1, - layers: 1, - distinct_failure_probability, - }, - delta, - ) { - SketchParams::UnivMon { heap_size, .. } => heap_size, - _ => heap_size, - }; - resized(heap_size, distinct_failure_probability) - .max(resized(heap_size2, distinct_failure_probability2)) -} - #[cfg(test)] mod tests { use super::*; diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 4374708cf..1e4a98a05 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -174,9 +174,7 @@ against the DataSketches maximum-rank-error characterization `2.446 / k^0.9433` at a per-trial failure probability of 0.01, not against this single-sided fit or its δ scaling. -A KLL shared by readers with different failure probabilities is sized at the -smallest one. Each reader's `k` is first resized to it at no larger rank -error (`kll::shared_k`), and the largest is kept. See +See [Apache DataSketches KLL accuracy](https://datasketches.apache.org/docs/KLL/KLLAccuracyAndSize.html) and its [C++ contract pinned at `a9b42755072b`](https://github.com/apache/datasketches-cpp/blob/a9b42755072b079fd90b29b9851adc121015c58e/kll/include/kll_sketch.hpp). From 2a04c9f6729e7cbfafc81b458eb3e9f6ea45b7af Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:41:44 +0000 Subject: [PATCH 13/30] refactor(types): InputOrTemporaryBuild; subquery groups in unlifted operators; delete merged_window_build_node (#1207 PR 0) - BuiltOrInput / BuiltInput renamed InputOrTemporaryBuild / TemporaryBuild: a build a rule constructs only while it derives, never stored in the memo. - SummaryStateDefinition::unlifted_operators is Vec>>: None marks an operator's main input, Some(g) another input group it reads (a scalar subquery's). States whose definitions read subqueries compare and merge; CoverageError::SubqueryInDefinition deleted. - merged_window_build_node deleted with its test; the accuracy guarantee of a read of merged tumbling windows is todo!() until PR 4 rewrites it over groups. --- crates/accuracy/src/propagation.rs | 45 ++------ crates/types/src/ir/operator/asap.rs | 8 +- crates/types/src/ir/operator/input.rs | 60 ++++++----- .../src/ir/operator/tumbling_window_chain.rs | 41 ++++--- .../src/ir/properties/summary_coverage.rs | 102 ++++++++++-------- crates/types/tests/tumbling_window_chain.rs | 24 ----- 6 files changed, 125 insertions(+), 155 deletions(-) diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index 7e55e8be4..11b5d8c02 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -7,7 +7,7 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::{merged_chain, merged_window_build_node}; +use asap_types::ir::operator::tumbling_window_chain::merged_chain; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::guarantee::{ AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, @@ -526,31 +526,28 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> /// The guarantee of an estimate or finalized value read from merged /// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range ([`merged_window_build_node`]: the whole -/// chain's, or its newest tumbling windows') when the +/// build over the merged tumbling windows' time range (the whole chain's, or +/// its newest tumbling windows') when the /// family's merged error for that read is exact or the same as that build's /// (the family-capability table); none when it is approximate (evidence /// only, for plan selection to judge) or the family has no merge. `None` for /// any node that does not read merged tumbling windows. fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let (state, merged) = match node.asap()? { + let merged = match node.asap()? { ASAPOp::SummaryEstimate { summary_input, query, } => { let chain = merged_chain(summary_input)?; let family = chain.asap()?.produced_state()?; - ( - summary_input, - FamilyCapability::of(family).merged_error(query), - ) + FamilyCapability::of(family).merged_error(query) } ASAPOp::FinalizeExactAccumulator { child } => { let chain = merged_chain(child)?; let family = chain.asap()?.produced_state()?; // An exact accumulator's merged state is exact. let merged = FamilyCapability::of(family).merge_inputs; - (child, merged.map(|_| MergedError::Exact)) + merged.map(|_| MergedError::Exact) } _ => return None, }; @@ -560,33 +557,9 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option ASAPOp { /// stands for, with its schema: the input's build (one build over a merged /// chain's window) grouped by the input columns the rollup's keys name. /// `None` for any other input, or a rollup whose build does not derive. -fn rolled_up_build(state: &I) -> Option<(ASAPOp>, Schema)> { +fn rolled_up_build( + state: &I, +) -> Option<(ASAPOp>, Schema)> { let Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: Some(keys), @@ -707,7 +709,7 @@ fn rolled_up_build(state: &I) -> Option<(ASAPOp { - op.map_children(|child| BuiltOrInput::Input(child.clone())) + op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) } _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { super::tumbling_window_chain::merged_window_build(&input) diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index 82a7127bb..1a4d034d5 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -42,37 +42,42 @@ impl OperatorInput for Rc { } } -/// An input a rule builds while it derives (one build over a chain's -/// window, the build a rollup stands for, the build a selection of states -/// stands for): an operator over `I`s that is neither a node of a tree nor -/// a memo group. A rule never builds one over another rule's built input, -/// so the type does not nest. +/// An input of a rule: a real input `I`, or a build the rule constructs +/// only while it derives. #[derive(Clone)] -pub(crate) enum BuiltOrInput { +pub(crate) enum InputOrTemporaryBuild { Input(I), - Built(Rc>), + TemporaryBuild(Rc>), } -pub(crate) struct BuiltInput { - operator: Operator>, +/// A build a rule constructs only while it derives (the whole-window build +/// of a chain, the build a rollup stands for, the build a selection of +/// states stands for): an operator over `I`s with its derived schema and +/// kind. Never stored in the memo. A rule never builds one over another +/// rule's temporary build, so the type does not nest. +pub(crate) struct TemporaryBuild { + operator: Operator>, schema: Schema, result_kind: OperatorResultKind, } -impl BuiltOrInput { +impl InputOrTemporaryBuild { /// A non-ASAP `operator` with its derived schema and kind. Only non-ASAP /// rules derive it, which build no input themselves. pub(crate) fn build( - operator: NonASAPOp>, + operator: NonASAPOp>, ) -> Result { let schema = operator.output_schema()?; Ok(Self::with_schema(Operator::NonASAP(operator), schema)) } /// `operator` with `schema` and its derived kind. - pub(crate) fn with_schema(operator: Operator>, schema: Schema) -> Self { + pub(crate) fn with_schema( + operator: Operator>, + schema: Schema, + ) -> Self { let result_kind = operator.output_kind(); - Self::Built(Rc::new(BuiltInput { + Self::TemporaryBuild(Rc::new(TemporaryBuild { operator, schema, result_kind, @@ -80,18 +85,18 @@ impl BuiltOrInput { } } -impl OperatorInput for BuiltOrInput { +impl OperatorInput for InputOrTemporaryBuild { fn schema(&self) -> &Schema { match self { Self::Input(input) => input.schema(), - Self::Built(built) => &built.schema, + Self::TemporaryBuild(build) => &build.schema, } } fn result_kind(&self) -> OperatorResultKind { match self { Self::Input(input) => input.result_kind(), - Self::Built(built) => built.result_kind, + Self::TemporaryBuild(build) => build.result_kind, } } @@ -100,27 +105,27 @@ impl OperatorInput for BuiltOrInput { Self::Input(input) => input .operator() .map_children(|child| Self::Input(child.clone())), - Self::Built(built) => built.operator.clone(), + Self::TemporaryBuild(build) => build.operator.clone(), } } fn summary_coverage(&self) -> Option<&SummaryCoverage> { match self { Self::Input(input) => input.summary_coverage(), - Self::Built(_) => None, + Self::TemporaryBuild(_) => None, } } } -impl BuiltOrInput> { - /// The node tree of this input: a built input as a new node over its - /// inputs' trees. +impl InputOrTemporaryBuild> { + /// The node tree of this input: a temporary build as a new node over + /// its inputs' trees. pub(crate) fn to_node(&self) -> Rc { match self { Self::Input(node) => Rc::clone(node), - Self::Built(built) => Rc::new(OperatorNode::with_schema( - built.operator.map_children(Self::to_node), - built.schema.clone(), + Self::TemporaryBuild(build) => Rc::new(OperatorNode::with_schema( + build.operator.map_children(Self::to_node), + build.schema.clone(), )), } } @@ -128,10 +133,13 @@ impl BuiltOrInput> { /// A build over node trees (`tumbling_window_chain`'s builds) as a node. pub(crate) fn node_of_build( - (build, schema): (super::asap::ASAPOp>>, Schema), + (build, schema): ( + super::asap::ASAPOp>>, + Schema, + ), ) -> OperatorNode { OperatorNode::with_schema( - Operator::ASAP(build.map_children(BuiltOrInput::to_node)), + Operator::ASAP(build.map_children(InputOrTemporaryBuild::to_node)), schema, ) } diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index d4a611915..add4528bf 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -18,7 +18,7 @@ use std::time::Duration; use super::asap::{summary_agg_schema, ASAPOp}; use super::input::node_of_build; -use super::input::{BuiltOrInput, OperatorInput}; +use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; #[cfg(any(test, feature = "test-support"))] @@ -77,7 +77,7 @@ fn merged_tumbling_window_count(state: &I) -> Option { } /// A build over the inputs of a chain over `I`, with its schema. -pub(crate) type BuildWithSchema = (ASAPOp>, Schema); +pub(crate) type BuildWithSchema = (ASAPOp>, Schema); /// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the /// chain's input whose state the merged tumbling windows of `chain` equal. @@ -111,14 +111,6 @@ pub(crate) fn merged_window_build( newest_tumbling_windows_build(&chain, tumbling_window_count) } -/// [`merged_window_build`] over a node tree, as a node (the accuracy -/// propagation reads node trees). -pub fn merged_window_build_node( - state: &Rc, -) -> Result { - merged_window_build(state).map(node_of_build) -} - /// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the /// chain's input: the state of the chain's newest `tumbling_window_count` tumbling windows. fn newest_tumbling_windows_build( @@ -139,7 +131,7 @@ fn newest_tumbling_windows_build( pub(crate) fn window_build( chain: &ASAPOp, tumbling_window_count: u32, -) -> Result>, SchemaDerivationError> { +) -> Result>, SchemaDerivationError> { let ASAPOp::SummaryTumblingWindowChain { child, tumbling_window_width_ms, @@ -151,7 +143,10 @@ pub(crate) fn window_build( let window = tumbling_window_width_ms .checked_mul(u64::from(tumbling_window_count)) .ok_or_else(|| invalid("tumbling-window chain window overflows"))?; - build_over(chain, range(BuiltOrInput::Input(child.clone()), window)?) + build_over( + chain, + range(InputOrTemporaryBuild::Input(child.clone()), window)?, + ) } #[cfg(any(test, feature = "test-support"))] @@ -190,9 +185,9 @@ pub(crate) fn tumbling_window( fn shifted( child: I, back_ms: i64, -) -> Result, SchemaDerivationError> { +) -> Result, SchemaDerivationError> { if back_ms == 0 { - return Ok(BuiltOrInput::Input(child)); + return Ok(InputOrTemporaryBuild::Input(child)); } let (shift, input) = match child.operator() { Operator::NonASAP(NonASAPOp::TimeShift { @@ -216,18 +211,18 @@ fn shifted( child, ), }; - BuiltOrInput::build(NonASAPOp::TimeShift { + InputOrTemporaryBuild::build(NonASAPOp::TimeShift { shift, - child: BuiltOrInput::Input(input), + child: InputOrTemporaryBuild::Input(input), }) } /// A range `TimeRange(width_ms)` over `child`. fn range( - child: BuiltOrInput, + child: InputOrTemporaryBuild, width_ms: u64, -) -> Result, SchemaDerivationError> { - BuiltOrInput::build(NonASAPOp::TimeRange { +) -> Result, SchemaDerivationError> { + InputOrTemporaryBuild::build(NonASAPOp::TimeRange { range: Duration::from_millis(width_ms), kind: TimeRangeKind::Range, child, @@ -237,8 +232,8 @@ fn range( /// The chain's `SummaryAgg` over `input`. fn build_over( chain: &ASAPOp, - input: BuiltOrInput, -) -> Result>, SchemaDerivationError> { + input: InputOrTemporaryBuild, +) -> Result>, SchemaDerivationError> { let ASAPOp::SummaryTumblingWindowChain { family, input: update, @@ -246,7 +241,7 @@ fn build_over( grouping, filter, .. - } = chain.map_children(|child| BuiltOrInput::Input(child.clone())) + } = chain.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) else { return Err(invalid("expected a tumbling-window chain")); }; @@ -262,7 +257,7 @@ fn build_over( /// `build` with its schema, named as the chain of schema `chain`. fn named_as_chain( - build: ASAPOp>, + build: ASAPOp>, chain: &Schema, ) -> Result, SchemaDerivationError> { let ASAPOp::SummaryAgg { diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index a6d195844..0e2dec55e 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; -use crate::ir::operator::input::{BuiltOrInput, OperatorInput}; +use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::operator::operator_properties::Reduction; @@ -44,10 +44,12 @@ pub struct SummaryStateDefinition { /// so states over one source with different matchers may merge. pub input_group: Option, /// The operators not lifted into the selection, listed top down from the - /// SummaryAgg, each with only its residual arguments; `()` marks its - /// single input (the next one, or input_group). Two states may merge only - /// if input_group and these operators are equal. - pub unlifted_operators: Vec>, + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. Two states may merge only if input_group + /// and these operators are equal. + pub unlifted_operators: Vec>>, } #[derive(Debug, Clone, PartialEq)] @@ -276,10 +278,6 @@ pub enum CoverageError { /// nothing, and a Hydra's groups share one state. Bad input. #[error("the inputs' family cannot merge different groups")] FamilyCannotRegroup, - /// A `SummaryAgg` whose filter reads a scalar subquery: its definition - /// cannot name the group the subquery reads, so it merges with nothing. - #[error("a summary filter with a scalar subquery has no comparable definition")] - SubqueryInDefinition, /// The build a regrouping `SummaryMerge` stands for does not derive: /// bad input. #[error("the regrouped build does not derive: {reason}")] @@ -288,7 +286,7 @@ pub enum CoverageError { /// A memo group read as an input, and the inputs a rule builds over such /// groups. -type GroupInput<'m> = BuiltOrInput>; +type GroupInput<'m> = InputOrTemporaryBuild>; impl SummaryCoverage { /// Coverage of a summary state a memo group computes with `operator` @@ -310,7 +308,7 @@ impl SummaryCoverage { reason: error.to_string(), }; let as_built = |op: &ASAPOp>| -> ASAPOp> { - op.map_children(|input| BuiltOrInput::Input(*input)) + op.map_children(|input| InputOrTemporaryBuild::Input(*input)) }; match operator { Operator::NonASAP(_) => match selected_build(operator)? { @@ -425,13 +423,13 @@ fn selected_build<'m>( if let Some(key) = unmapped { return Err(CoverageError::NotAGroupKey { key }); } - let mut built = |input: &MemoGroupAsInput<'m>| BuiltOrInput::Input(*input); + let mut built = |input: &MemoGroupAsInput<'m>| InputOrTemporaryBuild::Input(*input); let regrouped_build = |error: SchemaDerivationError| CoverageError::RegroupedBuild { reason: error.to_string(), }; - let rows = BuiltOrInput::build(NonASAPOp::Filter { + let rows = InputOrTemporaryBuild::build(NonASAPOp::Filter { pred: Predicate(rows_pred.map_operator_refs(&mut built)), - child: BuiltOrInput::Input(child), + child: InputOrTemporaryBuild::Input(child), }) .map_err(regrouped_build)?; let reduction = Reduction::Reduce(by); @@ -553,7 +551,7 @@ fn regrouped( // A direct build's definition, so its definition id is the same. let mut definition = merged.definition.clone(); definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { - child: (), + child: None, family: family.clone(), input: input.clone(), reduction, @@ -625,33 +623,27 @@ fn of_summary_agg(build: &ASAPOp>) -> Result cannot say which group a scalar subquery reads. - if build.children().len() > 1 { - return Err(CoverageError::SubqueryInDefinition); - } - // The operators a predicate can be lifted through, top to bottom, and - // the first input below them. One with a scalar subquery stays below. + // the first input below them. A subquery's group stays named in the + // residual of the operator that reads it. let mut chain: Vec>> = Vec::new(); let mut base = child.clone(); loop { let operator = base.operator(); - let passes = operator.children().len() == 1 - && match &operator { - Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, - Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => { - *kind == TimeRangeKind::Range - } - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), - _ => false, - }; + let passes = match &operator { + Operator::NonASAP(NonASAPOp::Project { .. } | NonASAPOp::Filter { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), + _ => false, + }; if !passes { break; } + // The main input is an operator's first child. base = operator.children()[0].clone(); chain.push(operator); } - let BuiltOrInput::Input(input_group) = &base else { + let InputOrTemporaryBuild::Input(input_group) = &base else { unreachable!("a rule builds only liftable operators over memo groups"); }; // Time is lifted only from exactly one range window. @@ -668,7 +660,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>> = chain + let kept: Vec>>> = chain .iter() .enumerate() .map(|(depth, link)| match link { @@ -702,7 +694,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>) -> Result>) -> Result { shift_ms += shift.offset_ms; } - other => unlifted_operators.push(other.map_children(|_| ())), + other => unlifted_operators.push(unlifted(other)), } } let input_group = match scan { @@ -764,18 +756,18 @@ fn of_summary_agg(build: &ASAPOp>) -> Result( - pred: &Predicate, +fn residual( + pred: &Predicate>, top: &Schema, - above: &[Operator], + above: &[Operator>], lifted: &mut BTreeMap, -) -> Option> { - let mut rest: Vec> = Vec::new(); +) -> Option>> { + let mut rest: Vec>> = Vec::new(); for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; @@ -798,7 +790,7 @@ fn residual( Some((column, constraint)) => { lifted.insert(column, constraint); } - None => rest.push(conjunct.map_operator_refs(&mut |_| ())), + None => rest.push(conjunct.map_operator_refs(&mut group_id)), } } match rest.len() { @@ -808,6 +800,30 @@ fn residual( } } +/// The memo group `input` is; `None` for a rule's temporary build. +fn group_id(input: &GroupInput<'_>) -> Option { + match input { + InputOrTemporaryBuild::Input(group) => Some(group.group_id), + InputOrTemporaryBuild::TemporaryBuild(_) => None, + } +} + +/// `link`, a liftable operator kept whole in a definition: its main input +/// `None`, every other input the group it is. +fn unlifted(link: &Operator>) -> Operator> { + let mut operator = link.map_children(group_id); + match &mut operator { + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::Project { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. }, + ) => *child = None, + _ => unreachable!("only liftable operators are walked"), + } + operator +} + /// Map a column of the input of `above.last()` up to the agg child's output: /// a `Project` passes it only as a direct column item. fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option { diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 82df7f48f..376ab310f 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -168,30 +168,6 @@ fn a_chain_reads_an_instant_vector() { assert!(empty.validate_structure().is_err()); } -fn newest(chain: Rc, newest_tumbling_windows: u32) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - newest_tumbling_windows: Some(newest_tumbling_windows), - group_by: None, - })) - .unwrap() -} - -/// A merge of the newest 2 of 5 tumbling windows is a state of one build over the -/// newest 2 minutes: the window build's schema, and a valid structure. -#[test] -fn a_merge_of_the_newest_tumbling_windows_covers_their_window() { - use asap_types::ir::operator::tumbling_window_chain::merged_window_build_node; - let partial = newest(chain(scan()), 2); - partial.validate_structure().unwrap(); - let build = merged_window_build_node(&partial).unwrap(); - let Some(NonASAPOp::TimeRange { range, .. }) = build.children()[0].non_asap() else { - panic!("{:?}", build.operator) - }; - assert_eq!(*range, Duration::from_millis(2 * MINUTE)); - assert_eq!(partial.schema, build.schema); -} - /// `newest_tumbling_windows` must name between 1 and fewer than all tumbling windows of the one /// chain the merge reads; anything else is rejected. #[test] From ac7273deadf98cf10c3ef801191edcad9f48295d Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:12:44 +0000 Subject: [PATCH 14/30] refactor(types): 2026-11-07 interfaces: chains without a window count, TimeBound, Pass 2 pruned list, Stage 2 entry (#1207 PR 0) - Delete WindowLayout, LogicalProperties::window_layout and its two error variants, SummaryMerge::newest_tumbling_windows and SummaryTumblingWindowChain::tumbling_window_count, with the count-based chain builds and expansion; behaviour that read the count is todo!() for PR 1, 2, 6, 7 or 9. - SelectionBox { columns } only, and TimeBound { At, FromEvaluationTime }; lifting TimeRange/TimeShift is todo!() for PR 2. - PrunedLogicalCandidate, PruneCause, RootPruneReason and LogicalASAPMemoCandidates::pruned. - error_propagation(consumer, metric) -> Option; Memo::set_input_requirements. - PhysicalCandidateKey deleted (PrunedCandidate), PrunedCandidate::group and PhysicalASAPMemoCandidates::query_roots dropped; new(logical, physical, edges, pruned). - Stage 2 entry takes &DeploymentCapabilities (moved into asap-types) and cannot fail; PruneReason follows the doc's Stage 2 table. - DataArrival::{Ingesting, Mixed} gain max_out_of_order_ms. - GroupProperties::summary_coverage -> aggregate_coverage. --- crates/cost/src/usage.rs | 198 ++------- crates/devtools/src/bin/asap_planner.rs | 40 +- crates/devtools/src/candidate_labels.rs | 23 +- crates/devtools/src/planner_fuzz/mod.rs | 7 +- crates/devtools/src/planner_fuzz/relations.rs | 192 +-------- .../devtools/src/planner_layering_examples.rs | 5 + crates/devtools/src/planning_stages.rs | 15 +- crates/devtools/src/planning_stages_text.rs | 2 +- crates/devtools/tests/asap_planner.rs | 2 +- crates/devtools/tests/viewer_contract.rs | 2 - crates/executor/src/physical_planner/mod.rs | 10 +- crates/executor/tests/common/mod.rs | 1 + crates/executor/tests/hydra_cms_execution.rs | 1 - .../executor/tests/precompute_population.rs | 1 - .../stored_tumbling_window_memory_limit.rs | 1 - crates/executor/tests/summary_projection.rs | 1 - .../tests/lowering_semantics.rs | 1 + crates/frontend-promql/src/lib.rs | 1 + crates/frontend-promql/tests/support.rs | 1 + crates/integration-tests/src/lib.rs | 1 + .../tests/joint_search_equivalence.rs | 3 +- .../tests/kll_tumbling_window_execution.rs | 1 - .../tests/planner_layering_common/mod.rs | 48 +-- .../integration-tests/tests/sketch_rollup.rs | 3 - crates/logical-asap-optimization/src/lib.rs | 7 - .../src/pass1/logical_memo_candidates.rs | 2 +- .../src/pass1/per_computation_candidates.rs | 4 +- .../src/pass2/memo/window_candidates.rs | 17 +- .../src/catalog/mod.rs | 43 +- crates/physical-asap-optimization/src/lib.rs | 7 +- crates/planner/src/installable.rs | 6 +- crates/planner/src/lib.rs | 6 +- crates/planner/src/pass.rs | 2 +- crates/planner/src/run.rs | 18 +- crates/planner/tests/e2e_plan.rs | 1 + crates/planner/tests/runtime_install.rs | 77 +--- crates/planner/tests/summary_sharing.rs | 1 + .../src/deployment_capabilities.rs | 8 +- crates/types/src/ir/materialized_view.rs | 4 +- crates/types/src/ir/operator/asap.rs | 93 +---- crates/types/src/ir/operator/input.rs | 39 +- .../src/ir/operator/tumbling_window_chain.rs | 380 +----------------- crates/types/src/ir/physical_export.rs | 33 +- .../src/ir/properties/summary_coverage.rs | 362 ++--------------- crates/types/src/lib.rs | 4 + crates/types/src/stages/candidates.rs | 81 ++-- .../src/stages/logical_memo_candidates.rs | 52 ++- crates/types/src/stages/memo.rs | 11 +- crates/types/src/stages/mod.rs | 10 +- crates/types/src/stages/physical/graph.rs | 2 +- .../src/stages/physical/memo_candidates.rs | 38 +- crates/types/src/stages/physical/mod.rs | 2 +- crates/types/src/stages/pruning.rs | 41 +- crates/types/src/test_support.rs | 14 - crates/types/src/workload/bounds.rs | 1 + crates/types/src/workload/mod.rs | 22 +- crates/types/tests/summary_merge_structure.rs | 3 - crates/types/tests/tumbling_window_chain.rs | 220 +--------- docs/design_docs/concepts/post-asap-ir.md | 6 +- .../proposals/asap-primitive-schema.md | 25 +- .../design_docs/proposals/planner-layering.md | 26 +- .../asap-aware-mapping-contracts.md | 15 +- 62 files changed, 475 insertions(+), 1768 deletions(-) rename crates/{logical-asap-optimization => types}/src/deployment_capabilities.rs (97%) diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 7601756e5..02dce919e 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -938,85 +938,24 @@ pub fn node_usage<'a>( ), ) } - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - family, - reduction, - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - // Each input row is retained by the one tumbling window its time falls in, - // as a time range keeps it (one operation), and updates - // that tumbling window. - let group_count = groups(reduction)?; - let states = group_count - .checked_mul(u64::from(*tumbling_window_count)) - .ok_or(AnalyticalCostError::Overflow)?; - // Each tumbling window's state sees its group's rows in one tumbling window: of - // one second's rows at ingestion time, of the rows read at - // query time (all of them for a chain materialized across - // evaluations, whose scan Stage 2 bounds to its newest tumbling window, - // the only one it builds). - let rows_per_tumbling_window = match (ingestion, node.lifetime) { - (true, _) => u64::try_from( - (u128::from(input.rows) * u128::from(*tumbling_window_width_ms)) - .div_ceil(1_000), - ) - .map_err(|_| AnalyticalCostError::Overflow)?, - (false, Lifetime::AcrossEvaluations(_)) => input.rows, - (false, _) => input - .rows - .div_ceil(u64::from(*tumbling_window_count).max(1)), - }; - let input_rows = StateInputRows { - rows_per_state: Some(rows_per_tumbling_window.div_ceil(group_count.max(1))), - non_negative: raw_data_samples_non_negative(), - }; - let (depth, state_bytes) = summary_shape(family, input_rows); - let state_bytes = measured_state_bytes.unwrap_or(state_bytes); - let out = EdgeStatistics { - rows: states, - bytes: states - .checked_mul(state_bytes) - .ok_or(AnalyticalCostError::Overflow)?, - }; - ( - out, - Ok(ResourceEstimate::new( - input.rows as f64 * (depth + 1.0), - out.bytes, - 0, - )), - format!( - "build {} in {tumbling_window_count} tumbling windows of {tumbling_window_width_ms} ms for \ - {group_count} groups: {} rows x (1 + depth {depth})", - family_name(family), - input.rows - ), - ) + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + todo!("#1207 PR 7: a chain's states and memory over the windows it retains (h_v)") } - Payload::ASAP(ASAPOp::SummaryMerge { - newest_tumbling_windows, - group_by, - .. - }) => { + Payload::ASAP(ASAPOp::SummaryMerge { group_by, .. }) => { // One operation per input state read; one state per group - // out, each tumbling window of a chain being a state of its group. A - // merge of a chain's newest `n` tumbling windows reads `n` per group. A - // rollup's groups are those of its keys, at most its input - // groups. + // out. A rollup's groups are those of its keys, at most its + // input groups. let mut group_count = 0; for producer in producers { - let tumbling_windows = match producer.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => u64::from(*tumbling_window_count), - _ => 1, - }; - group_count = group_count.max(producer.output.rows / tumbling_windows.max(1)); + if matches!( + producer.payload, + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) { + todo!("#1207 PR 7: the states a merge reads from a chain's windows") + } + group_count = group_count.max(producer.output.rows); } - let states = merged_states(*newest_tumbling_windows, producers)?; + let states = merged_states(producers)?; if let Some(keys) = group_by { group_count = group_count.min(groups(&Reduction::by(keys.clone()))?); } @@ -1071,10 +1010,7 @@ pub fn node_usage<'a>( // reads, otherwise the rows of its (first) input. let units = match &node.payload { Payload::NonASAP(NonASAPOp::Scan { .. }) => out.rows, - Payload::ASAP(ASAPOp::SummaryMerge { - newest_tumbling_windows, - .. - }) => merged_states(*newest_tumbling_windows, producers)?, + Payload::ASAP(ASAPOp::SummaryMerge { .. }) => merged_states(producers)?, _ => input.rows, }; let mut detail = detail; @@ -1387,16 +1323,10 @@ fn ingest_workers( (ingest_ms_per_s / 1_000.0).ceil().max(1.0) as u64 } -/// For each tumbling-window chain of `dag` materialized beyond one evaluation (at -/// ingestion time, or at query time across evaluations), how far back -/// from the newest evaluation its completed tumbling windows must be retained, in -/// milliseconds: how long before the evaluation its newest tumbling window ends (the -/// offset of its input, `chain_input_offset_ms`), plus its window -/// `tumbling_window_count · tumbling_window_width_ms`, plus the longest -/// `span_ms` ([`RangeEvaluation`]) among the queries reaching it, -/// whose earliest step reads that much further back. `reach` is -/// [`query_reach`] of `dag`; `range_evaluations[q]` is query `q`'s (none -/// when missing). Without range evaluations, each chain's window. +/// For each tumbling-window chain of `dag` materialized across +/// evaluations, how far back from the newest evaluation its completed +/// windows must be retained, in milliseconds. `reach` is [`query_reach`] of +/// `dag`; `range_evaluations[q]` is query `q`'s (none when missing). pub fn tumbling_window_retention_ms( dag: &PhysicalASAPDAG, reach: &[HashSet], @@ -1407,56 +1337,17 @@ pub fn tumbling_window_retention_ms( .iter() .filter_map(|node| match (&node.payload, node.lifetime) { ( - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - tumbling_window_count, - .. - }), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), Lifetime::AcrossEvaluations(_), ) => { - let span_ms = reach - .iter() - .enumerate() - .filter(|(_, reached)| reached.contains(&node.id)) - .filter_map(|(q, _)| range_evaluations.get(q).copied().flatten()) - .map(|range| range.span_ms) - .max() - .unwrap_or(0); - let end_offset_ms = chain_input_offset_ms(*child, &nodes); - Some(( - node.id, - end_offset_ms - + tumbling_window_width_ms * u64::from(*tumbling_window_count) - + span_ms, - )) + let _ = (&nodes, reach, range_evaluations); + todo!("#1207 PR 7: a chain's retention, max(offset + range) over the paths that read it") } _ => None, }) .collect() } -/// How long before the evaluation a chain's newest tumbling window ends: the offsets -/// of the `TimeShift`s (without an `@` anchor) directly below the chain, -/// added up; 0 for a negative total. -fn chain_input_offset_ms( - input: PhysicalASAPNodeId, - nodes: &HashMap, -) -> u64 { - let mut offset_ms: i64 = 0; - let mut current = input; - while let Some(Payload::NonASAP(NonASAPOp::TimeShift { shift, child })) = - nodes.get(¤t).map(|node| &node.payload) - { - if shift.at.is_some() { - break; - } - offset_ms = offset_ms.saturating_add(shift.offset_ms); - current = *child; - } - u64::try_from(offset_ms).unwrap_or(0) -} - /// The tumbling-window retention of every materialized tumbling-window chain of `dag` /// ([`tumbling_window_retention_ms`] over [`query_reach`]), for a plan whose query /// `q` has range evaluation `range_evaluations[q]` (none when missing): @@ -1469,26 +1360,12 @@ pub fn tumbling_window_retention_ms_of_plan( } /// The input states a `SummaryMerge` over `producers` reads: every row of -/// each, or of the one tumbling-window chain whose newest `newest_tumbling_windows` it merges, -/// that many tumbling windows' states per group. -fn merged_states( - newest_tumbling_windows: Option, - producers: &[ProducerOutput<'_>], -) -> Result { +/// each. +fn merged_states(producers: &[ProducerOutput<'_>]) -> Result { producers.iter().try_fold(0_u64, |total, producer| { - let rows = match (newest_tumbling_windows, producer.payload) { - ( - Some(count), - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }), - ) => (producer.output.rows / u64::from(*tumbling_window_count).max(1)) - .checked_mul(u64::from(count)) - .ok_or(AnalyticalCostError::Overflow)?, - _ => producer.output.rows, - }; - total.checked_add(rows).ok_or(AnalyticalCostError::Overflow) + total + .checked_add(producer.output.rows) + .ok_or(AnalyticalCostError::Overflow) }) } @@ -1513,10 +1390,9 @@ pub fn retained_tumbling_windows( /// A node's tumbling windows: a tumbling-window chain's count, 1 for any other node. pub fn tumbling_windows_of(node: &asap_types::ir::physical_export::PhysicalASAPDAGNode) -> u64 { match &node.payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => u64::from(*tumbling_window_count).max(1), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + todo!("#1207 PR 7: the windows a chain retains (h_v)") + } _ => 1, } } @@ -1557,12 +1433,8 @@ pub fn maintained_window_ms( producers_of: &HashMap>, ) -> Option { match &nodes[&node].payload { - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - return Some(tumbling_window_width_ms.saturating_mul(u64::from(*tumbling_window_count))) + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + todo!("#1207 PR 7: the window a chain maintains, its retention (h_v)") } Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => { return Some(range.as_millis() as u64) @@ -1648,11 +1520,9 @@ fn widest_window_read_ms( tumbling_window_width_ms, .. }) if materialized_chains.contains(&edge.consumer) => *tumbling_window_width_ms, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => tumbling_window_width_ms.saturating_mul(u64::from(*tumbling_window_count)), + Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( + "#1207 PR 7: the window a chain that is not materialized reads, from the TimeRange over it" + ), Payload::NonASAP(NonASAPOp::TimeShift { .. }) => { stack.push(edge.consumer); continue; diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 2e9d883b5..a236e5ecf 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -108,9 +108,9 @@ use asap_devtools::planning_stages::{ use asap_devtools::planning_stages_text::planning_stages_text; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::properties::{default_metric_for_root, AccuracyTarget}; use asap_types::ir::QueryRoot; @@ -577,21 +577,37 @@ fn lowered_together( } /// The arrival of two files' data planned together: ingesting (or mixed) -/// at the sum of their rates; the same arrival when both are at rest or -/// unknown; `None` when they differ. +/// at the sum of their rates, as late as the later of their out-of-order +/// bounds; the same arrival when both are at rest or unknown; `None` when +/// they differ. fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option { match (earlier, later) { ( - DataArrival::Ingesting { rows_per_second: a }, - DataArrival::Ingesting { rows_per_second: b }, + DataArrival::Ingesting { + rows_per_second: a, + max_out_of_order_ms: late_a, + }, + DataArrival::Ingesting { + rows_per_second: b, + max_out_of_order_ms: late_b, + }, ) => Some(DataArrival::Ingesting { rows_per_second: a + b, + max_out_of_order_ms: late_a.max(late_b), + }), + ( + DataArrival::Mixed { + rows_per_second: a, + max_out_of_order_ms: late_a, + }, + DataArrival::Mixed { + rows_per_second: b, + max_out_of_order_ms: late_b, + }, + ) => Some(DataArrival::Mixed { + rows_per_second: a + b, + max_out_of_order_ms: late_a.max(late_b), }), - (DataArrival::Mixed { rows_per_second: a }, DataArrival::Mixed { rows_per_second: b }) => { - Some(DataArrival::Mixed { - rows_per_second: a + b, - }) - } (DataArrival::AtRest, DataArrival::AtRest) => Some(DataArrival::AtRest), (DataArrival::Unknown, DataArrival::Unknown) => Some(DataArrival::Unknown), _ => None, @@ -650,7 +666,7 @@ fn stage2_pruned_candidates_json(stage2: Option<&PhysicalASAPMemoCandidates>) -> return Vec::new(); }; let _ = &stage2.pruned_candidates; - todo!("#1207 PR 9 (#1205): each pruned PhysicalCandidateKey with its reason") + todo!("#1207 PR 9 (#1205): each pruned PhysicalCandidate with its reason") } /// One plan's cost: per second, its predicted CPU, unit and source, its @@ -741,7 +757,7 @@ fn selection_json(selection: &Result) -> Value { "pruned_candidates": selection .pruned_candidates .iter() - .map(|p| json!({ "group": p.group.0, "candidate": p.candidate.0, "reason": format!("{:?}", p.reason) })) + .map(|p| json!({ "candidate": p.candidate.0, "reason": format!("{:?}", p.reason) })) .collect::>(), "search_report": selection.search_report, }) diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 8cf029620..6351d71e0 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -156,14 +156,13 @@ pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String } }; let label = match &node.payload { - // Name the chain: chains of one family and tumbling-window count differ by - // their queries and tumbling-window width. + // Name the chain: chains of one family differ by their queries + // and tumbling-window width. Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { tumbling_window_width_ms, - tumbling_window_count, .. }) => format!( - "{}, {tumbling_window_count} × {} tumbling windows", + "{}, {} tumbling windows", for_queries( state_label(&node.payload), &dag_readers(&candidate.dag, *id) @@ -187,17 +186,11 @@ pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String .join(" · ") } -/// A summary build, e.g. "Kll ×5 tumbling windows" or "Sum acc". +/// A summary build, e.g. "Kll tumbling windows" or "Sum acc". fn build_label(payload: &Operator) -> String { match payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => { - format!( - "{} ×{tumbling_window_count} tumbling windows", - state_label(payload) - ) + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { + format!("{} tumbling windows", state_label(payload)) } _ => state_label(payload), } @@ -356,9 +349,7 @@ fn find_details(value: &Value, details: &mut Vec, depth: usize) { // A time range's length, e.g. `60s`. "range" => details.push(duration_text(field)), // A window, and a tumbling-window chain's layout. - "window" | "tumbling_window_width_ms" | "tumbling_window_count" => { - details.push(format!("{key}={field}")) - } + "window" | "tumbling_window_width_ms" => details.push(format!("{key}={field}")), _ => find_details(field, details, depth + 1), } } diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index c40a2ef4a..da1b33e5c 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -41,8 +41,8 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use asap_cost::CostInputs; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_planner::{PlanSearchSettings, PlanningModels}; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::sketch::SketchAlgorithm; use asap_types::ir::QueryRoot; @@ -168,7 +168,10 @@ impl FuzzWorkload { } } if rows_per_second > 0.0 { - self.data.arrival = DataArrival::Ingesting { rows_per_second }; + self.data.arrival = DataArrival::Ingesting { + rows_per_second, + max_out_of_order_ms: None, + }; } } diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index 3bc274793..f1ede01a7 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -1142,7 +1142,6 @@ pub fn witness_root( .collect(); let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![state], - newest_tumbling_windows: None, group_by: Some(group_by), })) .map_err(|e| format!("rollup merge: {e}"))?; @@ -1153,45 +1152,10 @@ pub fn witness_root( .map_err(|e| format!("rollup estimate: {e}"))?; with_root_estimate(q_root, estimate, &dropped) } - Witness::NewestTumblingWindows { window_ms } => rewrite_root( - q_root, - "summary merge over a tumbling-window chain", - |node| { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by, - }) = node.asap() - else { - return None; - }; - let [chain] = children.as_slice() else { - return None; - }; - let Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = chain.asap() - else { - return None; - }; - let tumbling_windows = window_ms / tumbling_window_width_ms; - (window_ms % tumbling_window_width_ms == 0 - && tumbling_windows >= 1 - && (tumbling_windows as u32) < *tumbling_window_count) - .then(|| { - with_operator( - node, - Operator::ASAP(ASAPOp::SummaryMerge { - children: children.clone(), - newest_tumbling_windows: Some(tumbling_windows as u32), - group_by: group_by.clone(), - }), - ) - }) - }, - ), + Witness::NewestTumblingWindows { window_ms } => { + let _ = window_ms; + todo!("#1207 PR 9: a witness merging the windows a TimeRange over q's chain selects") + } Witness::QuantileRank { from, to } => { let mut done = false; rewrite_root(q_root, "quantile estimate", |node| { @@ -1286,7 +1250,6 @@ pub fn witness_root( // selections. let merge = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![state, other_state], - newest_tumbling_windows: None, group_by: None, })) .map_err(|e| format!("union merge: {e}"))?; @@ -1421,130 +1384,15 @@ pub fn witness_roots( Ok(roots) } -/// The first node under `node` (depth first, `node` included) that `pred` -/// accepts. -fn find_first( - node: &Rc, - pred: &dyn Fn(&OperatorNode) -> bool, -) -> Option> { - if pred(node) { - return Some(node.clone()); - } - node.operator - .children() - .into_iter() - .find_map(|child| find_first(child, pred)) -} - -/// The distinct nodes under `node` (`node` included) that `pred` accepts. -fn collect( - node: &Rc, - pred: &dyn Fn(&OperatorNode) -> bool, - found: &mut Vec>, -) { - if found.iter().any(|f| Rc::ptr_eq(f, node)) { - return; - } - if pred(node) { - found.push(node.clone()); - } - for child in node.operator.children() { - collect(child, pred, found); - } -} - -/// `W`'s roots with the tumbling-window chain `q` merges whole lengthened to cover -/// `window_ms`, every merge of it reading its old newest tumbling windows of the -/// longer chain; and `q`'s root merging the whole longer chain. +/// `W`'s roots with the reading of `q`'s tumbling-window chain lengthened +/// to cover `window_ms`, and `q`'s root reading the longer window. fn longer_chain_roots( base_roots: &[QueryRoot], target: usize, window_ms: u64, ) -> Result<(Vec, Rc), String> { - let q_root = base_roots[target] - .as_operator() - .ok_or("q is a scalar query")?; - let whole_chain_merge = |node: &OperatorNode| { - matches!( - node.asap(), - Some(ASAPOp::SummaryMerge { children, newest_tumbling_windows: None, .. }) - if matches!(children.as_slice(), [chain] if matches!(chain.asap(), Some(ASAPOp::SummaryTumblingWindowChain { .. }))) - ) - }; - let merge = find_first(q_root, &whole_chain_merge) - .ok_or("q's plan has no summary merge over a tumbling-window chain")?; - // Lengthening one of several chains (say, an SQL idiom's exact row - // count beside its statistic) would leave the others at q's window. - let mut merges = Vec::new(); - collect(q_root, &whole_chain_merge, &mut merges); - if merges.len() > 1 { - return Err(format!( - "q's plan merges {} tumbling-window chains whole", - merges.len() - )); - } - let Some(ASAPOp::SummaryMerge { children, .. }) = merge.asap() else { - unreachable!("found a merge"); - }; - let chain = children[0].clone(); - let mut operator = chain.operator.clone(); - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) = &mut operator - else { - unreachable!("found a tumbling-window chain"); - }; - let old_count = *tumbling_window_count; - if !window_ms.is_multiple_of(*tumbling_window_width_ms) - || window_ms / *tumbling_window_width_ms <= old_count as u64 - { - return Err(format!( - "q's chain of {old_count} tumbling windows of {tumbling_window_width_ms} ms does not extend to {window_ms} ms" - )); - } - *tumbling_window_count = (window_ms / *tumbling_window_width_ms) as u32; - let long_chain = with_operator(&chain, operator); - let reread = |node: &Rc, whole: bool| { - let Some(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows, - group_by, - }) = node.asap() - else { - return None; - }; - let [child] = children.as_slice() else { - return None; - }; - Rc::ptr_eq(child, &chain).then(|| { - with_operator( - node, - Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![long_chain.clone()], - newest_tumbling_windows: if whole { - None - } else { - Some(newest_tumbling_windows.unwrap_or(old_count)) - }, - group_by: group_by.clone(), - }), - ) - }) - }; - let mut cache = HashMap::new(); - let roots = base_roots - .iter() - .map(|root| match root.as_operator() { - Some(node) => { - rewrite(node, &mut |n| reread(n, false), &mut cache).map(QueryRoot::Operator) - } - None => Ok(root.clone()), - }) - .collect::, _>>()?; - let long_root = rewrite(q_root, &mut |n| reread(n, true), &mut HashMap::new())?; - Ok((roots, long_root)) + let _ = (base_roots, target, window_ms); + todo!("#1207 PR 9: lengthen the TimeRange over q's chain, which has no window count") } fn planned( @@ -1908,16 +1756,20 @@ pub fn disjoint_copy(workload: &FuzzWorkload) -> Option { both.data.sources.extend(copy.data.sources); both.tables.extend(copy.tables); both.data.arrival = match workload.data.arrival { - asap_types::workload::DataArrival::Ingesting { rows_per_second } => { - asap_types::workload::DataArrival::Ingesting { - rows_per_second: 2.0 * rows_per_second, - } - } - asap_types::workload::DataArrival::Mixed { rows_per_second } => { - asap_types::workload::DataArrival::Mixed { - rows_per_second: 2.0 * rows_per_second, - } - } + asap_types::workload::DataArrival::Ingesting { + rows_per_second, + max_out_of_order_ms, + } => asap_types::workload::DataArrival::Ingesting { + rows_per_second: 2.0 * rows_per_second, + max_out_of_order_ms, + }, + asap_types::workload::DataArrival::Mixed { + rows_per_second, + max_out_of_order_ms, + } => asap_types::workload::DataArrival::Mixed { + rows_per_second: 2.0 * rows_per_second, + max_out_of_order_ms, + }, other => other, }; Some(both) diff --git a/crates/devtools/src/planner_layering_examples.rs b/crates/devtools/src/planner_layering_examples.rs index f4ac75793..cb2c18941 100644 --- a/crates/devtools/src/planner_layering_examples.rs +++ b/crates/devtools/src/planner_layering_examples.rs @@ -72,6 +72,7 @@ pub fn example1() -> PlanningWorkload { let mut data = shared_data_workload( DataArrival::Ingesting { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, EXAMPLE1_METRIC, ); @@ -141,6 +142,7 @@ pub fn example2() -> (PlanningWorkload, asap_frontend_sql::SqlCatalog) { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000_000.0 / 15.0, + max_out_of_order_ms: None, }, sources: [( "flows".to_string(), @@ -305,6 +307,7 @@ pub fn pattern_b() -> PlanningWorkload { data_workload: Some(shared_data_workload( DataArrival::Ingesting { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, PATTERN_METRIC, )), @@ -322,9 +325,11 @@ mod tests { DataArrival::AtRest, DataArrival::Ingesting { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, DataArrival::Mixed { rows_per_second: SHARED_ROWS_PER_SECOND, + max_out_of_order_ms: None, }, ] { shared_data_workload(arrival, PATTERN_METRIC) diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 9efdd4fd9..5c5c65dbf 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -344,19 +344,8 @@ pub fn run_stages_until( return Ok(stages); } let logical = run.logical_asap_candidates.as_ref().expect("set above"); - let physical = match generate_physical_asap_memo_candidates( - logical, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - models.deployment_capabilities.raw_data_retained, - ) { - Ok(physical) => physical, - Err(error) => { - run.selection = Err(SelectionError::PhysicalASAPCandidates(error.to_string())); - return Ok(stages); - } - }; + let physical = + generate_physical_asap_memo_candidates(logical, data, models.deployment_capabilities); if until == RunUntil::Stage2 { run.physical_asap_candidates = Some(physical); return Ok(stages); diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index e136e4e9c..30478520d 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -17,7 +17,7 @@ //! G1: TimeRange 60s <- G0 //! G2: Aggregate rate() <- G1 //! [0] Pass1 FinalizeExactAccumulator ← SummaryAgg inputs=[G1] -//! [1] TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_count=6 tumbling_window_width_ms=10000 inputs=[G0] +//! [1] TumblingWindows new FinalizeExactAccumulator ← SummaryMerge ← SummaryTumblingWindowChain tumbling_window_width_ms=10000 inputs=[G0] //! ``` use std::fmt::Write; diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index c09f75a24..98f51f3da 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -1131,7 +1131,7 @@ fn workload_set_index_and_files_validate() { ); } if let Some(capabilities) = set.get("deployment_capabilities") { - serde_json::from_value::( + serde_json::from_value::( capabilities.clone(), ) .unwrap(); diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index 559223d2e..df73a824d 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -200,7 +200,6 @@ fn asap_samples() -> Vec> { }, ASAPOp::SummaryMerge { children: vec![], - newest_tumbling_windows: None, group_by: None, }, ASAPOp::SummaryTumblingWindowChain { @@ -211,7 +210,6 @@ fn asap_samples() -> Vec> { grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: 60_000, - tumbling_window_count: 5, }, ASAPOp::SummarySubtract { left: (), diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 8c9afa66a..5c527992f 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -87,19 +87,13 @@ fn compile_internal( roots: &[NodeId], ) -> Result { preflight_depth(dag)?; - // A tumbling-window chain stands for its tumbling windows; the deployment binds each tumbling window's - // raw input, so it runs only once expanded. - if let Some(node) = dag.nodes.iter().find(|n| { + if dag.nodes.iter().any(|n| { matches!( n.payload, Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) ) }) { - return Err(invalid(format!( - "node {}: a SummaryTumblingWindowChain runs as its tumbling windows; expand it with \ - `asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains` before compiling", - node.id - ))); + todo!("#1207 PR 6: run a chain, and a TimeRange or TimeShift over it by window index") } // Helper IDs lie above the u32 range; a larger Planner ID could collide. if let Some(node) = dag.nodes.iter().find(|n| n.id as u64 > u64::from(u32::MAX)) { diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index 31187af65..a567eff37 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -61,6 +61,7 @@ fn run_all_stages( let data = DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, ..Default::default() }; diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index fc23257c5..9fa2904b5 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -191,7 +191,6 @@ fn try_run( 2, Payload::ASAP(ASAPOp::SummaryMerge { children: vec![1], - newest_tumbling_windows: None, group_by: None, }), (*state).clone(), diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 50cef473b..cf72343ec 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -134,7 +134,6 @@ fn try_state_dag( id: 1, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryMerge { children: vec![0], - newest_tumbling_windows: None, group_by: None, }), output_state: ExecutionDataState::INGESTION_SUMMARY, diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 946e786fb..3b455e972 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -340,7 +340,6 @@ fn merge_over_a_stored_tumbling_window( 1, Payload::ASAP(ASAPOp::SummaryMerge { children: vec![0], - newest_tumbling_windows: None, group_by: None, }), schema, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 3e64a23c3..e14f47ff7 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -258,7 +258,6 @@ fn summary_merge_without_inputs_is_rejected() { id: 0, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryMerge { children: vec![], - newest_tumbling_windows: None, group_by: None, }), output_schema: schema, diff --git a/crates/frontend-metricsql/tests/lowering_semantics.rs b/crates/frontend-metricsql/tests/lowering_semantics.rs index 3a1eb7842..0db792586 100644 --- a/crates/frontend-metricsql/tests/lowering_semantics.rs +++ b/crates/frontend-metricsql/tests/lowering_semantics.rs @@ -131,6 +131,7 @@ fn workload_lowering_uses_the_declared_sample_interval() { let mut other_metric = workload(QueryLanguage::MetricsQL, "sum(up) + sum(down)"); other_metric.data_workload.as_mut().unwrap().arrival = DataArrival::Ingesting { rows_per_second: 1.0, + max_out_of_order_ms: None, }; for workload in [missing, other_metric] { assert!(matches!( diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index e6668bf66..38f9db6b0 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -252,6 +252,7 @@ mod tests { Some(DataArrival::Unknown), Some(DataArrival::Ingesting { rows_per_second: 1.0, + max_out_of_order_ms: None, }), None, ]; diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index d6a3912e1..38be69839 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -131,6 +131,7 @@ pub fn declared_data(root: &Rc) -> DataWorkload { DataWorkload { arrival: asap_types::workload::DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, sources, } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index d145c8228..aa8fe4680 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -146,6 +146,7 @@ pub mod planning { DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, sources, } diff --git a/crates/integration-tests/tests/joint_search_equivalence.rs b/crates/integration-tests/tests/joint_search_equivalence.rs index edd0dd2a3..3099cc6e7 100644 --- a/crates/integration-tests/tests/joint_search_equivalence.rs +++ b/crates/integration-tests/tests/joint_search_equivalence.rs @@ -10,8 +10,8 @@ use crate::planner_layering_common; use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel}; use asap_cost::{ClusterLimits, ClusterObjective, CostInputs, CostModel}; use asap_devtools::planning_stages::{lower_promql_roots, lower_sql_roots}; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; use asap_types::ir::QueryRoot; use asap_types::workload::{ @@ -44,6 +44,7 @@ fn workload_of(queries: &[(&str, u32)], accuracy: AccuracyTarget) -> PlanningWor data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 24.0, + max_out_of_order_ms: None, }, sources: ["m", "n", "c_total"] .into_iter() diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index 217f15cfe..dffe01447 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -97,7 +97,6 @@ fn fixture() -> Fixture { .collect(); let merge = new(asap_types::ir::Operator::ASAP(ASAPOp::SummaryMerge { children: tumbling_windows.clone(), - newest_tumbling_windows: None, group_by: None, })); let estimate = |q| { diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 283b0f173..9c53b8ac9 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -17,7 +17,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use asap_cost::CostInputs; use asap_devtools::planning_stages::{lower_promql_roots, run_planning_stages_over}; -use asap_logical_asap_optimization::DeploymentCapabilities; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::{flatten, FlatDag, NodeId}; use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::properties::ExecutionTiming; @@ -439,49 +439,3 @@ pub fn window_form(dag: &impl ExportedDag, build: NodeId) -> WindowForm { _ => WindowForm::None, } } - -/// How far back from an evaluation a materialized node's materialized output -/// reaches in event time, derived the way plan selection prices what it -/// keeps: a tumbling-window chain keeps its `N` -/// tumbling windows of width `w`, plus the one being built when maintained at -/// ingestion time; a whole-window build keeps its window, and the next one -/// being built at ingestion time; each plus the offset of the time shifts -/// below it. `None` for any other node. -pub fn retention_ms(p: &Physical, node: NodeId) -> Option { - let dag = p.dag(); - let at_ingestion = runs_at_ingestion(p, node); - let below = |id: NodeId| dag.producers(id).into_iter().next(); - let offset_below = |mut id: Option| { - let mut offset = 0u64; - while let Some(at) = id { - match dag.payload(at) { - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { - offset += shift.offset_ms.unsigned_abs() - } - Operator::NonASAP(NonASAPOp::Scan { .. }) => break, - _ => {} - } - id = below(at); - } - offset - }; - match dag.payload(node) { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - }) => { - let tumbling_windows = u64::from(*tumbling_window_count) + u64::from(at_ingestion); - Some(tumbling_windows * tumbling_window_width_ms + offset_below(below(node))) - } - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => { - let child = below(node)?; - let Operator::NonASAP(NonASAPOp::TimeRange { range, .. }) = dag.payload(child) else { - return None; - }; - let windows = 1 + u64::from(at_ingestion); - Some(windows * range.as_millis() as u64 + offset_below(below(child))) - } - _ => None, - } -} diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs index a49628beb..f2c173214 100644 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ b/crates/integration-tests/tests/sketch_rollup.rs @@ -214,7 +214,6 @@ fn rolled_up(coarse: &OperatorNode) -> Rc { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![finer], group_by: Some(vec![1]), - newest_tumbling_windows: None, })) .unwrap() } @@ -349,7 +348,6 @@ async fn invalid_rollups_are_rejected() { OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![Rc::clone(&finer)], group_by: Some(keys), - newest_tumbling_windows: None, })) .map_err(|error| error.to_string()) }; @@ -384,7 +382,6 @@ async fn invalid_rollups_are_rejected() { let merged_rates = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![rates], group_by: Some(vec![1]), - newest_tumbling_windows: None, })); assert!(merged_rates .unwrap_err() diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index a94771df1..c4b950e79 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -41,8 +41,6 @@ //! - `pass1::realization` — the summary families that can serve an aggregate //! intent, the intent's accuracy target and `(ε, δ)` budget, and default //! sketch sizing, shared by Pass 1 and Pass 2. -//! - `deployment_capabilities` — the deployment's supported summaries -//! ([`DeploymentCapabilities`]). //! //! Accuracy guarantees come from the built-in analytical accuracy rules in //! `asap-accuracy` (per-family estimators, composition and budget @@ -58,8 +56,6 @@ //! consumers; plan selection (Stage 3) checks each selected candidate's //! guarantee against its query's `AccuracyTarget` and never resizes. -/// The deployment's capabilities: supported summaries and what it retains. -mod deployment_capabilities; mod pass1; mod pass2; /// Stage 0: the memo groups of the workload's query roots. @@ -73,9 +69,6 @@ pub mod test_support; // developer tools. Every other item is crate-private; the ones other crates' // tests call are re-exported by `test_support`. -pub use deployment_capabilities::{ - DeploymentCapabilities, DeploymentCapabilitiesError, SupportedSummary, -}; // Stage 0: the memo groups of the workload's query roots, without // candidates (`asap_types::stages::Memo`). pub use stage0::memo_of_queries; diff --git a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs index 066f41724..58c8a80a4 100644 --- a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs @@ -8,10 +8,10 @@ use asap_types::workload::DataWorkload; use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; -use crate::deployment_capabilities::SupportedSummary; use crate::pass1::per_computation_candidates::{ generate_per_computation_candidates, PerComputationCandidateError, }; +use asap_types::deployment_capabilities::SupportedSummary; use asap_types::stages::Memo; /// Why [`pass1_memo`] failed. diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 96c7f469d..34e7cc85d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -129,8 +129,8 @@ use asap_types::ir::properties::AccuracyGuarantee; use asap_types::workload::bounds::{declared_non_negative, series_read, values_per_result}; use asap_types::workload::DataWorkload; -use crate::deployment_capabilities::{summary_supported, SupportedSummary}; use crate::pass1::realization::summary_candidates; +use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; use asap_types::stages::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, Memo, MemoGroupId, @@ -3570,14 +3570,12 @@ mod tests { grouping: grouping.clone(), filter: filter.clone(), tumbling_window_width_ms: 10_000, - tumbling_window_count: 10, })) .unwrap(), ); let merged = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![chain], - newest_tumbling_windows: None, group_by: None, })) .unwrap(), diff --git a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs index 62594119f..a3b660450 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs @@ -1,18 +1,21 @@ -//! Rule 1, local window composition: tumbling-window layouts $\omega_v$ as -//! candidates of a summary state group (#1160). +//! Rule 1, chain groups (concepts doc, section 3, "Chain groups"): a +//! `SummaryTumblingWindowChain[w](X)` group per width, and a `SummaryMerge` +//! over the `TimeRange` (and `TimeShift`s) over it as a candidate of the +//! summary state group. use crate::pass2::memo::rules::Pass2RuleError; use asap_types::stages::Memo; use asap_types::workload::QueryWorkloadEntry; -/// Rule 1: for each summary state group, a candidate per tumbling width $w$ -/// from the gcd closure of the ranges and steps of the queries that reach it, -/// offered only when $w$ divides the group's query range. `entries[i]` is -/// query `i`'s workload entry. +/// Rule 1: for each summary state group that survived dead-candidate +/// elimination, a chain group and a merge candidate per tumbling width $w$ +/// from the gcd closure of the evaluation grids, ranges and offsets of the +/// queries that reach it and repeat on a known grid. `entries[i]` is query +/// `i`'s workload entry. pub(crate) fn add_local_window_composition( memo: &mut Memo, entries: &[QueryWorkloadEntry], ) -> Result<(), Pass2RuleError> { let _ = (memo, entries); - todo!("#1207 PR 4: rule 1, tumbling layouts as candidates of the state group") + todo!("#1207 PR 4: rule 1, chain groups and the merges over them") } diff --git a/crates/physical-asap-optimization/src/catalog/mod.rs b/crates/physical-asap-optimization/src/catalog/mod.rs index d278afef4..e8c35a9be 100644 --- a/crates/physical-asap-optimization/src/catalog/mod.rs +++ b/crates/physical-asap-optimization/src/catalog/mod.rs @@ -1,5 +1,5 @@ //! Stage 2's physical candidates: expand each logical candidate into one -//! physical candidate per valid $(\tau, \ell, \sigma)$, and each logical edge +//! physical candidate per valid $(\tau, \ell)$, and each logical edge //! into the physical edges between the two candidates' alternatives //! ([`generate_physical_asap_memo_candidates`]); and realize one selected //! DAG as a physical plan (#1047). The output type, @@ -9,37 +9,24 @@ mod realize; pub use realize::Stage2AssignmentRealizer; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; +use asap_types::workload::DataWorkload; -/// Why [`generate_physical_asap_memo_candidates`] failed. -#[derive(Debug, thiserror::Error)] -pub enum ErrorGeneratingPhysicalASAPMemoCandidates { - /// Structural pruning found the candidates malformed. - #[error("{0}")] - Pruning(String), -} - -/// Stage 2: expand every logical candidate of `logical` into its physical -/// alternatives (decision-tree levels 4–6: timing, lifetime, schedule) the -/// data and deployment allow (`query_time_results_retained`; no query-time -/// raw scans unless `raw_data_retained`), and every logical edge into its -/// physical edges with $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$. Every -/// alternative not generated is recorded in the output's -/// `pruned_candidates` with its `PruneReason`. Prices nothing. +/// Stage 2: expand every logical candidate of `logical` that survived Pass 2 +/// into its physical alternatives (decision-tree levels 3–4: timing and +/// lifetime) the data and the deployment's `capabilities` allow, and every +/// logical edge into its physical edges with $\alpha^{\log}$ and +/// $\alpha^{\mathrm{phys}}$. Every alternative rejected is recorded in the +/// output's `pruned_candidates` with its `PruneReason`; a value outside a +/// dimension's declared value set (`AcrossEvaluations` on a candidate that +/// is not a chain) is not offered and not recorded. Costs nothing, and +/// cannot fail. pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, - entries: &[QueryWorkloadEntry], data: &DataWorkload, - query_time_results_retained: bool, - raw_data_retained: bool, -) -> Result { - let _ = ( - logical, - entries, - data, - query_time_results_retained, - raw_data_retained, - ); + capabilities: &DeploymentCapabilities, +) -> PhysicalASAPMemoCandidates { + let _ = (logical, data, capabilities); todo!("#1207 PR 5: physical expansion and physical edges") } diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index 252aecf16..24b02fb14 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -4,7 +4,7 @@ //! The planner facade calls [`generate_physical_asap_memo_candidates`] on //! Pass 2's output (`asap_types::stages::LogicalASAPMemoCandidates`): each //! logical candidate expanded into one physical candidate per valid -//! $(\tau, \ell, \sigma)$, each logical edge into its physical edges with +//! $(\tau, \ell)$, each logical edge into its physical edges with //! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives //! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from //! that output, and [`Stage2AssignmentRealizer`] builds the selected DAG as a @@ -12,7 +12,4 @@ mod catalog; -pub use catalog::{ - generate_physical_asap_memo_candidates, ErrorGeneratingPhysicalASAPMemoCandidates, - Stage2AssignmentRealizer, -}; +pub use catalog::{generate_physical_asap_memo_candidates, Stage2AssignmentRealizer}; diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index a2dcf5f48..96003283a 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -26,8 +26,8 @@ use std::collections::{HashMap, HashSet}; use asap_cost::{NodePriceSource, OperatorKind}; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_plan_selection::{SearchReport, Selection}; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, }; @@ -37,7 +37,6 @@ use asap_types::ir::physical_export::{ PhysicalASAPNodeId, }; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::SchemaDerivationError; use asap_types::stages::RealizedPlan; use asap_types::workload::QueryWorkloadEntry; @@ -153,9 +152,6 @@ pub struct PredictedQueryUsage { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum InstallablePlanError { - /// Internal bug: a tumbling-window chain of the selected plan does not expand. - #[error("tumbling-window chain expansion: {0}")] - TumblingWindowChainExpansion(#[from] SchemaDerivationError), /// Bad input (a plan file): written for another plan-file format; `None` /// for a file without `wire_version`, written before version 11. #[error("unsupported installable plan wire version {0:?}")] diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index 3aa06b879..364281454 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -51,16 +51,18 @@ pub use asap_cost::{ ResourceSpeed, ResourceUnitCost, SummaryOperation, }; pub use asap_frontend_sql::SqlCatalog; -pub use asap_logical_asap_optimization::{DeploymentCapabilities, SupportedSummary}; pub use asap_plan_selection::{ AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, SearchReport, SearchStatus, Selection, }; +pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; pub use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, ViewCoverage, }; pub use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -pub use asap_types::ir::properties::summary_coverage::{ColumnIdentity, Constraint, SelectionBox}; +pub use asap_types::ir::properties::summary_coverage::{ + ColumnIdentity, Constraint, SelectionBox, TimeBound, +}; pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::ir::schema::{SketchAlgorithm, SketchParams, StatisticKind}; pub use asap_types::workload::PlanningWorkload; diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 539ac72e8..532cf66fa 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -26,9 +26,9 @@ use asap_accuracy::{ NoAccuracyEvidence, }; use asap_cost::CostInputs; -use asap_logical_asap_optimization::DeploymentCapabilities; use asap_logical_asap_optimization::PerComputationCandidateError; use asap_plan_selection::{AccuracyCheckInputs, Selection}; +use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::physical_export::PhysicalASAPNodeId; static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 269ba9405..64dba4aa0 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -10,12 +10,12 @@ use std::rc::Rc; use std::time::Instant; use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; -use asap_logical_asap_optimization::DeploymentCapabilitiesError; use asap_logical_asap_optimization::{pass1_memo, Pass1Error, PerComputationCandidateError}; use asap_physical_asap_optimization::{ generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, }; use asap_plan_selection::{select_plan, ErrorSelectingPlan, SearchReport, Selection}; +use asap_types::deployment_capabilities::DeploymentCapabilitiesError; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::stages::{ @@ -49,9 +49,6 @@ pub enum SelectionError { /// Pass 1's memo candidates or Pass 2 failed. #[error("logical ASAP-aware optimization: {0}")] LogicalASAPCandidates(String), - /// Stage 2 failed. - #[error("physical ASAP-aware optimization: {0}")] - PhysicalASAPCandidates(String), /// Stage 3 selected no plan. #[error("plan selection: {0}")] Selection(ErrorSelectingPlan), @@ -181,17 +178,10 @@ pub fn run_planning( .sum(); // Stage 2. let started = Instant::now(); - let physical = generate_physical_asap_memo_candidates( - logical, - entries, - data, - models.deployment_capabilities.query_time_results_retained, - models.deployment_capabilities.raw_data_retained, - ); + let physical = + generate_physical_asap_memo_candidates(logical, data, models.deployment_capabilities); report.stage2_ms = elapsed_ms(started); - let physical = physical_asap_candidates.insert( - physical.map_err(|error| SelectionError::PhysicalASAPCandidates(error.to_string()))?, - ); + let physical = physical_asap_candidates.insert(physical); report.physical_candidates = physical.physical_candidates.len(); report.stage2_pruned_candidates = physical.pruned_candidates.len(); // Stage 3. diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index f32b355b4..2dcf278b9 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -329,6 +329,7 @@ async fn each_plan_carries_its_selected_root() { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, ..Default::default() }), diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index b8087aab7..09ae0c05b 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -19,12 +19,11 @@ use asap_executor::Error as ExecutorError; use asap_planner::{ plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, MaterializedView, OperatorKind, PhysicalASAPDAG, PlanningModels, PlanningWorkload, - SelectedPlanReport, SelectionBox, UserInput, + SelectedPlanReport, UserInput, }; use asap_types::ir::operator::Source; use futures::{executor::block_on, StreamExt}; -use std::ops::Bound; use time_series::{series_labels, time_series_tables, RawDataSample}; @@ -169,29 +168,9 @@ fn answers(batches: &[Batch]) -> Vec { answers } -/// The control plane: plan `queries` for a deployment that keeps no raw -/// samples, with its cost model read from JSON, and write the plan file. -async fn plan_json(queries: &[&str]) -> String { - // What the deployment keeps: no raw data samples, no query-time results - // across evaluations. Supported summaries are left unrestricted, so the - // Planner offers only what its own executor runs. - let capabilities = DeploymentCapabilities { - raw_data_retained: false, - ..DeploymentCapabilities::default() - }; - plan_json_for(workload(queries), &capabilities).await -} - -/// [`plan_json`] for `workload` on a deployment with `capabilities`. -async fn plan_json_for( - workload: PlanningWorkload, - capabilities: &DeploymentCapabilities, -) -> String { - serde_json::to_string(&plan_with_report_for(workload, capabilities).await.0).unwrap() -} - -/// The plan [`plan_json`] writes for `queries`, and Stage 3's report of the -/// selection it made (which the plan file does not carry). +/// The plan for `queries` on a deployment that keeps no raw samples, and +/// Stage 3's report of the selection it made (which the plan file does not +/// carry). async fn planned_with_report(queries: &[&str]) -> (InstallablePlan, Option) { let capabilities = DeploymentCapabilities { raw_data_retained: false, @@ -339,54 +318,6 @@ async fn an_instant_selector_reads_the_latest_sample_per_series() { // ── Materialized views refreshed from materialized views (issue #781) ────────────────── -/// The time selection `(from_ms, to_ms]` relative to the evaluation. -fn relative_time(from_ms: i64, to_ms: i64) -> Vec { - vec![SelectionBox::new( - Default::default(), - Some((Bound::Excluded(from_ms), Bound::Included(to_ms))), - )] -} - -/// Example 4b's selected tumbling windows cover the hour and share one definition id, -/// each with its own time selection; each reads its tumbling window of -/// raw data samples and nothing stored. -#[tokio::test] -async fn tumbling_windows_of_an_hour_share_one_definition_with_distinct_times() { - let plan: InstallablePlan = - serde_json::from_str(&plan_json(&["quantile_over_time(0.99, latency_ms[60m])"]).await) - .unwrap(); - let views = plan.materialized_views().unwrap(); - assert!(!views.is_empty()); - let mut tumbling_windows: Vec<&MaterializedView> = views.iter().collect(); - tumbling_windows.sort_by_key(|o| o.inputs.raw_data_windows()[0].end_offset_ms()); - let width_ms = tumbling_windows[0].inputs.raw_data_windows()[0].width_ms(); - assert!(width_ms > 0); - assert_eq!( - width_ms * tumbling_windows.len() as i64, - 3_600_000, - "complete hour coverage" - ); - for (i, tumbling_window) in tumbling_windows.iter().enumerate() { - let i = i as i64; - let [window] = tumbling_window.inputs.raw_data_windows() else { - panic!("{tumbling_window:?}"); - }; - assert_eq!( - (window.width_ms(), window.end_offset_ms()), - (width_ms, i * width_ms) - ); - assert!(tumbling_window.inputs.materialized_views.is_empty()); - assert_eq!( - tumbling_window.coverage().definition_id(), - tumbling_windows[0].coverage().definition_id() - ); - assert_eq!( - tumbling_window.coverage().selection(), - relative_time(-(i + 1) * width_ms, -i * width_ms) - ); - } -} - /// The MVP benchmark's p99 query: the per-series p99 of a gauge over 100 s, /// averaged per (region, service, job), every second, within 0.05 at /// failure probability 0.3, over 1,000 `data` series sampled every 100 ms, diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index d4d2dc649..b47d1997e 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -66,6 +66,7 @@ fn promql_workload(queries: &[(&str, f64)]) -> PlanningWorkload { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1.0, + max_out_of_order_ms: None, }, ..DataWorkload::sampled_every( queries diff --git a/crates/logical-asap-optimization/src/deployment_capabilities.rs b/crates/types/src/deployment_capabilities.rs similarity index 97% rename from crates/logical-asap-optimization/src/deployment_capabilities.rs rename to crates/types/src/deployment_capabilities.rs index add7d3ddd..0ba1be478 100644 --- a/crates/logical-asap-optimization/src/deployment_capabilities.rs +++ b/crates/types/src/deployment_capabilities.rs @@ -33,9 +33,9 @@ // The executor lists the families it builds as `SupportedSummary`s, so the // type lives with the readout vocabulary in `asap-types`. -pub use asap_types::ir::schema::SupportedSummary; -use asap_types::ir::schema::{SketchAlgorithm, StatisticKind}; -use asap_types::workload::{DataArrival, DataWorkload}; +pub use crate::ir::schema::SupportedSummary; +use crate::ir::schema::{SketchAlgorithm, StatisticKind}; +use crate::workload::{DataArrival, DataWorkload}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -93,7 +93,7 @@ impl DeploymentCapabilities { /// family and readout the planner can build) can build `family` and answer /// `readout` from it. Pass 1 reads only this field of /// [`DeploymentCapabilities`]. -pub(crate) fn summary_supported( +pub fn summary_supported( supported_summaries: Option<&[SupportedSummary]>, family: &SketchAlgorithm, readout: StatisticKind, diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index d9971be0c..cb5aa9250 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -1097,5 +1097,7 @@ fn coverage_of( coverage_by_node: &HashMap, ) -> Result { let _ = (id, inputs, ir, coverage_by_node); - todo!("#1207 PR 6: the view's coverage from its memo group's GroupProperties::summary_coverage") + todo!( + "#1207 PR 6: the view's coverage from its memo group's GroupProperties::aggregate_coverage" + ) } diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index c4490bb2d..8a2941a24 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -56,21 +56,12 @@ pub enum ASAPOp> { evaluation: PopulationStatistic, }, /// Merge compatible partial states of one family: every input state of - /// a group, including each tumbling window of a [`SummaryTumblingWindowChain`](ASAPOp::SummaryTumblingWindowChain), - /// into one state per group. + /// a group into one state per group, or, with `group_by`, regrouped. + /// Over a time-indexed state (a `TimeRange` over a + /// [`SummaryTumblingWindowChain`](ASAPOp::SummaryTumblingWindowChain)), the windows it + /// selects; there is no window-selection field. SummaryMerge { children: Vec, - /// Merge only the newest `n` tumbling windows of the one `SummaryTumblingWindowChain` this - /// merge reads, `1 ≤ n <` its `tumbling_window_count`: the state of one build - /// over the newest `n · tumbling_window_width_ms` of the chain's window, so a - /// query over a shorter window reads a chain kept for a longer one. - /// `None` merges every input state. - #[serde( - default, - skip_serializing_if = "Option::is_none", - alias = "newest_panes" - )] - newest_tumbling_windows: Option, /// A rollup (regrouping merge): the input's plain columns the /// merged states are keyed by, in output order. Every input state /// whose values in these columns match goes into one output state, @@ -79,19 +70,15 @@ pub enum ASAPOp> { #[serde(default, skip_serializing_if = "Option::is_none")] group_by: Option>, }, - /// A tumbling-window summary (#509 Pass 2, window-composition - /// rule): the `SummaryAgg` with these `family`, `input`, `reduction`, - /// `grouping` and `filter`, built over each of `tumbling_window_count` back-to-back - /// range windows of `tumbling_window_width_ms` that end at `child`'s evaluation - /// time, newest first. `child` is the input a range selector would - /// read, so tumbling window `i` covers `(-(i+1)·w, -i·w]` relative to it. Output: - /// the `SummaryAgg` schema, one state per group and tumbling window; a - /// `SummaryMerge` over the chain gives the state of one build over the - /// whole `tumbling_window_count · tumbling_window_width_ms` window - /// (`whole_window_build`). - /// One compact node whatever the tumbling-window count: costs and guarantees come - /// from formulas, and the executor expands the tumbling windows - /// (`expand_tumbling_window_chains`). + /// A chain of tumbling windows (concepts doc, section 3, "Chain + /// groups"): the `SummaryAgg` with these `family`, `input`, `reduction`, + /// `grouping` and `filter`, built over each window + /// `[k·w, (k+1)·w)` of `child`, `w` = `tumbling_window_width_ms`, on a + /// grid anchored at the epoch. Each window is built when it ends. Output: + /// one row per window, (window end, keys, state), a state with a time + /// index. It has no range and no window count: a `TimeRange` (and + /// `TimeShift`) over it selects the windows a query reads, and how many + /// windows it retains is $h_v$, a Stage 3 choice. // Written as `PaneChain` before #1163; the aliases read those documents. #[serde(alias = "PaneChain")] SummaryTumblingWindowChain { @@ -104,8 +91,6 @@ pub enum ASAPOp> { filter: Option>, #[serde(alias = "pane_width_ms")] tumbling_window_width_ms: u64, - #[serde(alias = "pane_count")] - tumbling_window_count: u32, }, // ── Reserved: migrated but unimplemented ── SummarySubtract { @@ -182,7 +167,6 @@ impl ASAPOp { grouping, filter, tumbling_window_width_ms, - tumbling_window_count, } => SummaryTumblingWindowChain { child: f(child), family: family.clone(), @@ -193,7 +177,6 @@ impl ASAPOp { .as_ref() .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), tumbling_window_width_ms: *tumbling_window_width_ms, - tumbling_window_count: *tumbling_window_count, }, SummaryEstimate { summary_input, @@ -211,13 +194,8 @@ impl ASAPOp { child: f(child), evaluation: evaluation.clone(), }, - SummaryMerge { - children, - newest_tumbling_windows, - group_by, - } => SummaryMerge { + SummaryMerge { children, group_by } => SummaryMerge { children: children.iter().map(&mut f).collect(), - newest_tumbling_windows: *newest_tumbling_windows, group_by: group_by.clone(), }, SummarySubtract { left, right } => SummarySubtract { @@ -354,10 +332,8 @@ impl ASAPOp { if let Some((build, schema)) = rolled_up_build(child) { return finalized_schema(Some(&build), &schema); } - // Merged tumbling windows finalize as one build over their window. if super::tumbling_window_chain::merged_chain(child).is_some() { - let (build, schema) = super::tumbling_window_chain::merged_window_build(child)?; - return finalized_schema(Some(&build), &schema); + todo!("#1207 PR 1: finalize the windows a TimeRange over a chain selects as one build over that TimeRange") } finalized_schema(summary_build(child).as_ref(), child.schema())? } @@ -477,35 +453,7 @@ impl ASAPOp { } }; match self { - SummaryMerge { - children, - newest_tumbling_windows, - group_by, - } => { - if newest_tumbling_windows.is_some() && group_by.is_some() { - return Err(SchemaDerivationError::InvalidScalarSignature( - "a rollup merges every input state, not the newest tumbling windows".into(), - )); - } - if let Some(n) = newest_tumbling_windows { - let tumbling_window_count = match children.as_slice() { - [chain] => match chain.operator() { - Operator::ASAP(SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => Some(tumbling_window_count), - _ => None, - }, - _ => None, - }; - if !tumbling_window_count.is_some_and(|count| (1..count).contains(n)) { - return Err(SchemaDerivationError::InvalidScalarSignature( - "a merge of the newest tumbling windows reads one tumbling-window chain, and fewer \ - tumbling windows than it has" - .into(), - )); - } - } + SummaryMerge { children, .. } => { let Some(first) = children.first() else { return Err(SchemaDerivationError::InvalidScalarSignature( "summary merge requires at least one state input".into(), @@ -595,12 +543,11 @@ impl ASAPOp { SummaryTumblingWindowChain { child, tumbling_window_width_ms, - tumbling_window_count, .. } => { - if *tumbling_window_width_ms == 0 || *tumbling_window_count == 0 { + if *tumbling_window_width_ms == 0 { return Err(SchemaDerivationError::InvalidScalarSignature( - "a tumbling-window chain needs a positive tumbling-window width and tumbling-window count".into(), + "a tumbling-window chain needs a positive tumbling-window width".into(), )); } // The chain is the range selector: it reads the series the @@ -712,9 +659,7 @@ fn rolled_up_build( op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) } _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { - super::tumbling_window_chain::merged_window_build(&input) - .ok()? - .0 + todo!("#1207 PR 1: roll up the windows a TimeRange over a chain selects as one build over that TimeRange") } _ => return None, }; diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index 1a4d034d5..b5d0ac29d 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -18,9 +18,9 @@ pub trait OperatorInput: Sized + Clone { fn result_kind(&self) -> OperatorResultKind; /// The input's own operator, its children as inputs again. fn operator(&self) -> Operator; - /// What the input covers when it is summary state in the memo; `None` - /// for any other input. - fn summary_coverage(&self) -> Option<&SummaryCoverage>; + /// What the input covers when it is an aggregate group in the memo; + /// `None` for any other input. + fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; } /// A front-end node tree. Node trees carry no summary coverage. @@ -37,7 +37,7 @@ impl OperatorInput for Rc { self.operator.clone() } - fn summary_coverage(&self) -> Option<&SummaryCoverage> { + fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { None } } @@ -109,37 +109,10 @@ impl OperatorInput for InputOrTemporaryBuild { } } - fn summary_coverage(&self) -> Option<&SummaryCoverage> { + fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { match self { - Self::Input(input) => input.summary_coverage(), + Self::Input(input) => input.aggregate_coverage(), Self::TemporaryBuild(_) => None, } } } - -impl InputOrTemporaryBuild> { - /// The node tree of this input: a temporary build as a new node over - /// its inputs' trees. - pub(crate) fn to_node(&self) -> Rc { - match self { - Self::Input(node) => Rc::clone(node), - Self::TemporaryBuild(build) => Rc::new(OperatorNode::with_schema( - build.operator.map_children(Self::to_node), - build.schema.clone(), - )), - } - } -} - -/// A build over node trees (`tumbling_window_chain`'s builds) as a node. -pub(crate) fn node_of_build( - (build, schema): ( - super::asap::ASAPOp>>, - Schema, - ), -) -> OperatorNode { - OperatorNode::with_schema( - Operator::ASAP(build.map_children(InputOrTemporaryBuild::to_node)), - schema, - ) -} diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index add4528bf..aca6285b0 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -1,39 +1,15 @@ -//! Tumbling-window summaries: what an [`ASAPOp::SummaryTumblingWindowChain`] computes, -//! as the plain `SummaryAgg` nodes it stands for. -//! -//! A chain of `n` tumbling windows of width `w` over `child` is tumbling window `i` (newest first, -//! `0 ≤ i < n`): the chain's `SummaryAgg` over a range `TimeRange(w)` over -//! `child` shifted back by `i·w`. A `SummaryMerge` over the chain is the -//! state of one build over `TimeRange(n·w)` (`whole_window_build`) when -//! the family merges (the family-capability table), and the executor runs -//! the chain as its tumbling windows (`expand_tumbling_window_chains`). A merge of only the -//! newest `m < n` tumbling windows (`SummaryMerge::newest_tumbling_windows`) is the state of one -//! build over `TimeRange(m·w)` (`merged_window_build`): one chain materialized for -//! the longest window serves every query whose window, on the same tumbling windows, -//! ends at the same time and is shorter. -#[cfg(any(test, feature = "test-support"))] -use std::collections::HashMap; -use std::rc::Rc; -use std::time::Duration; - -use super::asap::{summary_agg_schema, ASAPOp}; -use super::input::node_of_build; -use super::input::{InputOrTemporaryBuild, OperatorInput}; -use super::node::{Operator, OperatorNode}; -use super::non_asap::{NonASAPOp, TimeRangeKind}; -#[cfg(any(test, feature = "test-support"))] -use super::operator_properties::TimeShift; -use crate::ir::schema::Schema; -use crate::ir::SchemaDerivationError; - -fn invalid(message: &str) -> SchemaDerivationError { - SchemaDerivationError::InvalidScalarSignature(message.into()) -} - -/// The `SummaryTumblingWindowChain` a summary state carries merged: the chain itself, -/// or the one chain a `SummaryMerge` that keeps its groups reads, in full -/// or its newest tumbling windows (`merged_tumbling_window_count`). `None` for any other input, -/// a rollup (regrouping merge) included. +//! Tumbling-window chains ([`ASAPOp::SummaryTumblingWindowChain`]): one +//! row per window of width `w`, on a grid anchored at the epoch, a state +//! with a time index. A chain has no range and no window count: a +//! `TimeRange` (and `TimeShift`) over it selects the windows a query reads +//! (#1207 PR 1), and how many windows it retains is $h_v$ (#1207 PR 7). +use super::asap::ASAPOp; +use super::input::OperatorInput; +use super::node::Operator; + +/// The `SummaryTumblingWindowChain` a summary state carries merged: the +/// chain itself, or the one chain a `SummaryMerge` that keeps its groups +/// reads. `None` for any other input, a rollup (regrouping merge) included. pub fn merged_chain(state: &I) -> Option { match state.operator() { Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), @@ -55,335 +31,3 @@ pub fn merged_chain(state: &I) -> Option { _ => None, } } - -/// How many of its chain's tumbling windows, newest first, a summary state with -/// a [`merged_chain`] merges: the merge's `newest_tumbling_windows`, otherwise the -/// chain's `tumbling_window_count`. `None` for any other input. -fn merged_tumbling_window_count(state: &I) -> Option { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) = merged_chain(state)?.operator() - else { - return None; - }; - match state.operator() { - Operator::ASAP(ASAPOp::SummaryMerge { - newest_tumbling_windows: Some(newest), - .. - }) => Some(newest), - _ => Some(tumbling_window_count), - } -} - -/// A build over the inputs of a chain over `I`, with its schema. -pub(crate) type BuildWithSchema = (ASAPOp>, Schema); - -/// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the -/// chain's input whose state the merged tumbling windows of `chain` equal. -#[cfg(any(test, feature = "test-support"))] -pub(crate) fn whole_window_build( - chain: &I, -) -> Result, SchemaDerivationError> { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) = chain.operator() - else { - return Err(invalid("expected a tumbling-window chain")); - }; - newest_tumbling_windows_build(chain, tumbling_window_count) -} - -/// The `SummaryAgg` whose state a summary state with a -/// [`merged_chain`] carries: one build over the window of the tumbling windows it -/// merges (`merged_tumbling_window_count`). -pub(crate) fn merged_window_build( - state: &I, -) -> Result, SchemaDerivationError> { - let (Some(chain), Some(tumbling_window_count)) = - (merged_chain(state), merged_tumbling_window_count(state)) - else { - return Err(invalid( - "expected merged tumbling windows of one tumbling-window chain", - )); - }; - newest_tumbling_windows_build(&chain, tumbling_window_count) -} - -/// The `SummaryAgg` over `TimeRange(tumbling_window_count · tumbling_window_width_ms)` over the -/// chain's input: the state of the chain's newest `tumbling_window_count` tumbling windows. -fn newest_tumbling_windows_build( - chain: &I, - tumbling_window_count: u32, -) -> Result, SchemaDerivationError> { - let Operator::ASAP(chain_op) = chain.operator() else { - return Err(invalid("expected a tumbling-window chain")); - }; - named_as_chain( - window_build(&chain_op, tumbling_window_count)?, - chain.schema(), - ) -} - -/// [`newest_tumbling_windows_build`] of the chain operator `chain`, without -/// its schema. -pub(crate) fn window_build( - chain: &ASAPOp, - tumbling_window_count: u32, -) -> Result>, SchemaDerivationError> { - let ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - .. - } = chain - else { - return Err(invalid("expected a tumbling-window chain")); - }; - let window = tumbling_window_width_ms - .checked_mul(u64::from(tumbling_window_count)) - .ok_or_else(|| invalid("tumbling-window chain window overflows"))?; - build_over( - chain, - range(InputOrTemporaryBuild::Input(child.clone()), window)?, - ) -} - -#[cfg(any(test, feature = "test-support"))] -/// Tumbling window `index` (0 = newest) of `chain`: its `SummaryAgg` over the range -/// `TimeRange(tumbling_window_width_ms)` ending `index · tumbling_window_width_ms` before the -/// chain's input. -pub(crate) fn tumbling_window( - chain: &I, - index: u32, -) -> Result, SchemaDerivationError> { - let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - .. - }) = chain.operator() - else { - return Err(invalid("expected a tumbling-window chain")); - }; - let back = i64::try_from(tumbling_window_width_ms.saturating_mul(u64::from(index))) - .map_err(|_| invalid("tumbling window offset overflows"))?; - let Operator::ASAP(chain_op) = chain.operator() else { - return Err(invalid("expected a tumbling-window chain")); - }; - named_as_chain( - build_over( - &chain_op, - range(shifted(child, back)?, tumbling_window_width_ms)?, - )?, - chain.schema(), - ) -} - -#[cfg(any(test, feature = "test-support"))] -/// `child` evaluated `back_ms` earlier: one `TimeShift`, folded into a -/// `TimeShift` directly below. -fn shifted( - child: I, - back_ms: i64, -) -> Result, SchemaDerivationError> { - if back_ms == 0 { - return Ok(InputOrTemporaryBuild::Input(child)); - } - let (shift, input) = match child.operator() { - Operator::NonASAP(NonASAPOp::TimeShift { - shift, - child: input, - }) => ( - TimeShift { - offset_ms: shift - .offset_ms - .checked_add(back_ms) - .ok_or_else(|| invalid("tumbling window offset overflows"))?, - at: shift.at, - }, - input, - ), - _ => ( - TimeShift { - offset_ms: back_ms, - at: None, - }, - child, - ), - }; - InputOrTemporaryBuild::build(NonASAPOp::TimeShift { - shift, - child: InputOrTemporaryBuild::Input(input), - }) -} - -/// A range `TimeRange(width_ms)` over `child`. -fn range( - child: InputOrTemporaryBuild, - width_ms: u64, -) -> Result, SchemaDerivationError> { - InputOrTemporaryBuild::build(NonASAPOp::TimeRange { - range: Duration::from_millis(width_ms), - kind: TimeRangeKind::Range, - child, - }) -} - -/// The chain's `SummaryAgg` over `input`. -fn build_over( - chain: &ASAPOp, - input: InputOrTemporaryBuild, -) -> Result>, SchemaDerivationError> { - let ASAPOp::SummaryTumblingWindowChain { - family, - input: update, - reduction, - grouping, - filter, - .. - } = chain.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) - else { - return Err(invalid("expected a tumbling-window chain")); - }; - Ok(ASAPOp::SummaryAgg { - child: input, - family, - input: update, - reduction, - grouping, - filter, - }) -} - -/// `build` with its schema, named as the chain of schema `chain`. -fn named_as_chain( - build: ASAPOp>, - chain: &Schema, -) -> Result, SchemaDerivationError> { - let ASAPOp::SummaryAgg { - child, - family, - reduction, - .. - } = &build - else { - return Err(invalid("expected a summary build")); - }; - let mut schema = summary_agg_schema(child.schema(), family, reduction)?; - for (field, kept) in schema.fields.iter_mut().zip(&chain.fields) { - field.name = kept.name.clone(); - field.table = kept.table.clone(); - } - Ok((build, schema)) -} - -/// `root` with every `SummaryTumblingWindowChain` expanded into its tumbling windows: a `SummaryMerge` -/// reads each tumbling window of each chain it read, newest first, as plain -/// `SummaryAgg` nodes the executor runs; a merge of the newest `n` tumbling windows -/// reads the first `n`, the same nodes a merge of the whole chain reads. Every other node keeps its -/// guarantee. A chain read by anything but a `SummaryMerge` is an error. -#[cfg(any(test, feature = "test-support"))] -pub fn expand_tumbling_window_chains( - root: &Rc, -) -> Result, SchemaDerivationError> { - expand(root, &mut NodeToExpandedNodeCache::default()) -} - -#[cfg(any(test, feature = "test-support"))] -/// What one expansion has built: each expanded node by its input's -/// address, and each tumbling window by its chain's address and index, so a chain two -/// merges read expands into one set of tumbling windows. -#[derive(Default)] -struct NodeToExpandedNodeCache { - expanded_node_of: HashMap<*const OperatorNode, Rc>, - tumbling_window_of: HashMap<(*const OperatorNode, u32), Rc>, - /// Each chain expanded, in the order first reached. - chains: Vec>, - /// Each expanded node with its input, in the order expanded. - expanded: Vec<(Rc, Rc)>, -} - -#[cfg(any(test, feature = "test-support"))] -fn expand( - node: &Rc, - cache: &mut NodeToExpandedNodeCache, -) -> Result, SchemaDerivationError> { - if let Some(done) = cache.expanded_node_of.get(&Rc::as_ptr(node)) { - return Ok(Rc::clone(done)); - } - let out = match &node.operator { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - return Err(invalid( - "a tumbling-window chain is read only through a summary merge", - )) - } - Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows, - group_by, - }) => { - let mut inputs = Vec::new(); - for child in children { - match child.asap() { - Some(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }) => { - for index in 0..newest_tumbling_windows.unwrap_or(*tumbling_window_count) { - let key = (Rc::as_ptr(child), index); - if let Some(done) = cache.tumbling_window_of.get(&key) { - inputs.push(Rc::clone(done)); - continue; - } - let tumbling_window = - Rc::new(node_of_build(tumbling_window(child, index)?)); - if index == 0 { - cache.chains.push(Rc::clone(child)); - } - cache - .tumbling_window_of - .insert(key, Rc::clone(&tumbling_window)); - inputs.push(tumbling_window); - } - } - _ => inputs.push(expand(child, cache)?), - } - } - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: inputs, - newest_tumbling_windows: None, - group_by: group_by.clone(), - }))?; - Rc::new( - OperatorNode::with_schema(merge.operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()), - ) - } - _ => { - let mut changed = false; - let mut children = HashMap::new(); - for child in node.children() { - let new = expand(child, cache)?; - changed |= !Rc::ptr_eq(&new, child); - children.insert(Rc::as_ptr(child), new); - } - if changed { - Rc::new( - OperatorNode::with_schema( - node.operator - .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()), - ) - } else { - Rc::clone(node) - } - } - }; - cache - .expanded_node_of - .insert(Rc::as_ptr(node), Rc::clone(&out)); - cache.expanded.push((Rc::clone(node), Rc::clone(&out))); - Ok(out) -} diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index f6d383b28..7e5419dcb 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -437,14 +437,9 @@ impl PhysicalASAPDAG { })) if materialized_chains.contains(&consumer) => { u128::from(*tumbling_window_width_ms) } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - tumbling_window_count, - .. - })) => { - u128::from(*tumbling_window_width_ms) - * u128::from(*tumbling_window_count) - } + Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( + "#1207 PR 6: the scan window of a chain read through a TimeRange over it" + ), _ => return ScanWindow::Unbounded, }; let read = ScanWindow::Within { @@ -495,25 +490,9 @@ impl PhysicalASAPDAG { Some(Operator::NonASAP(NonASAPOp::TimeRange { .. })) => { time_range_window(id, |id| payloads.get(&id).copied()) } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - tumbling_window_count, - .. - })) => { - let width_ms = - tumbling_window_width_ms.checked_mul(u64::from(*tumbling_window_count)); - match ( - offset_below(*child, |id| payloads.get(&id).copied()), - width_ms, - ) { - (Some(end_offset_ms), Some(width_ms)) => ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }, - _ => ScanWindow::Unbounded, - } - } + Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( + "#1207 PR 6: the window a chain read through a TimeRange over it summarizes" + ), Some(payload) => { pending.extend(payload.children().into_iter().copied()); continue; diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 0e2dec55e..6509507a6 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -24,7 +24,6 @@ use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::operator::operator_properties::Reduction; -use crate::ir::operator::tumbling_window_chain::window_build; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; @@ -71,21 +70,26 @@ impl SummaryCoverage { } } -/// A conjunction of per-column constraints and an optional time window. -/// A column or time it does not mention is unrestricted. JSON: `columns` is -/// a list of `{column, constraint}` (its keys are not strings), and a bound -/// is `{"included": v}`, `{"excluded": v}` or `"unbounded"`. +/// A conjunction of per-column constraints. The timestamp column is one of +/// the columns; there is no separate time field. A column it does not +/// mention is unrestricted. JSON: `columns` is a list of +/// `{column, constraint}` (its keys are not strings), and a bound is +/// `{"included": v}`, `{"excluded": v}` or `"unbounded"`. #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SelectionBox { #[serde(with = "columns_as_list")] pub(crate) columns: BTreeMap, - /// Offsets from the evaluation time in milliseconds: a PromQL range - /// `TimeRange(w)` over `TimeShift(s)` is `(-(s + w), -s]`. Absolute - /// time will be an ordinary interval on the timestamp column once the IR - /// has timestamp literals. - #[serde(with = "relative_time_bounds")] - pub(crate) relative_time: Option<(Bound, Bound)>, +} + +/// An endpoint of an interval on a timestamp column. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TimeBound { + /// An absolute time, in ms since the epoch. + At(i64), + /// An offset, in ms, from the evaluation time t. + FromEvaluationTime(i64), } /// `SelectionBox::columns` as a list of `{column, constraint}`. @@ -155,33 +159,6 @@ impl From> for Bound { } } -/// `SelectionBox::relative_time` as `[lower, upper]` wire bounds. -mod relative_time_bounds { - use super::WireBound; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use std::ops::Bound; - - /// A relative time window, `(lower, upper)`. - type Window = (Bound, Bound); - - pub(super) fn serialize( - window: &Option, - serializer: S, - ) -> Result { - window - .as_ref() - .map(|(lower, upper)| (WireBound::from(lower), WireBound::from(upper))) - .serialize(serializer) - } - - pub(super) fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result, D::Error> { - let window = Option::<(WireBound, WireBound)>::deserialize(deserializer)?; - Ok(window.map(|(lower, upper)| (lower.into(), upper.into()))) - } -} - /// A column of the definition child's output, by `(table, name)`. A /// `Project` below renames its columns and sets their table to its /// qualifier (none by default). @@ -245,11 +222,6 @@ pub enum CoverageError { "coverage is derived only for SummaryAgg, SummaryTumblingWindowChain and SummaryMerge" )] NotSummary, - /// A `SummaryTumblingWindowChain` that does not expand to one build over its whole - /// window (its window overflows, or the build does not derive): bad - /// input. - #[error("tumbling-window chain does not expand to one build over its whole window: {reason}")] - TumblingWindowChainExpansion { reason: String }, /// A `SummaryMerge` without inputs: bad input. #[error("summary merge requires at least one input")] EmptyMerge, @@ -292,9 +264,7 @@ impl SummaryCoverage { /// Coverage of a summary state a memo group computes with `operator` /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. A chain covers what one build over its whole window - /// does, and a merge of its newest tumbling windows what one build over - /// their window does. A merge reads its inputs' stored coverage and fails + /// `SummaryAgg`. A merge reads its inputs' stored coverage and fails /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may @@ -303,10 +273,6 @@ impl SummaryCoverage { pub(crate) fn derive_from_inputs<'m>( operator: &Operator>, ) -> Result { - let expansion = - |error: SchemaDerivationError| CoverageError::TumblingWindowChainExpansion { - reason: error.to_string(), - }; let as_built = |op: &ASAPOp>| -> ASAPOp> { op.map_children(|input| InputOrTemporaryBuild::Input(*input)) }; @@ -316,29 +282,9 @@ impl SummaryCoverage { None => Err(CoverageError::NotSummary), }, Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), - // Back-to-back tumbling windows cover the whole window one build - // over it would. - Operator::ASAP( - op @ ASAPOp::SummaryTumblingWindowChain { - tumbling_window_count, - .. - }, - ) => of_summary_agg(&window_build(op, *tumbling_window_count).map_err(expansion)?), - // The newest tumbling windows of one chain cover what one build - // over their window does. - Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: Some(newest), - group_by: None, - }) => { - let [chain] = children.as_slice() else { - return Err(CoverageError::NotSummary); - }; - let Operator::ASAP(chain) = chain.operator() else { - return Err(CoverageError::NotSummary); - }; - of_summary_agg(&window_build(&chain, *newest).map_err(expansion)?) - } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( + "#1207 PR 2: a chain's coverage is a build's over its input, with the timestamp column unconstrained" + ), Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -346,7 +292,7 @@ impl SummaryCoverage { .iter() .map(|child| { child - .summary_coverage() + .aggregate_coverage() .cloned() .ok_or(CoverageError::NotSummary) }) @@ -464,14 +410,13 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result bool = match required { MergeInputRule::MayOverlap => |_, _| true, - MergeInputRule::DisjointInTime => SelectionBox::disjoint_in_time, + MergeInputRule::DisjointInTime => todo!( + "#1207 PR 2: disjoint intervals on the timestamp column, with TimeBound endpoints" + ), MergeInputRule::Disjoint => SelectionBox::disjoint, }; - // Windows in time order, each ending before the next starts, are - // pairwise disjoint in time, and so pairwise disjoint: O(b log b) for - // b boxes. Otherwise, every pair of boxes of different inputs. + // Every pair of boxes of different inputs. if required != MergeInputRule::MayOverlap - && !windows_ordered_apart(inputs) && inputs.iter().enumerate().any(|(i, input)| { inputs[..i].iter().any(|other| { input @@ -583,31 +528,6 @@ fn build_input_schema(state: &I) -> Option { } } -/// Whether the time windows of every box of `inputs` are pairwise apart: -/// sorted by start, each ends before the next starts. `false` when a box -/// has no time window or two windows overlap, boxes of one input included. -fn windows_ordered_apart(inputs: &[SummaryCoverage]) -> bool { - let mut windows = Vec::new(); - for selection_box in inputs.iter().flat_map(|input| &input.selection) { - match &selection_box.relative_time { - Some(window) => windows.push(window), - None => return false, - } - } - // A lower bound's position: unbounded first, then by value, an - // included value before the excluded one. - let start = |(lower, _): &&(Bound, Bound)| match lower { - Bound::Unbounded => (false, 0, false), - Bound::Included(at) => (true, *at, false), - Bound::Excluded(at) => (true, *at, true), - }; - windows.sort_by_key(start); - windows.windows(2).all(|pair| { - let ((_, upper), (lower, _)) = (pair[0], pair[1]); - ends_before(upper, lower, |a: &i64, b: &i64| Some(a.cmp(b))) - }) -} - /// The coverage of `build`, a `SummaryAgg` over memo groups (or over an /// input a rule built over them): its operators walked down through the /// liftable operators of its input groups' expressions. @@ -646,13 +566,6 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>) -> Result { @@ -712,12 +624,9 @@ fn of_summary_agg(build: &ASAPOp>) -> Result { - window_ms = range.as_millis() as i64; - } - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) if lift_time => { - shift_ms += shift.offset_ms; - } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) => todo!( + "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with TimeBound endpoints" + ), other => unlifted_operators.push(unlifted(other)), } } @@ -741,10 +650,6 @@ fn of_summary_agg(build: &ASAPOp>) -> Result, - relative_time: Option<(Bound, Bound)>, - ) -> Self { - Self { - columns, - relative_time, - } + pub fn new(columns: BTreeMap) -> Self { + Self { columns } } /// The per-column constraints. pub fn columns(&self) -> &BTreeMap { &self.columns } - - /// The time window, as offsets from the evaluation time in milliseconds. - pub fn relative_time(&self) -> Option<(Bound, Bound)> { - self.relative_time - } } impl SelectionBox { - /// Whether no row lies in both boxes: some column, or the time window, - /// is restricted by both and the restrictions are disjoint. + /// Whether no row lies in both boxes: some column is restricted by both + /// and the restrictions are disjoint. fn disjoint(&self, other: &Self) -> bool { - self.disjoint_in_time(other) - || self - .columns - .iter() - .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) - } - - /// Whether the boxes' time windows do not overlap. - fn disjoint_in_time(&self, other: &Self) -> bool { - match (&self.relative_time, &other.relative_time) { - (Some((al, au)), Some((bl, bu))) => { - intervals_disjoint((al, au), (bl, bu), |a: &i64, b: &i64| Some(a.cmp(b))) - } - _ => false, - } + self.columns + .iter() + .any(|(column, a)| other.columns.get(column).is_some_and(|b| a.disjoint(b))) } } -/// Join boxes that differ only in one dimension whose union is again one -/// constraint: touching time windows or value ranges, or the values of one -/// `In` column. Gaps stay separate boxes. -fn union(boxes: Vec) -> Vec { - union_ordered_time_boxes(&boxes).unwrap_or_else(|| union_general(boxes)) -} - -fn union_general(mut boxes: Vec) -> Vec { +/// Join boxes that differ only in one column whose union is again one +/// constraint: touching value ranges, or the values of one `In` column. +/// Gaps stay separate boxes. +fn union(mut boxes: Vec) -> Vec { let mut i = 0; while i < boxes.len() { let joined = (i + 1..boxes.len()).find_map(|j| join(&boxes[i], &boxes[j]).map(|b| (j, b))); @@ -1152,75 +1030,11 @@ fn union_general(mut boxes: Vec) -> Vec { boxes } -/// Tumbling-window coverage has identical column predicates and pairwise disjoint time -/// intervals. Sort once and join adjacent intervals, rather than removing one -/// element from a large vector for every tumbling window. General multidimensional or -/// overlapping coverage continues through the existing exact union algorithm. -fn union_ordered_time_boxes(boxes: &[SelectionBox]) -> Option> { - let first = boxes.first()?; - if !boxes - .iter() - .all(|b| b.columns == first.columns && b.relative_time.is_some()) - { - return None; - } - if !boxes - .iter() - .all(|b| match b.relative_time.as_ref().unwrap() { - (Bound::Unbounded, _) | (_, Bound::Unbounded) => true, - (Bound::Included(lower), Bound::Included(upper)) => lower <= upper, - ( - Bound::Included(lower) | Bound::Excluded(lower), - Bound::Included(upper) | Bound::Excluded(upper), - ) => lower < upper, - }) - { - return None; - } - let mut order: Vec<_> = (0..boxes.len()).collect(); - order.sort_by_key( - |&index| match boxes[index].relative_time.as_ref().unwrap().0 { - Bound::Unbounded => (false, 0, false), - Bound::Included(at) => (true, at, false), - Bound::Excluded(at) => (true, at, true), - }, - ); - if !order.windows(2).all(|pair| { - let (_, upper) = boxes[pair[0]].relative_time.as_ref().unwrap(); - let (lower, _) = boxes[pair[1]].relative_time.as_ref().unwrap(); - ends_before(upper, lower, |a: &i64, b: &i64| Some(a.cmp(b))) - }) { - return None; - } - let mut merged: Vec<(usize, SelectionBox)> = Vec::new(); - for index in order { - if let Some((first_index, previous)) = merged.last_mut() { - if let Some(joined) = join(previous, &boxes[index]) { - *first_index = (*first_index).min(index); - *previous = joined; - continue; - } - } - merged.push((index, boxes[index].clone())); - } - // The general union retains the earliest occurrence of each joined box. - merged.sort_by_key(|(index, _)| *index); - Some(merged.into_iter().map(|(_, selection)| selection).collect()) -} - fn join(a: &SelectionBox, b: &SelectionBox) -> Option { if a == b { return Some(a.clone()); } - if a.columns == b.columns { - let ((al, au), (bl, bu)) = (a.relative_time.as_ref()?, b.relative_time.as_ref()?); - let time = touching((al, au), (bl, bu), |x: &i64, y: &i64| Some(x.cmp(y)))?; - return Some(SelectionBox { - columns: a.columns.clone(), - relative_time: Some(time), - }); - } - if a.relative_time != b.relative_time || a.columns.len() != b.columns.len() { + if a.columns.len() != b.columns.len() { return None; } let mut differing = a @@ -1252,10 +1066,7 @@ fn join(a: &SelectionBox, b: &SelectionBox) -> Option { }; let mut columns = a.columns.clone(); columns.insert(column.clone(), joined); - Some(SelectionBox { - columns, - relative_time: a.relative_time, - }) + Some(SelectionBox { columns }) } /// The union of two intervals when one ends exactly where the other starts, @@ -1279,96 +1090,3 @@ fn touching( None } } - -#[cfg(test)] -mod tumbling_window_union_tests { - use super::*; - - fn tumbling_window(start: i64, end: i64) -> SelectionBox { - SelectionBox { - relative_time: Some((Bound::Excluded(start), Bound::Included(end))), - ..Default::default() - } - } - - #[test] - fn large_reverse_ordered_tumbling_window_union_is_one_exact_interval() { - let tumbling_windows: Vec<_> = (0..75_000) - .rev() - .map(|i| tumbling_window(i, i + 1)) - .collect(); - assert_eq!(union(tumbling_windows), vec![tumbling_window(0, 75_000)]); - } - - #[test] - fn interval_fast_path_matches_general_union_for_permutations() { - use Bound::{Excluded as E, Included as I, Unbounded as U}; - for intervals in [ - [(U, I(0)), (E(0), I(1)), (E(1), I(2)), (E(2), U)], - [(E(0), E(1)), (I(1), I(1)), (E(1), I(2)), (E(4), I(5))], - [(E(0), I(1)), (E(3), E(4)), (I(4), I(5)), (E(5), I(6))], - ] { - let boxes: Vec<_> = intervals - .into_iter() - .map(|relative_time| SelectionBox { - relative_time: Some(relative_time), - ..Default::default() - }) - .collect(); - for a in 0..4 { - for b in 0..4 { - for c in 0..4 { - for d in 0..4 { - let order = [a, b, c, d]; - if (0..4).any(|i| order[..i].contains(&order[i])) { - continue; - } - let shuffled: Vec<_> = - order.into_iter().map(|i| boxes[i].clone()).collect(); - assert_eq!(union(shuffled.clone()), union_general(shuffled)); - } - } - } - } - } - let empty = SelectionBox { - relative_time: Some((E(0), E(0))), - ..Default::default() - }; - let singleton = SelectionBox { - relative_time: Some((I(0), I(0))), - ..Default::default() - }; - let unusual = vec![empty.clone(), empty, singleton]; - assert!(union_ordered_time_boxes(&unusual).is_none()); - assert_eq!(union(unusual.clone()), union_general(unusual)); - } - - #[test] - fn sorted_union_preserves_gaps_boundaries_and_original_group_order() { - assert_eq!( - union(vec![ - tumbling_window(10, 11), - tumbling_window(1, 2), - tumbling_window(0, 1), - tumbling_window(11, 12) - ]), - vec![tumbling_window(10, 12), tumbling_window(0, 2)] - ); - let missing_endpoint = SelectionBox { - relative_time: Some((Bound::Excluded(0), Bound::Excluded(1))), - ..Default::default() - }; - assert_eq!( - union(vec![missing_endpoint.clone(), tumbling_window(1, 2)]), - vec![missing_endpoint, tumbling_window(1, 2)] - ); - let overlaps = vec![tumbling_window(0, 2), tumbling_window(1, 3)]; - assert!(union_ordered_time_boxes(&overlaps).is_none()); - assert_eq!(union(overlaps.clone()), overlaps); - assert_eq!( - union(vec![tumbling_window(0, 1), tumbling_window(0, 1)]), - vec![tumbling_window(0, 1)] - ); - } -} diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index e219b78f5..0f8987310 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -9,6 +9,9 @@ //! [`ir::properties`] (§2.2–2.3 accuracy, guarantee, execution timing and //! summary coverage), the flat DAG //! form ([`ir::flat`]) and the physical ASAP DAG ([`ir::physical_export`]). +//! - [`deployment_capabilities`] — what the deployment's executor can run and +//! keep ([`deployment_capabilities::DeploymentCapabilities`]), the deployment +//! input Stages 1–3 read. //! - [`workload`] — the planner inputs (#509): the query and data //! workloads and the parsed workload the query-language frontends produce //! ([`workload::parsed_workload`]). @@ -20,6 +23,7 @@ //! physical candidates ([`stages::PhysicalASAPMemoCandidates`]). //! //! No execution logic lives in this crate (issue #190). +pub mod deployment_capabilities; pub mod ir; pub mod physical; pub mod serde_f64; diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 04c78b7d5..76762609a 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -7,6 +7,7 @@ use std::collections::HashSet; use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; use crate::ir::properties::summary_coverage::SummaryCoverage; +use crate::ir::properties::AccuracyMetric; use crate::ir::schema::{SketchAlgorithm, SketchParams}; use crate::ir::ASAPOp; use crate::ir::Operator; @@ -79,6 +80,25 @@ impl Memo { todo!("#1207 PR 2") } + /// Replaces candidate `id`'s input requirements $r^{\log}_{c,i}$, one + /// per input group. The only way to change a candidate's + /// $r^{\log}$: Pass 2 sets each parameter floor from the input's caps + /// and direct needs after the rules have run. Panics when the vector + /// does not have one entry per input group. + pub fn set_input_requirements( + &mut self, + id: CandidateForMemoGroupId, + input_requirements: Vec, + ) { + let candidate = &mut self.groups[id.group.0 as usize].candidates[id.index_in_group]; + assert_eq!( + input_requirements.len(), + candidate.expression.operator.children().len(), + "one input requirement per input group of candidate {id:?}" + ); + candidate.input_requirements = input_requirements; + } + /// The groups, each after every group it or one of its candidates /// reads, the lower index first among those ready: a dependency order for /// candidate traversal, whatever order rules appended groups in. `None` @@ -134,7 +154,7 @@ fn derive_group_properties( let operator = expression.map_children(|&group_id| MemoGroupAsInput { memo, group_id }); let schema = operator.output_schema()?; operator.validate_inputs()?; - let summary_coverage = match SummaryCoverage::derive_from_inputs(&operator) { + let aggregate_coverage = match SummaryCoverage::derive_from_inputs(&operator) { Ok(coverage) => Some(coverage), // A merge is valid only over states of one definition it may merge. Err(error) if matches!(operator, Operator::ASAP(ASAPOp::SummaryMerge { .. })) => { @@ -145,7 +165,7 @@ fn derive_group_properties( Ok(GroupProperties { schema, result_kind: operator.output_kind(), - summary_coverage, + aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), }) } @@ -171,7 +191,8 @@ pub struct CandidateForMemoGroup { /// $p^{\log}_v$. pub logical_properties: LogicalProperties, /// $r^{\log}_{c,i}$, one per input group: per child of - /// `expression.operator`, in [`Operator::children`] order. + /// `expression.operator`, in [`Operator::children`] order. Changed only + /// through [`Memo::set_input_requirements`]. pub input_requirements: Vec, /// The groups a fused computation also implements, beyond the group /// whose candidate it is; empty otherwise. @@ -188,7 +209,7 @@ pub enum OriginOfCandidateForMemoGroup { Pass1, /// Canonical value preprocessing feeding a summary build. SummaryInput, - /// A tumbling-window layout of a summary state group. + /// A tumbling-window chain, or a merge reading one, of a summary state group. TumblingWindows, /// A coarse group regrouping a finer group. Rollup, @@ -207,49 +228,34 @@ pub struct CandidateExpression { pub operator: Operator, } -/// $p^{\log}_v = (\boldsymbol\theta_v, \omega_v)$. The family is the -/// operator's argument, not a property. [`Self::new`] rejects parameters of -/// another family, not in the family's list, missing on a sketch build or -/// present on an exact operator, and a window on an operator that cannot be -/// windowed. +/// $p^{\log}_v = \boldsymbol\theta_v$. The family is the operator's +/// argument, not a property, and a tumbling-window chain's width is the +/// chain operator's argument. [`Self::new`] rejects parameters of another +/// family, not in the family's list, missing on a sketch build (a chain +/// included) or present on an exact operator. #[derive(Debug, Clone)] pub struct LogicalProperties { /// $\boldsymbol\theta_v$, from the family's predefined list; `None` for /// an exact operator. parameters: Option, - /// $\omega_v$. - window_layout: Option, } impl LogicalProperties { pub fn new( expression: &CandidateExpression, parameters: Option, - window_layout: Option, ) -> Result { - let _ = (expression, parameters, window_layout); - todo!("#1207 PR 1: reject invalid parameters and invalid windows") + let _ = (expression, parameters); + todo!("#1207 PR 1: reject invalid parameters") } /// $\boldsymbol\theta_v$. pub fn parameters(&self) -> Option<&SketchParams> { self.parameters.as_ref() } - - /// $\omega_v$. - pub fn window_layout(&self) -> Option<&WindowLayout> { - self.window_layout.as_ref() - } -} - -/// $\omega_v$. The window count is the group's query range ÷ width; -/// retention is $h_v$ (Stage 3). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum WindowLayout { - Tumbling { width_ms: u64 }, } -/// Logical properties whose dimensions cannot go together: bad input to +/// Logical properties that do not fit their operator: bad input to /// [`LogicalProperties::new`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum LogicalPropertiesError { @@ -257,10 +263,6 @@ pub enum LogicalPropertiesError { ParametersOfAnotherFamily, #[error("an exact operator has no summary parameters")] ParametersOnExactOperator, - #[error("the operator cannot be windowed")] - WindowOnOperatorThatCannotBeWindowed, - #[error("a tumbling window has a positive width")] - ZeroWindowWidth, #[error("a sketch build needs parameters from its family's list")] SketchBuildWithoutParameters, #[error("the parameters are not in their family's predefined list")] @@ -298,12 +300,17 @@ impl LogicalRequirements { self.parameter_floor_for_required_accuracy.as_ref() } - /// $g_c$'s linear bound: $\beta_{c,i}$ and $b_{c,i}$ per input, read - /// from the operator and its arguments. An operator without an error - /// contract is reported as unsupported; no factor is guessed. - pub fn error_propagation(consumer: &Operator) -> ErrorPropagation { - let _ = consumer; - todo!("#1207 PR 4: the error contract of each operator") + /// $g_c$'s linear bound for errors in `metric`, each input's and the + /// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the + /// operator and its arguments. `None` when the consumer has no error + /// contract for `metric`: an input whose error is in `metric` then gives + /// no edge (the metric rule). No factor is guessed. + pub fn error_propagation( + consumer: &Operator, + metric: AccuracyMetric, + ) -> Option { + let _ = (consumer, metric); + todo!("#1207 PR 4: the error contract of each operator, per metric") } } diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index 9a313ec76..aa13b2b29 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -1,7 +1,11 @@ //! Stage 1, Pass 2's output, which Stage 2 expands: the memo after every -//! rule, and the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201). +//! rule, the logical edges $(u, c, i)$ with $\alpha^{\log}$ (#1201), and +//! the candidates dead-candidate elimination removed, each with its root +//! causes. use super::{CandidateForMemoGroupId, Memo}; +use crate::ir::properties::AccuracyMetric; +use crate::ir::schema::SketchParams; /// $(u, c, i)$ over logical candidates: `producer_candidate_id` is a /// candidate of $G_{c,i}$, the group input `input_index_of_consumer` of @@ -20,6 +24,50 @@ pub struct LogicalEdge { pub struct LogicalASAPMemoCandidates { /// The memo after the last rule. pub memo: Memo, - /// Every $(u, c, i)$ with $u \in G_{c,i}$, grouped by $(c, i)$. + /// Every $(u, c, i)$ with $u \in G_{c,i}$ between candidates that + /// survive dead-candidate elimination, grouped by $(c, i)$. pub edges: Vec, + /// The candidates dead-candidate elimination removed: no logical edge + /// names them, and Stage 2 does not expand them. + pub pruned: Vec, +} + +/// A candidate eliminated by Pass 2. +#[derive(Debug, Clone)] +pub struct PrunedLogicalCandidate { + pub candidate: CandidateForMemoGroupId, + /// Every cause: one per failing edge it is cut off by, so mixed causes + /// (a floor on one edge, the metric rule on another) are all listed. + pub causes: Vec, +} + +/// One path from the candidate, through consumers that are themselves +/// eliminated, to the edge that fails. One shortest path per failing edge. +#[derive(Debug, Clone)] +pub struct PruneCause { + /// The eliminated consumers on the path, nearest first. + pub through_pruned_consumers: Vec, + pub root: RootPruneReason, +} + +/// Why the edge at the end of a [`PruneCause`]'s path fails, or why the +/// candidate has no edge at all. +#[derive(Debug, Clone)] +pub enum RootPruneReason { + /// The producer's parameters are weaker than the consumer input's floor. + BelowAccuracyFloorRequirement { + consumer: CandidateForMemoGroupId, + input: usize, + floor: SketchParams, + }, + /// The consumer's error contract carries none of the producer's metrics. + MetricRule { + consumer: CandidateForMemoGroupId, + input: usize, + producer_metric: AccuracyMetric, + consumer_carries: Vec, + }, + /// The candidate never had a consumer and is not a query output. A + /// candidate whose consumers were eliminated is traced to their causes. + NoConsumer, } diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 153346093..e7c3948cd 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -41,8 +41,11 @@ pub struct MemoGroup { pub struct GroupProperties { pub schema: Schema, pub result_kind: OperatorResultKind, - /// What a summary-state group covers; `None` for any other group. - pub summary_coverage: Option, + /// What an aggregate group covers: a summary state, a chain, a merge + /// over a chain's windows, or an exact `Aggregate` result; `None` for + /// any other group. Derived today for summary states only; exact + /// `Aggregate` groups follow in #1207 PR 2. + pub aggregate_coverage: Option, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, @@ -78,8 +81,8 @@ impl OperatorInput for MemoGroupAsInput<'_> { }) } - fn summary_coverage(&self) -> Option<&SummaryCoverage> { - self.group().group_properties.summary_coverage.as_ref() + fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { + self.group().group_properties.aggregate_coverage.as_ref() } } diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 596aee4f4..f032bacad 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -18,9 +18,11 @@ mod realizer; pub use candidates::{ parameter_list, producer_satisfies_logical_requirements, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, - LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, WindowLayout, + LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, +}; +pub use logical_memo_candidates::{ + LogicalASAPMemoCandidates, LogicalEdge, PruneCause, PrunedLogicalCandidate, RootPruneReason, }; -pub use logical_memo_candidates::{LogicalASAPMemoCandidates, LogicalEdge}; pub(crate) use memo::MemoGroupAsInput; pub use memo::{queries_rooted_at, root_groups, GroupProperties, MemoGroup, MemoGroupId}; pub use per_computation_logical_memo_candidates::{ @@ -29,8 +31,8 @@ pub use per_computation_logical_memo_candidates::{ pub use physical::{ output_references, producer_satisfies_physical_requirements, DagOfSelectedCandidates, ExecutionContext, ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateId, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, RefreshWindowId, - SelectedEdge, SelectedNodeId, + PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, RefreshWindowId, SelectedEdge, + SelectedNodeId, }; pub use pruning::{PruneReason, PrunedCandidate}; pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs index 6e886c505..e2f9afb85 100644 --- a/crates/types/src/stages/physical/graph.rs +++ b/crates/types/src/stages/physical/graph.rs @@ -48,7 +48,7 @@ impl DagOfSelectedCandidates { /// of its group. Costs and accuracy are Stage 3's to check. pub fn validate(&self, physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { let memo = &physical.logical_candidates.memo; - let output_groups: Vec<_> = super::output_references(&physical.query_roots) + let output_groups: Vec<_> = super::output_references(&memo.query_roots) .into_iter() .map(|(_, group)| group) .collect(); diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 5b64ece64..7070ef3f3 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -10,9 +10,7 @@ use super::execution_context::ExecutionContext; use super::graph::PhysicalCandidateId; use crate::ir::QueryRoot; use crate::physical::PhysicalProperties; -use crate::stages::{ - CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PrunedCandidate, -}; +use crate::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, PrunedCandidate}; /// Index into [`PhysicalASAPMemoCandidates::edges`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -30,9 +28,9 @@ pub struct PhysicalCandidate { impl PhysicalCandidate { /// Whether a consumer running in `context` can read this producer (the /// concepts doc's alignment table, section 2). True for $\ell$ = None, - /// and for `OneEvaluation` in the same context. A windowed - /// `AcrossEvaluations` producer, with $w$ read from its logical window - /// layout in `memo`, needs every evaluation time of the context on its + /// and for `OneEvaluation` in the same context. A chain producer + /// (`SummaryTumblingWindowChain`) with $\ell$ = `AcrossEvaluations`, with + /// $w$ the chain operator's width in its expression in `memo`, needs every evaluation time of the context on its /// grid: $w$ divides the context's evaluation grid (interval, range step /// and phase), each scheduled time, or a one-off query's time; false /// when the times are not known (an interval without a phase, an @@ -66,15 +64,6 @@ pub struct PhysicalEdge { pub satisfies_physical_requirement: bool, } -/// A physical candidate Stage 2 pruned, named by what it would have been: -/// candidates are renumbered after pruning, so a pruned one has no -/// [`PhysicalCandidateId`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct PhysicalCandidateKey { - pub logical_candidate_id: CandidateForMemoGroupId, - pub physical_properties: PhysicalProperties, -} - /// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] pub struct PhysicalASAPMemoCandidates { @@ -83,11 +72,10 @@ pub struct PhysicalASAPMemoCandidates { pub logical_candidates: LogicalASAPMemoCandidates, /// $\mathcal C$, indexed by [`PhysicalCandidateId`]. pub physical_candidates: Vec, - /// Each query's root over groups, by index in the workload, passed - /// through from Pass 2. - pub query_roots: Vec>, - /// The candidates Stage 2 pruned, in pruning order. - pub pruned_candidates: Vec>, + /// The alternatives Stage 2 pruned, in pruning order, each as the + /// [`PhysicalCandidate`] it would have been: candidates are numbered + /// after pruning, so a pruned one has no [`PhysicalCandidateId`]. + pub pruned_candidates: Vec>, /// $\mathcal E$, grouped by (consumer, input); indexed by /// [`PhysicalEdgeId`]. edges: Vec, @@ -100,13 +88,19 @@ pub struct PhysicalASAPMemoCandidates { impl PhysicalASAPMemoCandidates { /// Stage 2's output over `physical_candidates` and `edges` (grouped by /// (consumer, input)), with the admissible-edge indexes built once. + /// Query roots are read from `logical_candidates`, not copied. pub fn new( logical_candidates: LogicalASAPMemoCandidates, physical_candidates: Vec, - query_roots: Vec>, edges: Vec, + pruned_candidates: Vec>, ) -> Self { - let _ = (logical_candidates, physical_candidates, query_roots, edges); + let _ = ( + logical_candidates, + physical_candidates, + edges, + pruned_candidates, + ); todo!("#1207 PR 5: index the admissible edges") } diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index b5ed3f780..fd2d1ba0b 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -9,5 +9,5 @@ pub use execution_context::{ExecutionContext, ExecutionContextId, RefreshWindowI pub use graph::{DagOfSelectedCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId}; pub use memo_candidates::{ output_references, producer_satisfies_physical_requirements, PhysicalASAPMemoCandidates, - PhysicalCandidate, PhysicalCandidateKey, PhysicalEdge, PhysicalEdgeId, + PhysicalCandidate, PhysicalEdge, PhysicalEdgeId, }; diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index a2f8446f4..360621bcf 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -1,40 +1,41 @@ //! What a stage removed, and why: one shape for every stage (#1047 P6). //! Each stage removes only what its own knowledge rules out (option C): -//! Stage 2 what cannot be implemented, built or materialized. Stage 0 fails with an +//! Stage 2 what cannot be built, run or read by a query output. Stage 0 fails with an //! error, Pass 1 records nothing it leaves out (#1205 will show it), and -//! Pass 2 only adds candidates. Stage 3 prunes before +//! Pass 2 records its eliminated candidates with their own causes +//! (`PrunedLogicalCandidate`). Stage 3 prunes before //! its selection MILP (dominated and uncostable candidates); its //! rejections of whole plans keep their own cause (`RejectionCause` in plan -//! selection). +//! selection). The planner does not check whether an executor implements an +//! operator or family, so no reason names it. -use super::{MemoGroupId, PhysicalCandidateId}; +use super::PhysicalCandidateId; -/// One candidate a stage removed from memo group `group`, and why. `C` is -/// the stage's candidate type: Stage 2's -/// [`PhysicalCandidateKey`](super::PhysicalCandidateKey), Stage 3's -/// [`PhysicalCandidateId`]. +/// One candidate a stage removed, and why. `C` is the stage's candidate +/// type: Stage 2's [`PhysicalCandidate`](super::PhysicalCandidate), Stage +/// 3's [`PhysicalCandidateId`]. The candidate names its group through its +/// logical candidate, so there is no separate group field. #[derive(Debug, Clone, PartialEq)] pub struct PrunedCandidate { - pub group: MemoGroupId, pub candidate: C, pub reason: PruneReason, } /// Why a stage removed a candidate: first Stage 2's (the physical candidate -/// cannot be built, used or materialized), then Stage 3's, before its selection -/// MILP. +/// cannot be built, run or read by a query output), then Stage 3's, before +/// its selection MILP. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PruneReason { - /// Stage 2: the executor cannot run its operator node, an exact - /// per-entity `Aggregate` (a per-series range reduction over raw data - /// samples) with more than one measure, a measure without an exact - /// per-entity kernel (e.g. `stddev_over_time`), a measure filter or a - /// `HAVING` (`agg_has_exact_per_entity_kernel`). Every timing and - /// materialization of the node is pruned. - PerEntityAggregateNotExecutable, + /// Stage 2: it runs at ingestion time, and the source has no ingestion + /// pipeline (its data is at rest or its arrival unknown). + NoIngestionPipeline, /// Stage 2: it scans raw data at query time, and the deployment does /// not retain raw data, so no window of it is there to scan. RawDataNotRetained, + /// Stage 2: it is a query-time result materialized (`OneEvaluation` or + /// `AcrossEvaluations`), and the deployment retains no query-time + /// results. + QueryTimeResultsNotRetained, /// Stage 2: one of its inputs has no producer candidate that can be /// built: none is left, or every one depends, directly or through a /// cycle, on an input that cannot be built. @@ -42,10 +43,6 @@ pub enum PruneReason { /// Stage 2: no query output can read it, directly or through other /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, - /// Stage 2: it is shared within one query evaluation - /// (`OneEvaluation`), which needs at least two reading queries, and fewer - /// queries can reach it. - TooFewQueryReaders, /// Stage 3: the candidate it names, in the same memo group with the /// same producers and consumers, is at least as good in every respect /// (guarantee, usage, latency, output, held state), so no optimal plan diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 71879fdc5..26a791f86 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -4,7 +4,6 @@ //! to the crate-private function of the same name. use crate::ir::materialized_view::RawDataWindow; -use crate::ir::operator::input::node_of_build; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; use crate::ir::properties::summary_coverage::SummaryCoverage; @@ -19,19 +18,6 @@ pub fn definition_id(coverage: &SummaryCoverage) -> String { coverage.definition_id() } -/// `tumbling_window_chain::whole_window_build`, as a node. -pub fn whole_window_build(chain: &Rc) -> Result { - crate::ir::operator::tumbling_window_chain::whole_window_build(chain).map(node_of_build) -} - -/// `tumbling_window_chain::tumbling_window`, as a node. -pub fn tumbling_window( - chain: &Rc, - index: u32, -) -> Result { - crate::ir::operator::tumbling_window_chain::tumbling_window(chain, index).map(node_of_build) -} - /// `Operator::output_schema`. pub fn output_schema(operator: &Operator) -> Result { operator.output_schema() diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 85b33b75c..6f852a44c 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -762,6 +762,7 @@ mod tests { let data = DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 4_000.0, + max_out_of_order_ms: None, }, sources: [("a", Some(100)), ("b", Some(300)), ("c", None)] .into_iter() diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index fe7dec720..3558df41c 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -445,12 +445,21 @@ pub enum DataArrival { /// it, when the source does not declare both its `series_count` and /// its `sample_interval` ([`SourceFacts`]). rows_per_second: f64, + /// How late a row may arrive relative to the newest row of its + /// series, in milliseconds. `None`: a PromQL source is taken to + /// arrive in order, and a SQL source gets no ingestion-time counter + /// build. + #[serde(default, skip_serializing_if = "Option::is_none")] + max_out_of_order_ms: Option, }, /// Older rows are stored and new rows keep arriving. Mixed { /// Rows (PromQL samples) arriving per second over all sources, as /// for [`DataArrival::Ingesting`]. rows_per_second: f64, + /// As for [`DataArrival::Ingesting`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + max_out_of_order_ms: Option, }, /// Not declared: no summary is built at ingestion time. #[default] @@ -461,9 +470,12 @@ impl DataArrival { /// Rows arriving per second, when the data is ingesting or mixed. pub fn rows_per_second(&self) -> Option { match self { - Self::Ingesting { rows_per_second } | Self::Mixed { rows_per_second } => { - Some(*rows_per_second) + Self::Ingesting { + rows_per_second, .. } + | Self::Mixed { + rows_per_second, .. + } => Some(*rows_per_second), Self::AtRest | Self::Unknown => None, } } @@ -1378,7 +1390,10 @@ mod tests { #[test] fn arrival_rate_and_sample_interval_are_validated() { let with = |rows_per_second: f64, sample_interval: Option| DataWorkload { - arrival: DataArrival::Mixed { rows_per_second }, + arrival: DataArrival::Mixed { + rows_per_second, + max_out_of_order_ms: None, + }, sources: BTreeMap::from([( "m".to_string(), SourceFacts { @@ -1611,6 +1626,7 @@ mod tests { data_workload: Some(DataWorkload { arrival: DataArrival::Ingesting { rows_per_second: 1_000.0, + max_out_of_order_ms: None, }, sources: BTreeMap::from([( "requests_total".to_string(), diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index afb075fa9..f98eaa7ef 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -58,7 +58,6 @@ fn family_state(family: FieldDataType, region: &str) -> Rc { fn compatible_tumbling_windows_merge_structurally() { let root = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children: vec![state(200, "us"), state(200, "eu")], - newest_tumbling_windows: None, group_by: None, })) .unwrap(); @@ -74,7 +73,6 @@ fn incompatible_merge_inputs_fail() { assert!( OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children, - newest_tumbling_windows: None, group_by: None, })) .is_err() @@ -130,7 +128,6 @@ fn cms_with_heap() -> FieldDataType { fn merge(children: Vec>) -> Result, String> { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { children, - newest_tumbling_windows: None, group_by: None, })) .map_err(|e| e.to_string()) diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 376ab310f..439a87edb 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -1,18 +1,13 @@ -//! A `SummaryTumblingWindowChain` of tumbling windows stands for its tumbling windows: it covers what one -//! build over its whole window does, and expands into the `SummaryAgg` -//! tumbling windows the executor runs (#509 Pass 2, window-composition rule). +//! A `SummaryTumblingWindowChain`: one state per window of its width, with +//! no range and no window count (concepts doc, section 3, "Chain groups"). use std::rc::Rc; use std::time::Duration; -use asap_types::ir::operator::operator_properties::{Reduction, Source, TimeShift}; -use asap_types::ir::operator::tumbling_window_chain::{ - expand_tumbling_window_chains, merged_chain, -}; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate}; +use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; -use asap_types::test_support::{tumbling_window, whole_window_build}; const MINUTE: u64 = 60_000; @@ -32,22 +27,11 @@ fn scan() -> Rc { .unwrap() } -fn shifted(child: Rc, offset_ms: i64) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift: TimeShift { - offset_ms, - at: None, - }, - child, - })) - .unwrap() -} - fn kll() -> FieldDataType { FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) } -/// Five 1-min KLL tumbling windows per series over `input`. +/// 1-min KLL tumbling windows per series over `input`. fn chain(input: Rc) -> Rc { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: input, @@ -57,91 +41,12 @@ fn chain(input: Rc) -> Rc { grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: MINUTE, - tumbling_window_count: 5, })) .unwrap() } -fn merge(children: Vec>) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: None, - group_by: None, - })) - .unwrap() -} - -/// The merged chain is a state of the build over the whole 5-min window: -/// the same schema, and it validates. -#[test] -fn merged_tumbling_windows_cover_the_whole_window() { - let chain = chain(scan()); - let merged = merge(vec![Rc::clone(&chain)]); - merged.validate_structure().unwrap(); - assert!(merged_chain(&merged).is_some_and(|c| Rc::ptr_eq(&c, &chain))); - let whole = whole_window_build(&chain).unwrap(); - let Some(NonASAPOp::TimeRange { range, kind, .. }) = whole.children()[0].non_asap() else { - panic!("{:?}", whole.operator) - }; - assert_eq!( - (*range, *kind), - (Duration::from_millis(5 * MINUTE), TimeRangeKind::Range) - ); - assert_eq!(merged.schema, whole.schema); -} - -/// Tumbling window `i` is the build over `(-(i+1)·1m, -i·1m]`, the query's own offset -/// folded into one shift; the tumbling windows are disjoint and tile the window. -#[test] -fn each_tumbling_window_is_one_shifted_minute() { - let offset = 3_600_000; - let chain = chain(shifted(scan(), offset)); - for i in 0..5 { - let tumbling_window = tumbling_window(&chain, i).unwrap(); - let back = offset + i64::from(i) * MINUTE as i64; - let range = tumbling_window.children()[0]; - let Some(NonASAPOp::TimeShift { shift, child }) = range.children()[0].non_asap() else { - panic!("tumbling window {i} reads a shift") - }; - assert_eq!(shift.offset_ms, back); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Scan { .. }))); - } -} - -/// Expansion gives the executor a merge over the five tumbling windows, newest first, -/// newest first; a chain read without a merge -/// does not expand. -#[test] -fn expansion_merges_the_tumbling_windows() { - let chain = chain(scan()); - let merged = merge(vec![Rc::clone(&chain)]); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&merged), - query: SketchStatistic::Quantile { q: 0.99 }, - })) - .unwrap(); - let expanded = expand_tumbling_window_chains(&estimate).unwrap(); - expanded.validate_structure().unwrap(); - assert_eq!(expanded.schema, estimate.schema); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = expanded.asap() else { - panic!("{:?}", expanded.operator) - }; - let Some(ASAPOp::SummaryMerge { children, .. }) = summary_input.asap() else { - panic!("{:?}", summary_input.operator) - }; - assert_eq!(children.len(), 5); - for (i, child) in children.iter().enumerate() { - assert_eq!( - **child, - tumbling_window(&chain, i as u32).unwrap(), - "tumbling window {i}" - ); - } - assert!(expand_tumbling_window_chains(&chain).is_err()); -} - -/// A chain needs a positive width and count, and reads the instant vector -/// a range selector would. +/// A chain needs a positive width, and reads the instant vector a range +/// selector would. #[test] fn a_chain_reads_an_instant_vector() { let ranged = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { @@ -161,109 +66,24 @@ fn a_chain_reads_an_instant_vector() { reduction: Reduction::PerEntity, grouping: GroupingStrategy::default(), filter: None, - tumbling_window_width_ms: MINUTE, - tumbling_window_count: 0, + tumbling_window_width_ms: 0, })) .unwrap(); assert!(empty.validate_structure().is_err()); } -/// `newest_tumbling_windows` must name between 1 and fewer than all tumbling windows of the one -/// chain the merge reads; anything else is rejected. -#[test] -fn a_merge_of_the_newest_tumbling_windows_reads_one_chain_and_fewer_than_all_tumbling_windows() { - let chain = chain(scan()); - let build = whole_window_build(&chain).unwrap(); - for (children, newest_tumbling_windows) in [ - (vec![Rc::clone(&chain)], 0), - (vec![Rc::clone(&chain)], 5), - (vec![Rc::clone(&chain)], 6), - (vec![Rc::clone(&chain), Rc::clone(&chain)], 2), - (vec![Rc::new(build)], 1), - ] { - let merge = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children, - newest_tumbling_windows: Some(newest_tumbling_windows), - group_by: None, - })); - assert!(merge.is_err(), "newest {newest_tumbling_windows}"); - } -} - -/// `newest_tumbling_windows` is left out of the JSON when `None`, and JSON without it -/// reads as `None`; `Some(n)` round-trips. -#[test] -fn newest_tumbling_windows_serializes_only_when_set() { - let full: ASAPOp<()> = ASAPOp::SummaryMerge { - children: vec![()], - newest_tumbling_windows: None, - group_by: None, - }; - let json = serde_json::to_string(&full).unwrap(); - assert!(!json.contains("newest_tumbling_windows"), "{json}"); - let read: ASAPOp<()> = serde_json::from_str(r#"{"SummaryMerge":{"children":[null]}}"#).unwrap(); - assert_eq!(read, full); - let partial: ASAPOp<()> = ASAPOp::SummaryMerge { - children: vec![()], - newest_tumbling_windows: Some(6), - group_by: None, - }; - let json = serde_json::to_string(&partial).unwrap(); - assert_eq!(serde_json::from_str::>(&json).unwrap(), partial); -} - -/// A rollup of a chain's tumbling windows to no keys (every series' tumbling windows into one -/// state) is no tumbling-window merge, and expansion keeps its keys. -#[test] -fn expansion_keeps_a_rollup_of_the_tumbling_windows() { - let chain = chain(scan()); - let rollup = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&chain)], - group_by: Some(vec![]), - newest_tumbling_windows: None, - })) - .unwrap(); - assert!(merged_chain(&rollup).is_none()); - let expanded = expand_tumbling_window_chains(&rollup).unwrap(); - let Some(ASAPOp::SummaryMerge { - children, group_by, .. - }) = expanded.asap() - else { - panic!("{:?}", expanded.operator) - }; - assert_eq!((children.len(), group_by), (5, &Some(vec![]))); - // A rollup merges every input state: it reads a merge of the newest - // tumbling windows, it is not one. - assert!(OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - newest_tumbling_windows: Some(2), - group_by: Some(vec![]), - })) - .is_err()); -} - -/// JSON written before #1163 names the chain `PaneChain`, its fields -/// `tumbling_window_width_ms` and `tumbling_window_count`, and a merge's `newest_tumbling_windows`; it reads -/// as today's operators. +/// JSON written before #1163 names the chain `PaneChain` and its width +/// `pane_width_ms`; it reads as today's operator. #[test] fn names_before_the_tumbling_window_rename_still_read() { - let chain = chain(scan()).operator.map_children(|_| ()); - let merge: Operator<()> = Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![()], - newest_tumbling_windows: Some(3), - group_by: None, - }); - for operator in [chain, merge] { - let old = serde_json::to_string(&operator) - .unwrap() - .replace("SummaryTumblingWindowChain", "PaneChain") - .replace("tumbling_window_width_ms", "pane_width_ms") - .replace("tumbling_window_count", "pane_count") - .replace("newest_tumbling_windows", "newest_panes"); - assert!(!old.contains("umbling"), "{old}"); - assert_eq!( - serde_json::from_str::>(&old).unwrap(), - operator - ); - } + let operator = chain(scan()).operator.map_children(|_| ()); + let old = serde_json::to_string(&operator) + .unwrap() + .replace("SummaryTumblingWindowChain", "PaneChain") + .replace("tumbling_window_width_ms", "pane_width_ms"); + assert!(!old.contains("umbling"), "{old}"); + assert_eq!( + serde_json::from_str::>(&old).unwrap(), + operator + ); } diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 6caa8b0a8..b95d1ec96 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -44,12 +44,12 @@ Implemented: population, including membership changes. - `EvaluatePopulation { child, evaluation }`: read an aggregate or TopK prefix from a maintained population. -- `SummaryMerge { children, newest_tumbling_windows, group_by }`: merge summary states with identical +- `SummaryMerge { children, group_by }`: merge summary states with identical schemas (same family, parameters and grouping) into the state of their union; its coverage check rejects inputs whose selections overlap. With `group_by` (a rollup) the output is keyed by those input group columns, and - each output state merges the input states of its key; `newest_tumbling_windows` - merges only the newest tumbling windows of one tumbling-window chain, never with `group_by`. The timing pass does + each output state merges the input states of its key. Over a `TimeRange` + over a tumbling-window chain it merges the windows that range selects. The timing pass does not place it yet (`ExecutionDataStateError::UnimplementedOperator`). Reserved (migrated but unimplemented; schema derivation rejects them with diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 4bf59e9ea..84250a404 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -665,11 +665,11 @@ With an exact target, the planner can keep all latencies per job in one maintain - **Implemented.** All children must be state with exactly one state field and identical schemas. - **Coverage.** It also requires equal `definition`s and selections that do not overlap, and computes the merged coverage. -- **Newest tumbling windows:** an optional `newest_tumbling_windows: n` merges only the newest `n` tumbling windows of the one `SummaryTumblingWindowChain` the merge reads (`1 ≤ n <` its tumbling-window count). Its coverage is one build over the time range those tumbling windows cover, so one chain kept for the longest window serves every query whose window, on the same tumbling windows, ends at the same time and is shorter. -- **Rollup:** an optional `group_by` field, so one operator does both merge and rollup. It lists the input's group columns the output is keyed by; `None` keeps the input's groups. A rollup merges every input state, so it does not take `newest_tumbling_windows`; it reads a merge of the newest tumbling windows instead: +- **Chain windows:** a merge over a `TimeRange` (and `TimeShift`s) over a `SummaryTumblingWindowChain` merges the windows that range selects; there is no window-selection field. Its coverage is that of one build over the same `TimeRange` and `TimeShift`s, so one chain serves every query over the same input whose range and offset lie on its windows. +- **Rollup:** an optional `group_by` field, so one operator does both merge and rollup. It lists the input's group columns the output is keyed by; `None` keeps the input's groups. A rollup merges every input state: ```rust -SummaryMerge { children: Vec, newest_tumbling_windows: Option, group_by: Option> } +SummaryMerge { children: Vec, group_by: Option> } ``` **Example A: time tumbling windows.** p99 of each series of metric `m` over the last 2 minutes: @@ -822,7 +822,7 @@ input groups output groups | Question | Answer | |---|---| | What comes out? | the children's schema; with `group_by`, its columns in order, then the state column. A `group_by` of no columns over a SQL relation's states stands for a global SQL aggregate, which has one row over no input rows: it outputs one state, empty when there is no input state | -| When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra); `group_by` together with `newest_tumbling_windows` | +| When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra) | | What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(−2m, −1m]` and `(−1m, 0]` give `(−2m, 0]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | | State or value? | state in, state out | @@ -985,9 +985,6 @@ pub enum ASAPOp> { SummaryMerge { /// The states to merge; all have the same schema. children: Vec, - /// Merge only the newest `n` tumbling windows of the one `SummaryTumblingWindowChain` read; - /// `None` merges every input state. Not with `group_by`. - newest_tumbling_windows: Option, /// A rollup: the input's group columns the merged states are keyed /// by, in output order; every input state whose values in them match /// goes into one output state. `None` keeps the input's groups. @@ -1358,11 +1355,17 @@ pub struct SummaryCoverage { pub struct SelectionBox { /// One constraint per restricted column. A column not in the map is /// unrestricted. + /// The timestamp column is one of them; there is no separate time + /// field. pub columns: BTreeMap, - /// The time window relative to the evaluation time, in ms, e.g. - /// `(Excluded(-120000), Included(-60000))` for `(−2m, −1m]`. - /// `None`: no time restriction. - pub relative_time: Option<(Bound, Bound)>, +} + +/// An endpoint of an interval on a timestamp column. +pub enum TimeBound { + /// An absolute time, in ms since the epoch. + At(i64), + /// An offset, in ms, from the evaluation time t. + FromEvaluationTime(i64), } /// Names a column across nodes, independent of its position. diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 3c0678a33..13a1c6b07 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -567,7 +567,7 @@ The sections below describe each stage. | 0. Frontends | nothing: a query that cannot be lowered fails planning with an error | — | — | | 1, Pass 1 | what the measure or the deployment cannot compute: `RequiresExact`, `NullableAvgInput`, `NoRelationalCountSketchRealization`, `UnsupportedByDeployment`, `QuantileOutsideUnitInterval`, `TopKHeapOverOneRowPerItem`; and, only when the deployment's accuracy model and evidence cannot accept an output without its guarantee, a summary no query reading it can use: `NoErrorBoundForQueryTarget` | `PerComputationLogicalMemoCandidates::pruned_candidates`, each on its aggregate's memo group | the way to compute the measure (`HowToComputeAggregateMeasure`) | | 1, Pass 2 | nothing: it only adds candidates | — | — | -| 2 | what cannot be implemented, built or materialized: `PerEntityAggregateNotExecutable` (an exact per-entity `Aggregate` the executor has no kernel for), `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, `TooFewQueryReaders` (shared within one evaluation with fewer than two reading queries) | `PhysicalASAPMemoCandidates::pruned_candidates`, each on its operator node's memo group | `PhysicalCandidateKey { operator_node, properties }` | +| 2 | what cannot be built or materialized: `NoIngestionPipeline`, `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `QueryTimeResultsNotRetained`, `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`; the planner does not check executor support | `PhysicalASAPMemoCandidates::pruned_candidates` | the `PhysicalCandidate` it would have been | | 3 | valid candidates that lose, and the checks only a whole plan can make (accuracy, latency, cluster capacity) | its own `RejectionCause` | — | Stage 2's key names what a pruned candidate would have been, since kept @@ -600,7 +600,7 @@ the memo path's entry points. | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -1120,9 +1120,9 @@ kept and more tumbling windows merged, so Stage 3's pricing decides which grid, any, pays; no tumbling-window-count limit is imposed beyond the tumbling-window count fitting in 32 bits. The chain covers the longest window and is sized for all of them, each parameter at its largest value among them, as the -summary-capability rule sizes a shared build. Each query merges only the -newest tumbling windows covering its own window (`SummaryMerge.newest_tumbling_windows`), since a -shorter window ending at the same time is the newest part of the longest. +summary-capability rule sizes a shared build. Each query merges the +windows a `TimeRange` over the chain selects for its own range; the chain +has no range and no window count of its own. For example, `quantile_over_time(0.5, m[1m])` and `quantile_over_time(0.9, m[5m])`, both every 10 s, read one chain of thirty 10-s KLL tumbling windows: the 1-min query merges the newest 6, the 5-min query all @@ -1286,15 +1286,13 @@ cheapest plan (the typical outcomes above). On the memo, Stage 2 prunes per operator node and records each pruned candidate in `PhysicalASAPMemoCandidates::pruned_candidates` ([Pruning by stage](#pruning-by-stage)). It does not generate a -configuration the deployment cannot run: an exact per-entity `Aggregate` -(a per-series range reduction over raw data samples) with more than one -measure, a measure without an exact per-entity kernel such as -`stddev_over_time`, a measure filter or a `HAVING` -(`PerEntityAggregateNotExecutable`; the rule reads one node, so no whole -plan is needed), and, without raw data retained, a query-time scan of raw -data (`RawDataNotRetained`). It then removes the candidates no buildable -input, query output or reader count can use (`InputHasNoBuildableProducer`, -`ReachesNoQueryOutput`, `TooFewQueryReaders`), +configuration the deployment cannot run: an ingestion-time alternative +without an ingestion pipeline (`NoIngestionPipeline`), a query-time scan +of raw data the deployment does not retain (`RawDataNotRetained`), and a +materialized query-time result the deployment does not retain +(`QueryTimeResultsNotRetained`). It does not check executor support. It +then removes the candidates no buildable input or query output can use +(`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`), repeating until nothing more is removed. Stage 2 makes no execution-scope copies: how many times a candidate runs is Stage 3's choice, and a selection names each run as a node of its `DagOfSelectedCandidates`. diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 98c9219ce..1b0e785e7 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -92,13 +92,14 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types - On the memo (`generate_physical_asap_memo_candidates`), every physical candidate Stage 2 prunes is recorded in `PhysicalASAPMemoCandidates::pruned_candidates` as - `PrunedCandidate` with a `PruneReason`: an exact - per-entity `Aggregate` the executor cannot run - (`PerEntityAggregateNotExecutable`) and, without raw data retained, a - query-time scan of raw data (`RawDataNotRetained`) are not generated; a - candidate no buildable input, query output or reader count can use is - removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`, - `TooFewQueryReaders`). Stage 2 emits one candidate per operator node, + `PrunedCandidate` with a `PruneReason`: an + ingestion-time alternative without an ingestion pipeline + (`NoIngestionPipeline`), a query-time scan of raw data the deployment does + not retain (`RawDataNotRetained`) and a materialized query-time result the + deployment does not retain (`QueryTimeResultsNotRetained`) are not + generated; a candidate no buildable input or query output can use is + removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`). The + planner does not check executor support. Stage 2 emits one candidate per operator node, timing and lifetime; it makes no execution-scope copies. ## 5. Stage 3 From 8a9886676d08ec0c01b22471ed518d44b6a17e80 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:50:02 +0000 Subject: [PATCH 15/30] refactor(types): IntervalConstraintLimit ends for Constraint::Interval; ScalarValue::Timestamp (#1207 PR 0) Replace TimeBound by IntervalConstraintLimit { FixedValue(ScalarValue), EvaluationTimePlusMs(i64) }: Constraint::Interval holds Bound on every column, so the timestamp column and value columns share one interval type. Doc comments carry the examples (m[5m], offset 1h, @ T, SQL ts >= literal, price > 10). ScalarValue::Timestamp(ms since the epoch) is the fixed value of an @ T or a SQL timestamp literal. --- crates/executor/src/expressions/columnar.rs | 1 + crates/executor/src/expressions/planner.rs | 1 + crates/planner/src/lib.rs | 2 +- .../src/ir/properties/summary_coverage.rs | 187 +++++++++++++++--- crates/types/src/ir/scalar/expr_ir.rs | 3 + crates/types/src/ir/scalar/mod.rs | 1 + .../proposals/asap-primitive-schema.md | 29 ++- 7 files changed, 183 insertions(+), 41 deletions(-) diff --git a/crates/executor/src/expressions/columnar.rs b/crates/executor/src/expressions/columnar.rs index 9c15bc96b..dbb37b317 100644 --- a/crates/executor/src/expressions/columnar.rs +++ b/crates/executor/src/expressions/columnar.rs @@ -180,6 +180,7 @@ fn literal(value: &ScalarValue) -> (Value, DataType) { ScalarValue::Float64(v) => (Value::Float64(*v), DataType::Float64), ScalarValue::Utf8(v) => (Value::Utf8(v.as_str().into()), DataType::Utf8), ScalarValue::Boolean(v) => (Value::Bool(*v), DataType::Bool), + ScalarValue::Timestamp(ms) => (Value::Timestamp(*ms), DataType::Timestamp), ScalarValue::Interval { months, days, diff --git a/crates/executor/src/expressions/planner.rs b/crates/executor/src/expressions/planner.rs index dc49634a6..349496419 100644 --- a/crates/executor/src/expressions/planner.rs +++ b/crates/executor/src/expressions/planner.rs @@ -71,6 +71,7 @@ pub(super) fn evaluate( ScalarValue::Utf8(value) => Value::Utf8(value.clone().into()), ScalarValue::Boolean(value) => Value::Bool(*value), ScalarValue::Null => Value::Null, + ScalarValue::Timestamp(ms) => Value::Timestamp(*ms), }), ScalarExpr::Cast { expr, to, .. } => { let value = evaluate(expr, row, schema, evaluation_time_ms, regexes)?; diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index 364281454..f114db832 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -61,7 +61,7 @@ pub use asap_types::ir::materialized_view::{ }; pub use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; pub use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, SelectionBox, TimeBound, + ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::ir::schema::{SketchAlgorithm, SketchParams, StatisticKind}; diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 6509507a6..8a9e5893f 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -82,16 +82,6 @@ pub struct SelectionBox { pub(crate) columns: BTreeMap, } -/// An endpoint of an interval on a timestamp column. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TimeBound { - /// An absolute time, in ms since the epoch. - At(i64), - /// An offset, in ms, from the evaluation time t. - FromEvaluationTime(i64), -} - /// `SelectionBox::columns` as a list of `{column, constraint}`. mod columns_as_list { use super::{ColumnIdentity, Constraint}; @@ -179,28 +169,57 @@ impl ColumnIdentity { } } +/// A constraint on one column of a selection. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Constraint { + /// The value is one of these. In(Vec), + /// The value is none of these. NotIn(Vec), + /// The value lies between `lower` and `upper`; each end is `Included`, + /// `Excluded` or `Unbounded`. Examples, on the timestamp column `ts` + /// (ms) unless stated: + /// + /// | Query fragment | `lower` | `upper` | + /// |---|---|---| + /// | PromQL `m[5m]` | `Included(EvaluationTimePlusMs(-300_000))` | `Excluded(EvaluationTimePlusMs(0))` | + /// | PromQL `m[5m] offset 1h` | `Included(EvaluationTimePlusMs(-3_900_000))` | `Excluded(EvaluationTimePlusMs(-3_600_000))` | + /// | PromQL `m[5m] @ 1700000000` | `Included(FixedValue(Timestamp(1_699_999_700_000)))` | `Excluded(FixedValue(Timestamp(1_700_000_000_000)))` | + /// | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | + /// | SQL `WHERE price > 10` (column `price`) | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | Interval { - #[serde(with = "value_bound")] - lower: Bound, - #[serde(with = "value_bound")] - upper: Bound, + #[serde(with = "limit_bound")] + lower: Bound, + #[serde(with = "limit_bound")] + upper: Bound, }, } +/// An end of a [`Constraint::Interval`]. Two limits compare only when they +/// are of the same variant: a fixed value against a fixed value of the same +/// type, an offset from the evaluation time against another such offset. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IntervalConstraintLimit { + /// A constant: `Float64(10.0)` for SQL `price > 10`; + /// `Timestamp(1_700_000_000_000)` for PromQL `@ 1700000000` or a SQL + /// timestamp literal. + FixedValue(ScalarValue), + /// The evaluation time t of a repeatedly evaluated query plus this many + /// ms (negative: before t); only on the timestamp column. `m[5m]` is + /// `[EvaluationTimePlusMs(-300_000), EvaluationTimePlusMs(0))`. + EvaluationTimePlusMs(i64), +} + /// A `Constraint::Interval` bound as a [`WireBound`]. -mod value_bound { - use super::WireBound; - use crate::ir::scalar::expr_ir::ScalarValue; +mod limit_bound { + use super::{IntervalConstraintLimit, WireBound}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::ops::Bound; pub(super) fn serialize( - bound: &Bound, + bound: &Bound, serializer: S, ) -> Result { WireBound::from(bound).serialize(serializer) @@ -208,7 +227,7 @@ mod value_bound { pub(super) fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, - ) -> Result, D::Error> { + ) -> Result, D::Error> { Ok(WireBound::deserialize(deserializer)?.into()) } } @@ -411,7 +430,7 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result bool = match required { MergeInputRule::MayOverlap => |_, _| true, MergeInputRule::DisjointInTime => todo!( - "#1207 PR 2: disjoint intervals on the timestamp column, with TimeBound endpoints" + "#1207 PR 2: disjoint intervals on the timestamp column, with IntervalConstraintLimit ends" ), MergeInputRule::Disjoint => SelectionBox::disjoint, }; @@ -625,7 +644,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result todo!( - "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with TimeBound endpoints" + "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with IntervalConstraintLimit ends" ), other => unlifted_operators.push(unlifted(other)), } @@ -818,8 +837,12 @@ fn flip(op: &CompareOpKind) -> Option { }) } +/// An interval between two fixed values. fn interval(lower: Bound, upper: Bound) -> Constraint { - Constraint::Interval { lower, upper } + Constraint::Interval { + lower: lower.map(IntervalConstraintLimit::FixedValue), + upper: upper.map(IntervalConstraintLimit::FixedValue), + } } fn dedup(values: Vec) -> Vec { @@ -862,10 +885,37 @@ fn compare(a: &ScalarValue, b: &ScalarValue) -> Option { (ScalarValue::Float64(a), ScalarValue::Float64(b)) => a.partial_cmp(b), (ScalarValue::Utf8(a), ScalarValue::Utf8(b)) => Some(a.cmp(b)), (ScalarValue::Boolean(a), ScalarValue::Boolean(b)) => Some(a.cmp(b)), + (ScalarValue::Timestamp(a), ScalarValue::Timestamp(b)) => Some(a.cmp(b)), _ => None, } } +/// Order of two interval limits; `None` across variants or when the fixed +/// values do not compare. +fn compare_limits(a: &IntervalConstraintLimit, b: &IntervalConstraintLimit) -> Option { + use IntervalConstraintLimit::*; + match (a, b) { + (FixedValue(a), FixedValue(b)) => compare(a, b), + (EvaluationTimePlusMs(a), EvaluationTimePlusMs(b)) => Some(a.cmp(b)), + _ => None, + } +} + +/// Whether value `v` lies inside the interval; `None` when unsure. +fn value_inside( + v: &ScalarValue, + bounds: ( + &Bound, + &Bound, + ), +) -> Option { + inside( + &IntervalConstraintLimit::FixedValue(v.clone()), + bounds, + compare_limits, + ) +} + /// Whether `v` lies inside the interval; `None` when it cannot be compared. fn inside( v: &T, @@ -910,18 +960,18 @@ fn intervals_disjoint( /// The tighter of two lower (`want = Greater`) or upper (`Less`) bounds. fn tighter( - a: &Bound, - b: &Bound, + a: &Bound, + b: &Bound, want: Ordering, -) -> Option> { - let value = |bound: &Bound| match bound { +) -> Option> { + let value = |bound: &Bound| match bound { Bound::Included(v) | Bound::Excluded(v) => Some(v.clone()), Bound::Unbounded => None, }; Some(match (value(a), value(b)) { (None, _) => b.clone(), (_, None) => a.clone(), - (Some(x), Some(y)) => match compare(&x, &y)? { + (Some(x), Some(y)) => match compare_limits(&x, &y)? { Ordering::Equal if matches!(a, Bound::Excluded(_)) => a.clone(), Ordering::Equal => b.clone(), order if order == want => a.clone(), @@ -942,7 +992,7 @@ impl Constraint { (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => { let mut kept = Vec::new(); for v in a { - if inside(v, (lower, upper), compare)? { + if value_inside(v, (lower, upper))? { kept.push(v.clone()); } } @@ -973,7 +1023,7 @@ impl Constraint { (In(a), NotIn(b)) | (NotIn(b), In(a)) => a.iter().all(|v| among(v, b) == Some(true)), (In(a), Interval { lower, upper }) | (Interval { lower, upper }, In(a)) => a .iter() - .all(|v| inside(v, (lower, upper), compare) == Some(false)), + .all(|v| value_inside(v, (lower, upper)) == Some(false)), ( Interval { lower: al, @@ -983,7 +1033,7 @@ impl Constraint { lower: bl, upper: bu, }, - ) => intervals_disjoint((al, au), (bl, bu), compare), + ) => intervals_disjoint((al, au), (bl, bu), compare_limits), (NotIn(_), _) | (_, NotIn(_)) => false, } } @@ -1059,7 +1109,7 @@ fn join(a: &SelectionBox, b: &SelectionBox) -> Option { upper: bu, }, ) => { - let (lower, upper) = touching((al, au), (bl, bu), compare)?; + let (lower, upper) = touching((al, au), (bl, bu), compare_limits)?; Constraint::Interval { lower, upper } } _ => return None, @@ -1090,3 +1140,78 @@ fn touching( None } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::scalar::ExprSemantics; + use IntervalConstraintLimit::{EvaluationTimePlusMs, FixedValue}; + + fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Included(EvaluationTimePlusMs(lower_ms)), + upper: Bound::Excluded(EvaluationTimePlusMs(upper_ms)), + } + } + + /// SQL `price > 10` lifts to `(FixedValue(10), ∞)`. + #[test] + fn a_value_comparison_is_an_interval_between_fixed_values() { + let price_gt_10: ScalarExpr<()> = ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + }; + assert_eq!( + constraint_of(&price_gt_10), + Some(( + 0, + Constraint::Interval { + lower: Bound::Excluded(FixedValue(ScalarValue::Float64(10.0))), + upper: Bound::Unbounded, + } + )) + ); + } + + /// `m[5m]` and `m[5m] offset 5m` are disjoint and touch; neither + /// compares with an `@ T` interval. + #[test] + fn limits_compare_only_within_one_variant() { + let last_5m = relative(-300_000, 0); + let offset_5m = relative(-600_000, -300_000); + assert!(last_5m.disjoint(&offset_5m)); + let at = Constraint::Interval { + lower: Bound::Included(FixedValue(ScalarValue::Timestamp(1_699_999_700_000))), + upper: Bound::Excluded(FixedValue(ScalarValue::Timestamp(1_700_000_000_000))), + }; + assert!(!last_5m.disjoint(&at)); + assert_eq!(last_5m.intersect(&at), None); + let column = ColumnIdentity::new(None, "ts"); + let one = |constraint: Constraint| SelectionBox { + columns: BTreeMap::from([(column.clone(), constraint)]), + }; + assert_eq!( + union(vec![one(last_5m), one(offset_5m)]), + vec![one(relative(-600_000, 0))] + ); + } + + #[test] + fn an_interval_round_trips_through_json() { + let constraint = relative(-3_900_000, -3_600_000); + let json = serde_json::to_value(&constraint).unwrap(); + assert_eq!( + json, + serde_json::json!({"interval": { + "lower": {"included": {"evaluation_time_plus_ms": -3_900_000}}, + "upper": {"excluded": {"evaluation_time_plus_ms": -3_600_000}}, + }}) + ); + assert_eq!( + serde_json::from_value::(json).unwrap(), + constraint + ); + } +} diff --git a/crates/types/src/ir/scalar/expr_ir.rs b/crates/types/src/ir/scalar/expr_ir.rs index 50606f185..3ee1dd56d 100644 --- a/crates/types/src/ir/scalar/expr_ir.rs +++ b/crates/types/src/ir/scalar/expr_ir.rs @@ -39,6 +39,9 @@ pub enum ScalarValue { Utf8(String), Boolean(bool), Null, + /// A point in time, in ms since the Unix epoch: PromQL `@ 1700000000` + /// is `Timestamp(1_700_000_000_000)`. + Timestamp(i64), /// A calendar duration, in Arrow's three independent fields. Not collapsed /// into one nanosecond count: a month is not a fixed number of nanoseconds, /// so `INTERVAL '1' MONTH` has no faithful scalar form. DataFusion's three diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 4a312b0b9..01ef2984a 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -382,6 +382,7 @@ impl ScalarExpr { ScalarValue::Boolean(_) => (DataType::Bool, false), ScalarValue::Null => (DataType::Null, true), ScalarValue::Interval { .. } => (DataType::Interval, false), + ScalarValue::Timestamp(_) => (DataType::Timestamp, false), }, ScalarExpr::Compare { left, right, op, .. diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 84250a404..b29642abe 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -1360,14 +1360,6 @@ pub struct SelectionBox { pub columns: BTreeMap, } -/// An endpoint of an interval on a timestamp column. -pub enum TimeBound { - /// An absolute time, in ms since the epoch. - At(i64), - /// An offset, in ms, from the evaluation time t. - FromEvaluationTime(i64), -} - /// Names a column across nodes, independent of its position. pub struct ColumnIdentity { /// The table or alias the column comes from; `None` if unqualified. @@ -1384,10 +1376,29 @@ pub enum Constraint { NotIn(Vec), /// The value lies between `lower` and `upper`. Each end is `Included`, /// `Excluded` or `Unbounded`. - Interval { lower: Bound, upper: Bound }, + Interval { lower: Bound, upper: Bound }, // HashPartition { columns, of, index }: added with its first producer. } +/// An end of an interval. Limits compare only within one variant. +pub enum IntervalConstraintLimit { + /// A constant, e.g. `Float64(10.0)` or `Timestamp(ms since the epoch)`. + FixedValue(ScalarValue), + /// The evaluation time t of a repeatedly evaluated query plus this + /// many ms; only on the timestamp column. + EvaluationTimePlusMs(i64), +} +``` + +| Query fragment | `lower` | `upper` | +|---|---|---| +| PromQL `m[5m]` | `Included(EvaluationTimePlusMs(-300_000))` | `Excluded(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Included(EvaluationTimePlusMs(-3_900_000))` | `Excluded(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ 1700000000` | `Included(FixedValue(Timestamp(1_699_999_700_000)))` | `Excluded(FixedValue(Timestamp(1_700_000_000_000)))` | +| SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | +| SQL `WHERE price > 10` | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | + +```rust impl SummaryCoverage { /// Computes the coverage of a summary node from its sub-DAG (§4.2.2). /// Errors: `NotSummary` for a node that is not a `SummaryAgg`, From 31f25bbe907d7efc142f1b98f42190c241e1a9d2 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:03:03 +0000 Subject: [PATCH 16/30] refactor(types): PromQL ranges are (t - r, t] in the interval examples; delete ParametersOfAnotherFamily (#1207 PR 0) A range m[r] evaluated at t reads (t - r, t]: IntervalConstraintLimit and Constraint::Interval examples, the chain window doc and the primitive-schema table now use Excluded lower and Included upper ends. A parameter list holds only its own family's parameters, so ParametersNotInList also rejects another family's parameters. --- crates/types/src/ir/operator/asap.rs | 2 +- .../src/ir/properties/summary_coverage.rs | 25 ++++++++++--------- crates/types/src/stages/candidates.rs | 2 -- .../proposals/asap-primitive-schema.md | 8 +++--- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 8a2941a24..962c1884f 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -73,7 +73,7 @@ pub enum ASAPOp> { /// A chain of tumbling windows (concepts doc, section 3, "Chain /// groups"): the `SummaryAgg` with these `family`, `input`, `reduction`, /// `grouping` and `filter`, built over each window - /// `[k·w, (k+1)·w)` of `child`, `w` = `tumbling_window_width_ms`, on a + /// `(k·w, (k+1)·w]` of `child`, `w` = `tumbling_window_width_ms`, on a /// grid anchored at the epoch. Each window is built when it ends. Output: /// one row per window, (window end, keys, state), a state with a time /// index. It has no range and no window count: a `TimeRange` (and diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 8a9e5893f..9a391957a 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -178,14 +178,15 @@ pub enum Constraint { /// The value is none of these. NotIn(Vec), /// The value lies between `lower` and `upper`; each end is `Included`, - /// `Excluded` or `Unbounded`. Examples, on the timestamp column `ts` - /// (ms) unless stated: + /// `Excluded` or `Unbounded`. A PromQL range `m[r]` evaluated at t reads + /// the samples in (t − r, t]: left-open, right-closed. Examples, on the + /// timestamp column `ts` (ms) unless stated: /// /// | Query fragment | `lower` | `upper` | /// |---|---|---| - /// | PromQL `m[5m]` | `Included(EvaluationTimePlusMs(-300_000))` | `Excluded(EvaluationTimePlusMs(0))` | - /// | PromQL `m[5m] offset 1h` | `Included(EvaluationTimePlusMs(-3_900_000))` | `Excluded(EvaluationTimePlusMs(-3_600_000))` | - /// | PromQL `m[5m] @ 1700000000` | `Included(FixedValue(Timestamp(1_699_999_700_000)))` | `Excluded(FixedValue(Timestamp(1_700_000_000_000)))` | + /// | PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | + /// | PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | + /// | PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | /// | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | /// | SQL `WHERE price > 10` (column `price`) | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | Interval { @@ -208,7 +209,7 @@ pub enum IntervalConstraintLimit { FixedValue(ScalarValue), /// The evaluation time t of a repeatedly evaluated query plus this many /// ms (negative: before t); only on the timestamp column. `m[5m]` is - /// `[EvaluationTimePlusMs(-300_000), EvaluationTimePlusMs(0))`. + /// `(EvaluationTimePlusMs(-300_000), EvaluationTimePlusMs(0)]`. EvaluationTimePlusMs(i64), } @@ -1149,8 +1150,8 @@ mod tests { fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { Constraint::Interval { - lower: Bound::Included(EvaluationTimePlusMs(lower_ms)), - upper: Bound::Excluded(EvaluationTimePlusMs(upper_ms)), + lower: Bound::Excluded(EvaluationTimePlusMs(lower_ms)), + upper: Bound::Included(EvaluationTimePlusMs(upper_ms)), } } @@ -1183,8 +1184,8 @@ mod tests { let offset_5m = relative(-600_000, -300_000); assert!(last_5m.disjoint(&offset_5m)); let at = Constraint::Interval { - lower: Bound::Included(FixedValue(ScalarValue::Timestamp(1_699_999_700_000))), - upper: Bound::Excluded(FixedValue(ScalarValue::Timestamp(1_700_000_000_000))), + lower: Bound::Excluded(FixedValue(ScalarValue::Timestamp(1_699_999_700_000))), + upper: Bound::Included(FixedValue(ScalarValue::Timestamp(1_700_000_000_000))), }; assert!(!last_5m.disjoint(&at)); assert_eq!(last_5m.intersect(&at), None); @@ -1205,8 +1206,8 @@ mod tests { assert_eq!( json, serde_json::json!({"interval": { - "lower": {"included": {"evaluation_time_plus_ms": -3_900_000}}, - "upper": {"excluded": {"evaluation_time_plus_ms": -3_600_000}}, + "lower": {"excluded": {"evaluation_time_plus_ms": -3_900_000}}, + "upper": {"included": {"evaluation_time_plus_ms": -3_600_000}}, }}) ); assert_eq!( diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 76762609a..fd2614201 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -259,8 +259,6 @@ impl LogicalProperties { /// [`LogicalProperties::new`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum LogicalPropertiesError { - #[error("the parameters belong to another family than the operator's")] - ParametersOfAnotherFamily, #[error("an exact operator has no summary parameters")] ParametersOnExactOperator, #[error("a sketch build needs parameters from its family's list")] diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index b29642abe..5011ca6be 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -1390,11 +1390,13 @@ pub enum IntervalConstraintLimit { } ``` +A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-open, right-closed. + | Query fragment | `lower` | `upper` | |---|---|---| -| PromQL `m[5m]` | `Included(EvaluationTimePlusMs(-300_000))` | `Excluded(EvaluationTimePlusMs(0))` | -| PromQL `m[5m] offset 1h` | `Included(EvaluationTimePlusMs(-3_900_000))` | `Excluded(EvaluationTimePlusMs(-3_600_000))` | -| PromQL `m[5m] @ 1700000000` | `Included(FixedValue(Timestamp(1_699_999_700_000)))` | `Excluded(FixedValue(Timestamp(1_700_000_000_000)))` | +| PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | | SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(..)))` | `Unbounded` | | SQL `WHERE price > 10` | `Excluded(FixedValue(Float64(10.0)))` | `Unbounded` | From 0d992a1b732441865c666bb3c89bf77a7c53557f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:54:54 +0000 Subject: [PATCH 17/30] refactor(types): 2026-11-08 interfaces: summary types, required accuracy, the plan is one PhysicalASAPDAG (#1207 PR 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Terminology: ops' `family` is `summary_type`; FieldDataType::Sketch(SketchAlgorithm) (SketchKind, GroupingStrategy, HydraKind and the ops' `grouping` deleted); FamilyCapability is SummaryTypeCapability; parameter_list(algorithm). - θ is structure only: SketchParams derives equality; KLL failure_probability, UnivMon distinct_failure_probability and HLL/KMV relative_error deleted. - ValueSign and GroupProperties::column_sign; SourceFacts::column_signs replaces values_non_negative. - LogicalRequirements { required_accuracy: Option }; RootPruneReason::BelowRequiredAccuracy; Memo::add_query takes the target and n_q; AggIntent measures carry no accuracy and front ends take none. - asap-accuracy: AccuracyEvidence::measured_guarantee, theoretical_guarantee (was local_guarantee), producer_satisfies_logical_requirements(memo, producer, consumer, input_index, evidence); the AccuracyModel trait and evidence provider, Pass 1's accuracy pruning and AccuracyCheckInputs deleted. - Physical: ExecutionContext::Evaluation { grid: EvaluationGrid }, IngestionPipeline(RefreshWindow); RefreshWindowId deleted; producer_satisfies_physical_requirements(producer, consumer_timing); PruneReason::CountMinWeightsNotProvenNonNegative. - Plan: PlanOutput { plan: PhysicalASAPDAG }; PhysicalASAPDAGNode:: selected_physical_candidate; select_plan returns (PhysicalASAPDAG, Selection); Selection { accepted_accuracy_per_query, pruned_candidates, milp_* }; RealizedPlan, PhysicalWorkloadPlan, DagOfSelectedCandidates, SelectedEdge, SelectedNodeId, AssignmentRealizer, ScanWindow and the scan/view window functions, SearchReport, SearchStatus, InexactObjective, ObjectiveDiffersFromPlanCost, SelectedPlanFailedCheck and PlanCost::cost_unit deleted. - Plan file wire version 14 (summary_type, Sketch(algorithm), no grouping). - Cost names: NodeCostSource, cost_source, PlanRanking::Cost; Serialize on PruneReason, PhysicalProperties, OriginOfCandidateForMemoGroup and RootPruneReason. --- crates/accuracy/src/estimators/cardinality.rs | 62 +- crates/accuracy/src/estimators/cms.rs | 5 +- crates/accuracy/src/estimators/hll.rs | 8 +- crates/accuracy/src/estimators/kll.rs | 141 +- crates/accuracy/src/estimators/mod.rs | 211 +-- crates/accuracy/src/estimators/univmon.rs | 96 +- crates/accuracy/src/evidence.rs | 12 - crates/accuracy/src/lib.rs | 34 +- crates/accuracy/src/model.rs | 209 +-- crates/accuracy/src/propagation.rs | 53 +- crates/accuracy/src/sizing.rs | 79 +- crates/accuracy/src/test_support.rs | 17 - crates/cost/src/lib.rs | 20 +- crates/cost/src/measured.rs | 10 +- crates/cost/src/ranking.rs | 6 +- crates/cost/src/state_bytes.rs | 33 +- crates/cost/src/usage.rs | 176 +- .../devtools/examples/canonical_examples.rs | 12 +- crates/devtools/examples/topk_ir.rs | 5 +- crates/devtools/src/bin/analyze_corpora.rs | 12 +- crates/devtools/src/bin/asap_planner.rs | 200 +-- crates/devtools/src/bin/sketch_coverage.rs | 62 +- crates/devtools/src/bin/variant_coverage.rs | 21 +- crates/devtools/src/candidate_labels.rs | 280 +--- crates/devtools/src/lib.rs | 6 +- crates/devtools/src/planner_fuzz/README.md | 4 +- crates/devtools/src/planner_fuzz/generate.rs | 2 +- crates/devtools/src/planner_fuzz/mod.rs | 13 +- crates/devtools/src/planner_fuzz/relations.rs | 17 +- crates/devtools/src/planner_fuzz/report.rs | 2 +- crates/devtools/src/planning_stages.rs | 118 +- crates/devtools/src/rqe_atomic_costs.rs | 16 +- crates/devtools/tests/asap_planner.rs | 5 +- crates/devtools/tests/cross_language.rs | 8 +- .../workloads/quantile-grouped.json | 2 +- .../workloads/quantile-single.json | 2 +- .../random-mixed-seed-862-long-w01-q3.json | 2 +- .../random-mixed-seed-862-w01-q3-q8.json | 2 +- .../random-mixed-seed-862-w02-q2-q7.json | 2 +- .../workloads/ratio-of-quantiles-1h.json | 2 +- .../workloads/topk-rate.json | 2 +- crates/devtools/tests/viewer_contract.rs | 12 +- crates/executor/src/capability.rs | 81 +- .../executor/src/operators/aggregate/mod.rs | 4 +- .../src/operators/aggregate/temporal.rs | 11 +- .../executor/src/operators/series_window.rs | 2 +- crates/executor/src/operators/summary/mod.rs | 9 +- .../executor/src/physical_planner/compiled.rs | 4 +- crates/executor/src/physical_planner/mod.rs | 51 +- crates/executor/src/sources/mod.rs | 3 +- crates/executor/src/sources/window.rs | 195 +++ .../executor/src/summary_kernels/factory.rs | 18 +- .../executor/src/summary_kernels/hydra_cms.rs | 2 +- .../summary_kernels/state_column_updater.rs | 30 +- .../src/summary_kernels/weighted_frequency.rs | 5 +- crates/executor/src/supported_summaries.rs | 25 +- crates/executor/src/values/batch.rs | 4 +- crates/executor/src/values/mod.rs | 13 +- crates/executor/tests/blocking_resources.rs | 2 +- crates/executor/tests/common/mod.rs | 17 +- crates/executor/tests/current_series_heap.rs | 2 +- crates/executor/tests/current_timestamp.rs | 1 + crates/executor/tests/deployment.rs | 26 +- crates/executor/tests/hydra_cms_execution.rs | 14 +- .../tests/keyed_summary_row_weight.rs | 14 +- crates/executor/tests/physical_dag.rs | 25 +- crates/executor/tests/physical_semantics.rs | 49 +- crates/executor/tests/plan_properties.rs | 10 +- .../executor/tests/precompute_population.rs | 21 +- crates/executor/tests/promql_binary.rs | 5 +- crates/executor/tests/promql_values.rs | 8 +- crates/executor/tests/raw_scan.rs | 6 +- crates/executor/tests/state_column_updater.rs | 13 +- .../stored_tumbling_window_memory_limit.rs | 8 +- .../tests/summary_build_memory_limit.rs | 2 +- crates/executor/tests/summary_projection.rs | 8 +- crates/executor/tests/univmon_execution.rs | 10 +- .../executor/tests/weighted_topk_binding.rs | 10 +- crates/frontend-common/src/canonicalize.rs | 16 +- crates/frontend-common/src/resolve.rs | 38 +- crates/frontend-metricsql/src/lib.rs | 33 +- .../tests/external_corpus.rs | 4 +- crates/frontend-metricsql/tests/lowering.rs | 15 +- .../tests/lowering_semantics.rs | 20 +- crates/frontend-promql/src/lib.rs | 1 - crates/frontend-promql/src/promql.rs | 48 +- .../frontend-promql/tests/count_planning.rs | 20 +- .../frontend-promql/tests/planner_support.rs | 3 +- .../tests/promql_binding_regressions.rs | 2 +- .../tests/promql_conformance.rs | 35 +- .../frontend-promql/tests/promql_lowering.rs | 56 +- crates/frontend-promql/tests/support.rs | 7 +- .../tests/univmon_candidates.rs | 13 +- crates/frontend-sql/src/frequency.rs | 38 +- crates/frontend-sql/src/lib.rs | 14 +- crates/frontend-sql/src/sql/mod.rs | 43 +- crates/frontend-sql/src/test_support.rs | 9 +- .../tests/bgp_analytics/bgp_analytics.rs | 11 +- .../bgp_jan2024_workload.rs | 9 +- .../synthetic_packet_trace.rs | 9 +- .../tests/data_quality_check/tpch_deequ.rs | 3 +- .../frontend-sql/tests/frequency_entropy.rs | 26 +- crates/frontend-sql/tests/frequency_l2.rs | 49 +- crates/frontend-sql/tests/netflow/netflow.rs | 5 +- crates/frontend-sql/tests/pearson_corr.rs | 12 +- crates/frontend-sql/tests/sql_lowering.rs | 246 +-- crates/frontend-sql/tests/temporal_types.rs | 29 +- crates/integration-tests/src/lib.rs | 104 +- crates/integration-tests/tests/aggregate.rs | 14 +- .../tests/filtered_aggregates.rs | 38 +- .../tests/finer_aggregate_reuse.rs | 5 +- .../tests/frontend_timestamps.rs | 10 +- .../tests/identical_exact_queries.rs | 5 +- .../tests/independent_plan_seed.rs | 5 +- .../tests/joint_search_equivalence.rs | 59 +- .../tests/kll_tumbling_window_execution.rs | 8 +- .../plan_wire_version_10.json | 2 +- .../tests/operator_design_examples.rs | 17 +- .../tests/operator_sharing.rs | 2 +- .../tests/per_computation_candidates_sql.rs | 31 +- .../tests/planner_layering_common/mod.rs | 9 +- .../tests/production_rollup_workloads.rs | 50 +- .../tests/promql_numeric_regressions.rs | 24 +- .../tests/promql_to_post_asap.rs | 19 +- .../tests/rollup_controlled.rs | 20 +- .../tests/sharing_issue_fixtures/922.json | 2 +- .../tests/sharing_issue_fixtures/923.json | 4 +- .../tests/sharing_issue_fixtures/924.json | 4 +- .../tests/sharing_issue_fixtures/925.json | 4 +- .../tests/sharing_issue_regressions.rs | 5 +- .../integration-tests/tests/sketch_rollup.rs | 41 +- .../tests/sql_cardinality.rs | 2 - .../integration-tests/tests/sql_frequency.rs | 5 +- .../tests/sql_to_post_asap.rs | 85 +- .../tests/stage1_logical_asap_optimization.rs | 56 +- crates/integration-tests/tests/time_range.rs | 14 +- crates/logical-asap-optimization/src/lib.rs | 9 +- crates/logical-asap-optimization/src/pass1.rs | 2 + .../src/pass1/composition.rs | 1054 ++++++++++++ .../src/pass1/logical_memo_candidates.rs | 23 +- .../src/pass1/per_computation_candidates.rs | 1457 +---------------- .../pass1/per_computation_candidates_tests.rs | 325 ++-- .../src/pass1/realization.rs | 160 +- .../src/pass2/logical_memo_candidates.rs | 17 +- .../logical-asap-optimization/src/stage0.rs | 17 +- .../src/test_support.rs | 14 +- .../src/catalog/mod.rs | 10 +- .../src/catalog/realize.rs | 34 - crates/physical-asap-optimization/src/lib.rs | 6 +- crates/plan-selection/src/lib.rs | 10 +- crates/plan-selection/src/milp/model.rs | 37 +- crates/plan-selection/src/models.rs | 17 - crates/plan-selection/src/report.rs | 196 +-- crates/plan-selection/src/select.rs | 31 +- crates/planner/Cargo.toml | 3 +- crates/planner/src/installable.rs | 79 +- crates/planner/src/lib.rs | 33 +- crates/planner/src/pass.rs | 403 +---- crates/planner/src/run.rs | 140 +- crates/planner/tests/e2e_plan.rs | 73 +- crates/planner/tests/runtime_install.rs | 13 +- crates/planner/tests/summary_sharing.rs | 64 +- crates/types/src/ir/materialized_view.rs | 193 +-- crates/types/src/ir/operator/agg_intent.rs | 107 +- crates/types/src/ir/operator/asap.rs | 101 +- crates/types/src/ir/physical_export.rs | 323 +--- crates/types/src/ir/properties/accuracy.rs | 51 +- crates/types/src/ir/properties/guarantee.rs | 4 +- .../src/ir/properties/summary_coverage.rs | 46 +- crates/types/src/ir/schema/mod.rs | 37 +- crates/types/src/ir/schema/sketch.rs | 419 +---- ...pability.rs => summary_type_capability.rs} | 240 +-- crates/types/src/physical/mod.rs | 11 +- crates/types/src/physical/properties.rs | 2 +- crates/types/src/physical/workload_plan.rs | 32 - crates/types/src/stages/candidates.rs | 114 +- .../src/stages/logical_memo_candidates.rs | 15 +- crates/types/src/stages/memo.rs | 16 +- crates/types/src/stages/mod.rs | 19 +- .../src/stages/physical/execution_context.rs | 52 +- crates/types/src/stages/physical/graph.rs | 189 --- .../src/stages/physical/memo_candidates.rs | 11 +- crates/types/src/stages/physical/mod.rs | 6 +- crates/types/src/stages/pruning.rs | 15 +- crates/types/src/stages/realizer.rs | 81 - crates/types/src/workload/bounds.rs | 87 +- crates/types/src/workload/mod.rs | 27 +- crates/types/tests/physical_export.rs | 145 +- crates/types/tests/schema_rebuilding.rs | 5 +- crates/types/tests/structure_contract.rs | 14 +- crates/types/tests/summary_merge_structure.rs | 318 +--- crates/types/tests/tumbling_window_chain.rs | 10 +- .../evidence-dependent-candidates.md | 6 +- .../architecture/input-output-workflow.md | 40 +- ...d_interface_with_pluggable_optimization.md | 8 +- docs/design_docs/concepts/accuracy-models.md | 15 +- .../proposals/asap-primitive-schema.md | 94 +- .../joint-planning-implementation-plan.md | 6 +- .../design_docs/proposals/planner-layering.md | 12 +- .../asap-aware-mapping-architecture.md | 18 +- .../asap-aware-mapping-contracts.md | 15 +- .../end-to-end-accuracy-guarantees.md | 38 +- .../develop_docs/extend-asap-aware-mapping.md | 19 +- docs/develop_docs/library-api.md | 23 +- .../promql/rangeheavy-t5-q05-1h-60s-300s.json | 2 +- .../promql/rangeheavy-t5-q05-q09-1h.json | 2 +- .../promql/rangeheavy-t5-q09-15m-1h.json | 2 +- .../promql/rangeheavy-t7-t4-1h.json | 2 +- .../promql/rangeheavy-t8-t5-1h.json | 2 +- .../promql/ratio-of-quantiles-5m.json | 2 +- .../rollupheavy-t11-service-endpoint-5m.json | 2 +- tools/dag-viewer/stages.js | 3 +- .../workload-01.json | 2 +- .../workload-02.json | 2 +- .../workload-03.json | 2 +- .../workload-04.json | 2 +- .../workload-05.json | 2 +- .../workload-06.json | 2 +- .../workload-07.json | 2 +- .../workload-08.json | 2 +- .../workload-09.json | 2 +- .../workload-10.json | 2 +- .../random-mixed-seed-862-x5/workload-01.json | 2 +- .../random-mixed-seed-862-x5/workload-02.json | 2 +- .../random-mixed-seed-862-x5/workload-03.json | 2 +- .../random-mixed-seed-862-x5/workload-04.json | 2 +- .../random-mixed-seed-862-x5/workload-05.json | 2 +- .../random-mixed-seed-862-x5/workload-06.json | 2 +- .../random-mixed-seed-862-x5/workload-07.json | 2 +- .../random-mixed-seed-862-x5/workload-08.json | 2 +- .../random-mixed-seed-862-x5/workload-09.json | 2 +- .../random-mixed-seed-862-x5/workload-10.json | 2 +- .../random-mixed-seed-862/workload-01.json | 2 +- .../random-mixed-seed-862/workload-02.json | 2 +- .../random-mixed-seed-862/workload-03.json | 2 +- .../random-mixed-seed-862/workload-04.json | 2 +- .../random-mixed-seed-862/workload-05.json | 2 +- .../random-mixed-seed-862/workload-06.json | 2 +- .../random-mixed-seed-862/workload-07.json | 2 +- .../random-mixed-seed-862/workload-08.json | 2 +- .../random-mixed-seed-862/workload-09.json | 2 +- .../random-mixed-seed-862/workload-10.json | 2 +- 242 files changed, 3534 insertions(+), 7901 deletions(-) create mode 100644 crates/executor/src/sources/window.rs create mode 100644 crates/logical-asap-optimization/src/pass1/composition.rs delete mode 100644 crates/physical-asap-optimization/src/catalog/realize.rs delete mode 100644 crates/plan-selection/src/models.rs rename crates/types/src/ir/schema/{family_capability.rs => summary_type_capability.rs} (56%) delete mode 100644 crates/types/src/physical/workload_plan.rs delete mode 100644 crates/types/src/stages/physical/graph.rs delete mode 100644 crates/types/src/stages/realizer.rs diff --git a/crates/accuracy/src/estimators/cardinality.rs b/crates/accuracy/src/estimators/cardinality.rs index 8d6217911..a437da6de 100644 --- a/crates/accuracy/src/estimators/cardinality.rs +++ b/crates/accuracy/src/estimators/cardinality.rs @@ -41,10 +41,9 @@ pub(super) fn guarantee( query: &SketchStatistic, ) -> Option { match params { - SketchParams::Kmv { k, relative_error } => { - // Not sized for a target: its relative standard error. - let eps = relative_error - .unwrap_or_else(|| 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt()); + SketchParams::Kmv { k } => { + // Its relative standard error. + let eps = 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(); Some(super::bounded_guarantee( algorithm, params, @@ -113,61 +112,6 @@ pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { #[cfg(test)] mod tests { use super::*; - use asap_types::ir::operator::agg_intent::default_cardinality; - - fn error_and_delta(params: &SketchParams) -> (f64, f64) { - let g = guarantee(&SketchAlgorithm::Kmv, params, &SketchStatistic::Cardinality).unwrap(); - ( - g.spec.error.evaluate().unwrap(), - g.spec.failure_probability.evaluate().unwrap(), - ) - } - - /// KMV sized for (ε, δ) certifies both (#700): its failure probability - /// is the requested δ's, not a fixed 0.01. - #[test] - fn kmv_sized_for_a_failure_probability_meets_it() { - for (eps, delta) in [(0.01, 0.001), (0.05, 0.01), (0.02, 1e-6)] { - let params = - super::super::size_params(SketchAlgorithm::Kmv, &default_cardinality(), eps, delta); - let (error, failure_probability) = error_and_delta(¶ms); - assert!(error <= eps, "{params:?}: {error}"); - assert!( - failure_probability <= delta, - "{params:?}: {failure_probability}" - ); - } - } - - /// KMV's sizing grows `k` for a smaller δ at the same ε, near the - /// module's closed form, and a KMV sized for one δ is enlarged for a - /// smaller one at the same relative error. - #[test] - fn kmv_k_follows_the_failure_probability() { - // Ten times smaller than the former Chebyshev calibration's 40,002. - assert_eq!(kmv_k(0.05, 0.01), 4323); - assert_eq!(kmv_k(0.02, 0.01), 26656); - assert!(kmv_k(0.05, 1e-6) > kmv_k(0.05, 0.01)); - assert!(kmv_failure_probability(kmv_k(0.05, 0.01) - 1, 0.05) > 0.01); - assert_eq!(kmv_k(0.0, 0.01), KMV_MAX_K); - let sized = - super::super::size_params(SketchAlgorithm::Kmv, &default_cardinality(), 0.05, 0.01); - let enlarged = super::super::with_failure_probability(&sized, 1e-4); - let (error, delta) = error_and_delta(&enlarged); - assert_eq!(error, 0.05); - assert!(delta <= 1e-4); - assert_ne!(enlarged, sized); - // Not sized for a relative error: nothing to enlarge for. - let unsized_kmv = SketchParams::Kmv { - k: 1024, - relative_error: None, - }; - assert_eq!( - super::super::with_failure_probability(&unsized_kmv, 1e-4), - unsized_kmv - ); - assert_eq!(error_and_delta(&unsized_kmv).1, 1.0); - } /// The bound dominates the exact failure probability of the order /// statistic for every number of distinct items checked: exact below diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/estimators/cms.rs index ebce49b97..629f3d583 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/estimators/cms.rs @@ -86,10 +86,7 @@ mod tests { #[test] fn topk_heap_capacity_respects_accuracy_and_output_count() { use crate::estimators::size_params; - let intent = AggIntent::TopK { - k: 25, - accuracy: AccuracyTarget::max_error(0.01), - }; + let intent = AggIntent::TopK { k: 25 }; assert_eq!( size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.01, 0.01), SketchParams::CmsWithHeap { diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/estimators/hll.rs index 65ff8512f..25ba433cd 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/estimators/hll.rs @@ -465,14 +465,10 @@ mod tests { use asap_types::ir::operator::agg_intent::default_cardinality; for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { let params = size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); - let SketchParams::Hll { - precision: sized, - relative_error, - } = params - else { + let SketchParams::Hll { precision: sized } = params else { panic!("{params:?}") }; - assert_eq!((sized, relative_error), (precision, Some(eps))); + assert_eq!(sized, precision); } } diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index 3225994e4..ccbf54de0 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -9,17 +9,13 @@ pub(super) fn guarantee( params: &SketchParams, query: &SketchStatistic, ) -> Option { - let SketchParams::Kll { - k, - failure_probability, - } = params - else { + let SketchParams::Kll { k } = params else { return None; }; if *k > KLL_MAX_K { return None; } - let delta = failure_probability.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY); + let delta = KLL_CALIBRATED_FAILURE_PROBABILITY; let scale = delta_scale(delta)?; let contract = if delta < KLL_CALIBRATED_FAILURE_PROBABILITY { KLL_DELTA_SCALED_CONTRACT @@ -139,44 +135,10 @@ fn normal_quantile(p: f64) -> f64 { /// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. pub(crate) fn size_params(eps: f64, delta: f64) -> SketchParams { if delta.is_nan() || delta <= 0.0 || delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { - return SketchParams::Kll { - k: kll_k(eps), - failure_probability: None, - }; + return SketchParams::Kll { k: kll_k(eps) }; } SketchParams::Kll { k: kll_k(eps / delta_scale(delta).unwrap_or(1.0)), - failure_probability: Some(delta), - } -} - -/// `params` resized to fail with probability at most `delta` at no larger -/// rank error: `k` for its current error at `delta`'s scale, never smaller. -/// Unchanged when it already fails no more often. -pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { - let SketchParams::Kll { - k, - failure_probability, - } = params - else { - return params.clone(); - }; - let current = failure_probability.unwrap_or(KLL_CALIBRATED_FAILURE_PROBABILITY); - let Some(scale) = delta_scale(current) else { - return params.clone(); - }; - if delta >= current { - return params.clone(); - } - match size_params(kll_rank_error_99(*k) * scale, delta) { - SketchParams::Kll { - k: resized, - failure_probability, - } => SketchParams::Kll { - k: resized.max(*k), - failure_probability, - }, - other => other, } } @@ -217,61 +179,6 @@ mod tests { ))); } - /// Below δ = 0.01, KLL is sized by the KLL bound's δ dependence: k is - /// larger than at δ = 0.01, and the guarantee meets ε = 0.01, δ = 0.001. - #[test] - fn sizing_meets_a_failure_probability_below_one_percent() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - let q = default_quantile(0.99); - let k = |delta| match size_params(SketchAlgorithm::Kll, &q, 0.01, delta) { - SketchParams::Kll { k, .. } => k, - other => panic!("{other:?}"), - }; - assert!(k(0.001) > k(0.01), "{} vs {}", k(0.001), k(0.01)); - let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.001); - let g = guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); - assert_eq!(g.spec.failure_probability.evaluate(), Some(0.001)); - } - - /// A KLL sized below δ = 0.01 names its own contract, not the empirical - /// 99% calibration's. - #[test] - fn sizing_below_one_percent_names_a_distinct_contract() { - let contract = |delta| { - let g = guarantee( - &SketchAlgorithm::Kll, - &size_params(0.01, delta), - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - match &g.provenance[..] { - [GuaranteeSource::SketchEvaluation { contract, .. }] => contract.clone(), - other => panic!("{other:?}"), - } - }; - assert_eq!( - contract(0.01), - "apache_datasketches_kll_empirical_99_a9b42755072b" - ); - assert_eq!( - contract(0.001), - "apache_datasketches_kll_empirical_99_delta_scaled_a9b42755072b" - ); - } - /// `k` never exceeds the 26 602 `asap_sketchlib` silently caps it at, so /// a guarantee never claims a larger `k` than runs. #[test] @@ -286,10 +193,7 @@ mod tests { let g = |k| { guarantee( &SketchAlgorithm::Kll, - &SketchParams::Kll { - k, - failure_probability: None, - }, + &SketchParams::Kll { k }, &SketchStatistic::Quantile { q: 0.99 }, ) }; @@ -319,32 +223,10 @@ mod tests { size_params(0.01, 0.001), SketchParams::Kll { k: kll_k(0.01 / ratio), - failure_probability: Some(0.001), } ); } - /// A hand-built failure probability outside (0, 1) gives no guarantee - /// rather than an infinite or NaN error bound. - #[test] - fn an_invalid_failure_probability_gives_no_guarantee() { - for delta in [0.0, -0.1, 1.0, 1.5, f64::NAN, f64::INFINITY] { - let params = SketchParams::Kll { - k: 200, - failure_probability: Some(delta), - }; - assert!( - guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .is_none(), - "{delta}" - ); - } - } - /// KLL `k` inverts the rank-error fit: ε = 0.01 → 269, ε = 0.05 → 52. #[test] fn accuracy_target_drives_kll_k() { @@ -353,17 +235,11 @@ mod tests { let q = default_quantile(0.99); assert_eq!( size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01), - SketchParams::Kll { - k: 269, - failure_probability: None - } + SketchParams::Kll { k: 269 } ); assert_eq!( size_params(SketchAlgorithm::Kll, &q, 0.05, 0.01), - SketchParams::Kll { - k: 52, - failure_probability: None - } + SketchParams::Kll { k: 52 } ); } @@ -374,10 +250,7 @@ mod tests { use asap_types::ir::operator::agg_intent::default_quantile; assert_eq!( size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.0, 0.01), - SketchParams::Kll { - k: 26_602, - failure_probability: None - } + SketchParams::Kll { k: 26_602 } ); } } diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 69a14f3a1..21649811d 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -1,7 +1,7 @@ //! Dispatch committed estimator parameters to their accuracy models. use super::*; use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::FamilyCapability; +use asap_types::ir::schema::SummaryTypeCapability; pub(crate) mod cardinality; pub(crate) mod cms; @@ -57,88 +57,23 @@ fn bounded_guarantee( ) } -/// The built-in guarantee of reading `query` out of a summary of family -/// `family` built over an **exact** input, derived from the family's -/// committed parameters by inverting the same sizing formulas -/// [`size_params`] uses. Exact for a plain value -/// or an exact aggregate. `None` when a sketch does not answer `query` (its -/// family-capability table row), and when the family has no error model -/// (`SummarySample`, `Wavelet`, `StatModel`, and sketch evaluations without a -/// certified bound): Pass 1 then leaves the guarantee `None` and keeps the -/// candidate for plan selection to decide. -pub(crate) fn local_guarantee( - family: &FieldDataType, - query: &SketchStatistic, +/// The theoretical guarantee of reading `statistic` from a state of +/// `summary_type` with parameters `parameters`: the algorithm's error +/// formula (concepts doc, section 2, "Where the guarantee comes from"). +/// `None` when the summary type does not answer `statistic` (its +/// capability row) or has no error model (`SummarySample`, `Wavelet`, +/// `StatModel`, and sketch readouts without a certified bound). +pub fn theoretical_guarantee( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, ) -> Option { - match family { - FieldDataType::Plain(_) => Some(AccuracyGuarantee::exact("Plain value")), - FieldDataType::ExactAggregate(kind, _) => Some(AccuracyGuarantee::exact(format!( - "ExactAggregate({kind:?})" - ))), - FieldDataType::Sketch(..) if FamilyCapability::of(family).answers(query) => { - todo!( - "#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters" - ) + match summary_type { + FieldDataType::Sketch(_) if SummaryTypeCapability::of(summary_type).answers(statistic) => { + let _ = parameters; + todo!("#1207 PR 4: the algorithm's error formula at θ, at the required failure probability") } - FieldDataType::Sketch(..) => None, - // No error model is registered for these families. - FieldDataType::SummarySample(..) - | FieldDataType::Wavelet(..) - | FieldDataType::StatModel(..) => None, - } -} -/// `params` with the parameters its per-value failure probability depends -/// on enlarged, never reduced, so that each value it answers fails with -/// probability at most `delta` and its error bound is unchanged or smaller -/// (#669b): Count-Min's depth, Count Sketch's depth (the binomial tail of -/// its median), KLL's failure probability (its δ scale), a distinct count -/// UnivMon's heap, and KMV's `k` at the relative error it was sized for -/// (#694). Other parameters are returned as they are: DDSketch never fails; -/// classic HLL's failure probability comes from its source's distinct bound -/// ([`hll::min_precision`]); a KMV not -/// sized for a relative error and an L2 or entropy UnivMon are not -/// enlarged. -pub fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { - if !(delta > 0.0 && delta < 1.0) { - return params.clone(); - } - match params { - SketchParams::Cms { width, depth } => SketchParams::Cms { - width: *width, - depth: (*depth).max(cms::cms_depth(delta)), - }, - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => SketchParams::CmsWithHeap { - width: *width, - depth: (*depth).max(cms::cms_depth(delta)), - heap_size: *heap_size, - }, - SketchParams::CountSketch { width, depth } => SketchParams::CountSketch { - width: *width, - depth: (*depth).max(count_sketch::count_sketch_depth(*width, delta)), - }, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => SketchParams::CountSketchWithHeap { - width: *width, - depth: (*depth).max(count_sketch::count_sketch_depth(*width, delta)), - heap_size: *heap_size, - }, - SketchParams::Kll { .. } => kll::with_failure_probability(params, delta), - SketchParams::Kmv { - k, - relative_error: Some(eps), - } => SketchParams::Kmv { - k: (*k).max(cardinality::kmv_k(*eps, delta)), - relative_error: Some(*eps), - }, - SketchParams::UnivMon { .. } => univmon::with_failure_probability(params, delta), - _ => params.clone(), + _ => None, } } @@ -158,7 +93,6 @@ pub(crate) fn size_params( }, SketchAlgorithm::Hll => SketchParams::Hll { precision: hll::hll_precision(eps, delta), - relative_error: Some(eps), }, SketchAlgorithm::CmsWithHeap => { let k = match intent { @@ -177,7 +111,6 @@ pub(crate) fn size_params( SketchAlgorithm::DDSketch => ddsketch::size_params(eps), SketchAlgorithm::Kmv => SketchParams::Kmv { k: cardinality::kmv_k(eps, delta), - relative_error: Some(eps), }, SketchAlgorithm::CountSketch => { let width = count_sketch::count_sketch_width(eps); @@ -267,97 +200,6 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { #[cfg(test)] mod tests { use super::*; - use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; - - fn delta_and_error(params: &SketchParams, query: &SketchStatistic) -> (f64, f64) { - let algorithm = match params { - SketchParams::Cms { .. } => SketchAlgorithm::Cms, - SketchParams::CountSketch { .. } => SketchAlgorithm::CountSketch, - SketchParams::Kll { .. } => SketchAlgorithm::Kll, - other => panic!("{other:?}"), - }; - let g = sketch_guarantee(&algorithm, params, query).unwrap(); - ( - g.spec.failure_probability.evaluate().unwrap(), - g.spec.error.evaluate().unwrap(), - ) - } - - /// Enlarging for a smaller failure probability deepens Count-Min and - /// Count Sketch and grows KLL until each meets it, at no larger error; - /// it never shrinks a sketch that already meets it. (Count Sketch's - /// depth is capped at 255 rows, which a non-power-of-two width reaches - /// before 1e-6.) - #[test] - fn enlarging_meets_the_failure_probability_at_no_larger_error() { - let point = SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }; - let quantile = SketchStatistic::Quantile { q: 0.99 }; - for (params, query, delta) in [ - ( - size_params(SketchAlgorithm::Cms, &default_cardinality(), 0.01, 0.01), - &point, - 1e-6, - ), - ( - size_params( - SketchAlgorithm::CountSketch, - &default_cardinality(), - 0.01, - 0.01, - ), - &point, - 1e-3, - ), - ( - size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.01, 0.01), - &quantile, - 1e-6, - ), - ] { - let (_, error) = delta_and_error(¶ms, query); - let enlarged = with_failure_probability(¶ms, delta); - let (new_delta, new_error) = delta_and_error(&enlarged, query); - assert!(new_delta <= delta, "{enlarged:?}"); - assert!(new_error <= error * (1.0 + 1e-9), "{enlarged:?}"); - assert_ne!(enlarged, params); - assert_eq!(with_failure_probability(&enlarged, 0.01), enlarged); - } - let ddsketch = SketchParams::DDSketch { alpha: 0.01 }; - assert_eq!(with_failure_probability(&ddsketch, 1e-6), ddsketch); - } - - /// A distinct-count UnivMon grows its heap for a smaller failure - /// probability at no larger relative error. - #[test] - fn a_distinct_count_univmon_grows_its_heap() { - let params = size_params(SketchAlgorithm::UnivMon, &default_cardinality(), 0.05, 0.01); - let enlarged = with_failure_probability(¶ms, 1e-4); - let error = |p: &SketchParams| { - univmon::distinct_guarantee(p, "m") - .unwrap() - .spec - .error - .evaluate() - .unwrap() - }; - let ( - SketchParams::UnivMon { heap_size: k, .. }, - SketchParams::UnivMon { - heap_size: grown, - distinct_failure_probability, - .. - }, - ) = (¶ms, &enlarged) - else { - panic!("{enlarged:?}") - }; - assert!(grown > k); - assert_eq!(*distinct_failure_probability, Some(1e-4)); - assert!(error(&enlarged) <= error(¶ms) * (1.0 + 1e-9)); - } /// Classic HLL's precision grows until its failure probability over the /// source's distinct bound meets the target, up to 18. @@ -369,25 +211,4 @@ mod tests { assert!(hll::classic_failure_probability(p - 1, 0.05, 64) > 0.01); assert_eq!(hll::min_precision(0.05, 1e-30, 4096, 12), 12); } - - /// A sketch read for a statistic its table row does not answer (an HLL - /// for a total) has no guarantee; one it answers keeps its guarantee, - /// and a plain value or exact aggregate stays exact. - #[test] - fn local_guarantee_needs_a_statistic_the_family_answers() { - use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::{DataType, ExactKind, ExactParams, GroupingStrategy}; - let hll = FieldDataType::Sketch(SketchAlgorithm::Hll, GroupingStrategy::default()); - let total = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - assert!(local_guarantee(&hll, &total).is_none()); - for exact in [ - FieldDataType::Plain(DataType::Float64), - FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - ] { - assert!(local_guarantee(&exact, &total).is_some(), "{exact:?}"); - } - } } diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index 41b694d1a..fac4a3951 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -55,7 +55,6 @@ const BASELINE: SketchParams = SketchParams::UnivMon { sketch_rows: 5, sketch_cols: 1024, layers: 16, - distinct_failure_probability: None, }; pub(super) fn guarantee( @@ -252,27 +251,30 @@ pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct } } -/// A UnivMon distinct count over `source`'s own rows: relative error at -/// most `√(1.35/(k δ_c))`, where `δ_c` is the share of the sized failure -/// probability left after the top layer's, failing with probability -/// `δ_c` plus the chance of an incomplete top layer, left symbolic in the -/// source's distinct bound. `None` for a shape that was not sized for a -/// distinct count, a heap under 8, or a shape the library would not build. +/// A UnivMon distinct count over `source`'s own rows, at failure +/// probability `delta`: relative error at most `√(1.35/(k δ_c))`, where +/// `δ_c` is the share of `delta` left after the top layer's, failing with +/// probability `δ_c` plus the chance of an incomplete top layer, left +/// symbolic in the source's distinct bound. `None` for a heap under 8, a +/// `delta` too small for the heap, or a shape the library would not build. #[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. -pub(crate) fn distinct_guarantee(params: &SketchParams, source: &str) -> Option { +pub(crate) fn distinct_guarantee( + params: &SketchParams, + delta: f64, + source: &str, +) -> Option { let SketchParams::UnivMon { heap_size, sketch_rows, sketch_cols, layers, - distinct_failure_probability: Some(delta), } = params else { return None; }; if *heap_size < DISTINCT_MIN_HEAP || !hash_independent(*sketch_rows, *sketch_cols, *layers) - || !(*delta > 0.0 && *delta < 1.0) + || !(delta > 0.0 && delta < 1.0) { return None; } @@ -308,35 +310,6 @@ pub(crate) fn distinct_guarantee(params: &SketchParams, source: &str) -> Option< )) } -/// A distinct-count `params` resized to fail with probability at most -/// `delta` at no larger relative error: the heap for its current error at -/// `delta` ([`distinct_heap`]), never smaller. Unchanged for a shape not -/// sized for a distinct count, or one that already fails no more often. -pub(crate) fn with_failure_probability(params: &SketchParams, delta: f64) -> SketchParams { - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - distinct_failure_probability: Some(current), - } = params - else { - return params.clone(); - }; - let chebyshev = current - incomplete_top_layer_bound(*heap_size); - if delta >= *current || chebyshev <= 0.0 { - return params.clone(); - } - let relative_error = (DISTINCT_VARIANCE / (f64::from(*heap_size) * chebyshev)).sqrt(); - SketchParams::UnivMon { - heap_size: distinct_heap(relative_error, delta).max(*heap_size), - sketch_rows: *sketch_rows, - sketch_cols: *sketch_cols, - layers: *layers, - distinct_failure_probability: Some(delta), - } -} - /// Size a UnivMon for one readout's `(eps, delta)`: /// - L2: the smallest rows and columns meeting the target ([`l2_shape`]), /// one layer (L2 reads only layer 0) and a one-key heap; @@ -356,7 +329,6 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar sketch_rows, sketch_cols, layers: 1, - distinct_failure_probability: None, } } AggIntent::Cardinality { .. } => SketchParams::UnivMon { @@ -364,7 +336,6 @@ pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchPar sketch_rows: 1, sketch_cols: 1, layers: MAX_LAYERS, - distinct_failure_probability: Some(delta), }, _ => BASELINE, } @@ -376,25 +347,15 @@ mod tests { use asap_types::ir::operator::agg_intent::default_cardinality; fn l2_intent() -> AggIntent { - AggIntent::FrequencyL2 { - col: None, - accuracy: AccuracyTarget::exact(), - } + AggIntent::FrequencyL2 { col: None } } - fn univmon( - heap_size: u32, - rows: u32, - cols: u32, - layers: u8, - delta: Option, - ) -> SketchParams { + fn univmon(heap_size: u32, rows: u32, cols: u32, layers: u8) -> SketchParams { SketchParams::UnivMon { heap_size, sketch_rows: rows, sketch_cols: cols, layers, - distinct_failure_probability: delta, } } @@ -492,9 +453,9 @@ mod tests { #[test] fn l2_shapes_stay_within_the_hash_budget() { // 8 · (16 + 1) = 136 > 128. - assert!(l2(&univmon(1, 8, 1 << 16, 1, None)).is_none()); - assert!(l2(&univmon(1, 7, 1 << 16, 1, None)).is_some()); - assert!(l2(&univmon(1, 5, 1024, 20, None)).is_none()); + assert!(l2(&univmon(1, 8, 1 << 16, 1)).is_none()); + assert!(l2(&univmon(1, 7, 1 << 16, 1)).is_some()); + assert!(l2(&univmon(1, 5, 1024, 20)).is_none()); for (eps, delta) in [(0.001, 1e-6), (1e-5, 0.01), (0.01, 1e-12)] { let params = size_params(&l2_intent(), eps, delta); let SketchParams::UnivMon { @@ -521,8 +482,8 @@ mod tests { fn univmon_past_twenty_rows_is_not_hash_independent() { assert!(hash_independent(20, 32, 1)); assert!(!hash_independent(21, 32, 1)); - assert!(l2(&univmon(1, 20, 32, 1, None)).is_some()); - assert!(l2(&univmon(1, 21, 32, 1, None)).is_none()); + assert!(l2(&univmon(1, 20, 32, 1)).is_some()); + assert!(l2(&univmon(1, 21, 32, 1)).is_none()); } /// Each distinct-count heap meets its target once the source's bound @@ -532,7 +493,8 @@ mod tests { for (eps, delta) in TARGETS { let params = size_params(&default_cardinality(), eps, delta); let bind = |params: &SketchParams, max_distinct| { - let guarantee = distinct_guarantee(params, "m").expect("a distinct guarantee"); + let guarantee = + distinct_guarantee(params, delta, "m").expect("a distinct guarantee"); guarantee.bind(&|statistic| { let (k, layers, source) = parse_incomplete_top_layer_statistic(statistic)?; (source == "m") @@ -571,7 +533,7 @@ mod tests { #[test] fn distinct_guarantee_needs_the_source_bound_and_enough_layers() { let params = size_params(&default_cardinality(), 0.05, 0.01); - let guarantee = distinct_guarantee(¶ms, "m").unwrap(); + let guarantee = distinct_guarantee(¶ms, 0.01, "m").unwrap(); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); assert!(!meets(&guarantee, 0.05, 0.01)); let SketchParams::UnivMon { heap_size: k, .. } = params else { @@ -591,15 +553,13 @@ mod tests { assert_eq!(incomplete_top_layer_probability(8, 19, u64::MAX), 1.0); } - /// Only a shape sized for a distinct count, with a heap of at least 8 - /// and a buildable shape, gets a distinct guarantee; the guarantee - /// without a source is none. + /// Only a heap of at least 8 and a buildable shape gets a distinct + /// guarantee; the guarantee without a source is none. #[test] - fn distinct_guarantee_needs_a_distinct_sizing() { - assert!(distinct_guarantee(&univmon(256, 5, 1024, 16, None), "m").is_none()); - assert!(distinct_guarantee(&univmon(7, 1, 1, 19, Some(0.5)), "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 20, Some(0.5)), "m").is_none()); - assert!(distinct_guarantee(&univmon(8, 1, 1, 19, Some(0.5)), "m").is_some()); + fn distinct_guarantee_needs_a_heap_of_eight_and_a_buildable_shape() { + assert!(distinct_guarantee(&univmon(7, 1, 1, 19), 0.5, "m").is_none()); + assert!(distinct_guarantee(&univmon(8, 1, 1, 20), 0.5, "m").is_none()); + assert!(distinct_guarantee(&univmon(8, 1, 1, 19), 0.5, "m").is_some()); let sized = size_params(&default_cardinality(), 0.05, 0.01); assert!(guarantee( &SketchAlgorithm::UnivMon, diff --git a/crates/accuracy/src/evidence.rs b/crates/accuracy/src/evidence.rs index 7a66c62ea..802589c91 100644 --- a/crates/accuracy/src/evidence.rs +++ b/crates/accuracy/src/evidence.rs @@ -1,18 +1,6 @@ //! Scoped source contracts and statistics the accuracy rules consult. use super::*; -/// An enforced domain for the true values of one operand of an exact -/// arithmetic operation, in every evaluation: they are finite and lie in -/// `[lower, upper]`. Plan selection uses it to bound a product or quotient -/// of estimates whose errors are not both relative. Sampled min/max -/// statistics are not a proof: the source or execution layer must enforce -/// it. -#[derive(Debug, Clone, PartialEq)] -pub struct OperandDomain { - pub lower: f64, - pub upper: f64, -} - /// Statistics a propagation rule may consult. Every field is optional and /// defaults to "unknown": a rule that needs a missing statistic emits a /// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 92b623583..817710727 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -1,16 +1,14 @@ -//! `asap-accuracy` — the accuracy model the planning stages share (Pass 1, -//! Pass 2 and Stage 3): the planner's built-in analytical accuracy rules -//! (estimators, composition, guarantee propagation through a DAG, sizing, -//! budget split and target check) and the deployment's accuracy model and -//! evidence traits. +//! `asap-accuracy` — the accuracy rules the planning stages share: the +//! planner's built-in analytical rules (estimators, composition, guarantee +//! propagation through a DAG, sizing, budget split and target check), the +//! deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ +//! of a logical edge ([`producer_satisfies_logical_requirements`]). //! -//! Estimator models derive local guarantees, composition propagates them, -//! `propagation` composes them through every node of a DAG, `sizing` sizes -//! a summary for its target, the budget split sizes each approximate input -//! of a composition for an equal share of its target, and `evidence` holds -//! the scoped contracts and statistics the rules consult. The deployment's accuracy model -//! ([`AccuracyModel`]) and evidence provider ([`AccuracyEvidenceProvider`]) -//! are read only by plan selection (Stage 3). +//! Estimator models derive theoretical guarantees, composition propagates +//! them, `propagation` composes them through every node of a DAG, `sizing` +//! sizes a summary for its target, the budget split sizes each approximate +//! input of a composition for an equal share of its target, and `evidence` +//! holds the scoped contracts and statistics the rules consult. //! See `docs/design_docs/concepts/accuracy-models.md` for the design. //! //! It depends only on `asap-types`, never on a stage crate, so every stage @@ -28,12 +26,9 @@ pub mod sizing; pub mod test_support; pub use budget::per_input_target; -pub use evidence::OperandDomain; +pub use estimators::theoretical_guarantee; pub(crate) use evidence::PropagationStats; -pub use model::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, - NoAccuracyEvidence, -}; +pub use model::{producer_satisfies_logical_requirements, AccuracyEvidence}; #[cfg(test)] use asap_types::ir::properties::AccuracyScope; @@ -56,9 +51,8 @@ const SATISFACTION_TOLERANCE: f64 = 1e-9; /// in that metric (or one with a sound direct comparison): a bound in /// another metric does not satisfy, however small. When it sets none, a /// bound in any metric does, each compared in its own metric. Unknown -/// required dimensions fail. Plan selection's default accuracy model -/// ([`BuiltinAccuracyModel`]) delegates to it. -pub(crate) fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { +/// required dimensions fail. +pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { let within = |value: Option, limit: f64| { value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) }; diff --git a/crates/accuracy/src/model.rs b/crates/accuracy/src/model.rs index 865694196..992688d14 100644 --- a/crates/accuracy/src/model.rs +++ b/crates/accuracy/src/model.rs @@ -1,177 +1,44 @@ -//! The deployment's accuracy model, read only by plan selection (#509 -//! Stage 3). -//! -//! Stage 1 composes each candidate's guarantee with the planner's built-in -//! analytical rules (this crate's [`composition`](crate::composition)) and -//! keeps a candidate it has no rule for with `guarantee = None`. Stage 3 decides, -//! with the model here, whether a guarantee meets its query's target (after -//! the data workload and the deployment's evidence, -//! [`AccuracyEvidenceProvider`], bind the statistics Stage 1 left unknown), -//! and whether the deployment accepts an output directly, whatever its -//! guarantee says. - -use crate::OperandDomain; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric, AccuracyTarget}; -use asap_types::ir::{OperatorNode, QueryRoot}; - -/// The deployment's accuracy decisions in plan selection: whether a -/// guarantee meets a query's target, and whether the deployment accepts an -/// output directly, whatever its guarantee. This crate ships -/// [`BuiltinAccuracyModel`]; a deployment with its own target check (a -/// registered cross-metric conversion, say) or its own measurements -/// implements this trait and passes it in -/// `asap_planner::PlanningModels::accuracy`. -/// Guarantees are composed with the built-in rules only -/// (`propagate`). -pub trait AccuracyModel { - /// Whether `guarantee` meets `target`: a bound within its ε, and within - /// its δ when it states one, over the values the target's scope covers - /// (a per-value δ does not meet a whole-result target). When the target - /// sets a metric, the bound must be in that metric (or one with a sound - /// direct comparison): a bound in another metric does not satisfy, - /// however small. When it sets none, a bound in any metric does, each - /// compared in its own metric. Unknown required dimensions fail; - /// selection separately excludes missing accuracy evidence. - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool; - - /// Whether the deployment accepts `output`, one query's output in a - /// candidate, for `target` directly, whatever its guarantee says: - /// missing (Stage 1 had no analytical rule), met or missed. Stage 3 asks - /// this for every query output; when it returns `true`, the output is - /// accepted, and the guarantee and evidence check only records whether - /// it would have rejected it (`AcceptedAccuracy::guarantee_check_rejection`). - /// It does not override the executability check before it: a plan that - /// reads Count-Min (plain, CmsWithHeap or Hydra Count-Min) over update - /// weights not proven non-negative is rejected - /// (`RejectionCause::CountMinWeightsNotProvenNonNegative`) whatever this - /// returns, because the executor refuses CmsWithHeap and HydraCms when it - /// builds the plan and plain Count-Min adds a negative weight, which voids - /// its never-underestimate bound. The default accepts - /// none, so Stage 3 checks the Stage 1 guarantee, then the evidence - /// ([`AccuracyEvidenceProvider::output_guarantee`]) with - /// [`Self::satisfies`]. - fn accepts_output_regardless_of_guarantee( +//! The deployment's measured accuracy, and α^log of a logical edge. + +use asap_types::ir::properties::AccuracyGuarantee; +use asap_types::ir::schema::{FieldDataType, SketchParams, SketchStatistic}; +use asap_types::stages::{CandidateForMemoGroupId, Memo}; + +/// The deployment's measured accuracy, per summary configuration. A +/// producer's guarantee for a readout is its measurement when there is one, +/// else [`theoretical_guarantee`](crate::estimators::theoretical_guarantee); +/// the source is decided before Pass 2 builds the logical edges (concepts +/// doc, section 2, "Where the guarantee comes from"). +#[derive(Debug, Clone, Default)] +pub struct AccuracyEvidence {} + +impl AccuracyEvidence { + /// The measured guarantee of reading `statistic` from a state of + /// `summary_type` with parameters `parameters`; `None` when the + /// deployment has no measurement for it. No measurement source exists + /// yet, so every configuration has none. + pub fn measured_guarantee( &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - false - } - - /// Whether [`Self::accepts_output_regardless_of_guarantee`] may accept - /// some output. Plan selection discards a query's candidate before - /// composition for a missing or unknown guarantee only when this is - /// `false`: otherwise the model might accept the same plan with larger - /// summaries, which sharing with another query can give it. The default - /// is `true`, which keeps every such candidate. - fn may_accept_regardless_of_guarantee(&self) -> bool { - true - } - - /// Whether a guarantee stated in `metric` can meet `target` with some - /// error bound and failure probability, however small: Pass 1 prunes a - /// summary with which no reading query's output can. The default is - /// [`Self::satisfies`]'s built-in rule: an exact target accepts only an - /// exact output, and a target that sets a metric only a bound in that - /// metric (or an L2-scaled bound for an L1-scaled target). A model that - /// converts between metrics overrides it. - fn metric_can_meet(&self, metric: AccuracyMetric, target: &AccuracyTarget) -> bool { - !target.is_exact() - && target - .accuracy_metric - .is_none_or(|wanted| crate::metric_meets(metric, wanted)) - } -} - -/// The planner's built-in analytical rules as an [`AccuracyModel`]: the -/// conservative target check `satisfies` makes, and no -/// output accepted regardless of its guarantee. -/// The default in `asap_planner::PlanningModels::builtin`. -#[derive(Debug, Default, Clone, Copy)] -pub struct BuiltinAccuracyModel; - -impl AccuracyModel for BuiltinAccuracyModel { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - crate::satisfies(guarantee, target) - } - - fn may_accept_regardless_of_guarantee(&self) -> bool { - false - } -} - -/// The deployment's planning-time accuracy evidence, read only by plan -/// selection: enforced operand domains the propagation rules need, and -/// empirical guarantees for whole query outputs. Passed in -/// `asap_planner::PlanningModels::evidence`. -pub trait AccuracyEvidenceProvider { - /// Enforced domain of `operand`'s true values ([`OperandDomain`]), scoped - /// to this complete expression, including its source, filters, grouping - /// and window: the operands of a product or quotient of estimates whose - /// errors are not both relative. `None` means unknown. - fn operand_domain(&self, _operand: &OperatorNode) -> Option { + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + ) -> Option { + let _ = (summary_type, parameters, statistic); None } - - /// The deployment's empirical accuracy for one candidate's whole query - /// output, as a guarantee. Plan selection (#509 Stage 3) accepts a - /// candidate whose output has no theoretical guarantee that meets the - /// target (e.g. a difference of two quantile estimates) when this shows - /// the output meets it. Stage 1 never reads it: root guarantees there - /// come from the built-in analytical rules. `None` means no evidence. - fn output_guarantee(&self, _output: &QueryRoot) -> Option { - None - } - - /// Whether this provider may return any evidence: an operand domain or - /// an output guarantee. Plan selection discards a query's candidate - /// before composition for a missing or unknown guarantee only when this - /// is `false`: otherwise evidence might exist for the same plan with - /// larger summaries, which sharing with another query can give it. The - /// default is `true`, which keeps every such candidate. - fn may_provide_evidence(&self) -> bool { - true - } -} - -/// No deployment evidence: every operand domain and output guarantee is -/// unknown. The default in `asap_planner::PlanningModels::builtin`. -#[derive(Debug, Default, Clone, Copy)] -pub struct NoAccuracyEvidence; - -impl AccuracyEvidenceProvider for NoAccuracyEvidence { - fn may_provide_evidence(&self) -> bool { - false - } } -/// Accepts every query output regardless of its guarantee, for comparing -/// candidates on cost alone (e.g. a benchmark of sketch plans that have no -/// analytical guarantee); [`Self::satisfies`] is the built-in rule. The -/// facade passes it to plan selection when the caller asks to ignore -/// accuracy guarantees (`PlanningModels::ignore_accuracy_guarantees`). -/// Executability checks still run: Count-Min over update weights not proven -/// non-negative is rejected. Each output accepted only because of it is -/// marked in the report (`AcceptedAccuracy::guarantee_check_rejection`). -#[derive(Debug, Default, Clone, Copy)] -pub struct IgnoreAccuracyGuarantees; - -impl AccuracyModel for IgnoreAccuracyGuarantees { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - BuiltinAccuracyModel.satisfies(guarantee, target) - } - - fn accepts_output_regardless_of_guarantee( - &self, - _output: &QueryRoot, - _target: &AccuracyTarget, - ) -> bool { - true - } - - /// It accepts every output, so no candidate is discarded early for - /// its accuracy. - fn may_accept_regardless_of_guarantee(&self) -> bool { - true - } +/// $\alpha^{\log}$ of one edge (concepts doc, section 2): for a readout's +/// input, the producer's guarantee for the readout's statistic at the +/// required failure probability is within the required error, and a +/// top-k's heap holds k; for any other input, the metric rule (section 4). +pub fn producer_satisfies_logical_requirements( + memo: &Memo, + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input_index_of_consumer: usize, + evidence: &AccuracyEvidence, +) -> bool { + let _ = (memo, producer, consumer, input_index_of_consumer, evidence); + todo!("#1207 PR 4: the readout rule against RequiredAccuracy, else the metric rule") } diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index 11b5d8c02..84274bb21 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -17,9 +17,8 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{DataType, Schema}; use asap_types::ir::schema::{ - ExactKind, FamilyCapability, FieldDataType, GroupingStrategy, MergedError, - NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, - WeightDomain, + ExactKind, FieldDataType, MergedError, NonNegativeWeightProof, SketchAlgorithm, + SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, }; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, @@ -30,7 +29,6 @@ use crate::composition::{ additive_as_absolute, propagate, raw_data_samples_sign_statistic, StreamNorm, GROUP_ROW_COUNT, PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, }; -use crate::estimators::local_guarantee; use crate::function_rules::approximate_input_composition; use crate::PropagationStats; @@ -97,10 +95,10 @@ pub fn approximate_inputs(root: &QueryRoot) -> Vec> { node.asap(), Some( ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } ) @@ -346,13 +344,13 @@ pub fn composed_guarantee(node: &OperatorNode) -> Option { match &node.operator { Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap() { Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), + summary_type: FieldDataType::ExactAggregate(..), .. }) if exact_inputs(child) => Some(AccuracyGuarantee::exact( "exact accumulator over exact input", )), Some(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(kind, _), + summary_type: FieldDataType::ExactAggregate(kind, _), child: input, reduction, .. @@ -377,7 +375,7 @@ pub fn composed_guarantee(node: &OperatorNode) -> Option { query, }) => match summary_input.asap() { Some(ASAPOp::SummaryAgg { - family, + summary_type: family, input: update, child: input, reduction, @@ -388,7 +386,9 @@ pub fn composed_guarantee(node: &OperatorNode) -> Option { let local = match estimate_bound_over_raw_data_samples(family, update, query) { Some(gated) => gated, None => source_distinct_guarantee(family, query, input) - .or_else(|| local_guarantee(family, query))?, + .or_else(|| { + todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters") + })?, }; let local = interpolation_guarantee(family, update, query, input, local)?; if exact_inputs(summary_input) { @@ -501,9 +501,11 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); let family = state.asap()?.produced_state()?; let merged = match node.asap()? { - ASAPOp::SummaryEstimate { query, .. } => FamilyCapability::of(family).merged_error(query), + ASAPOp::SummaryEstimate { query, .. } => { + SummaryTypeCapability::of(family).merged_error(query) + } // An exact accumulator merged across groups is exact. - _ => FamilyCapability::merges_groups(family).then_some(MergedError::Exact), + _ => SummaryTypeCapability::merges_groups(family).then_some(MergedError::Exact), }; if !matches!( merged, @@ -540,13 +542,13 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option { let chain = merged_chain(summary_input)?; let family = chain.asap()?.produced_state()?; - FamilyCapability::of(family).merged_error(query) + SummaryTypeCapability::of(family).merged_error(query) } ASAPOp::FinalizeExactAccumulator { child } => { let chain = merged_chain(child)?; let family = chain.asap()?.produced_state()?; // An exact accumulator's merged state is exact. - let merged = FamilyCapability::of(family).merge_inputs; + let merged = SummaryTypeCapability::of(family).merge_inputs; merged.map(|_| MergedError::Exact) } _ => return None, @@ -575,8 +577,7 @@ pub(crate) fn source_distinct_guarantee( query: &SketchStatistic, input: &Rc, ) -> Option { - let FieldDataType::Sketch(algorithm, GroupingStrategy::PerSubpopulationInstance) = family - else { + let FieldDataType::Sketch(algorithm) = family else { return None; }; if *query != SketchStatistic::Cardinality { @@ -652,7 +653,7 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match child.asap()? { ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), child, filter: None, .. @@ -1108,7 +1109,7 @@ pub(crate) fn stream_norm(node: &Rc) -> Option { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(algorithm, _), + summary_type: FieldDataType::Sketch(algorithm), .. }) if matches!( algorithm, @@ -1487,7 +1488,7 @@ pub(crate) fn estimate_has_bound( ) -> bool { let count_min = matches!( family, - FieldDataType::Sketch(algorithm, _) + FieldDataType::Sketch(algorithm) if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) ); !matches!(query, SketchStatistic::TopK { .. }) @@ -1505,7 +1506,7 @@ fn interpolation_guarantee( input: &OperatorNode, mut guarantee: AccuracyGuarantee, ) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch, _)) + let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch)) && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { return Some(guarantee); @@ -1554,16 +1555,8 @@ pub(crate) fn estimate_bound_over_raw_data_samples( if !estimate_has_bound(family, &proven, query) { return None; } - let mut local = local_guarantee(family, query)?; - local.spec.error = asap_types::ir::properties::BoundExpr::Product { - factors: vec![ - local.spec.error, - asap_types::ir::properties::BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic(metric), - }, - ], - }; - Some(local) + let _ = metric; + todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters, its bound gated on the metric's sign") } pub fn split_and(expr: &ScalarExpr, out: &mut Vec) { diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index cdda5845c..b13ab82e7 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -1,19 +1,10 @@ -//! Sizing: the target an approximate-capable intent's summary is sized -//! against, its `(ε, δ)` budget, the default sketch parameters for it, a -//! family sized for one equal share of that target, and the failure -//! probability each approximate input needs over the values the target -//! covers. +//! Sizing: the statistic a summary is read for, and the default sketch +//! parameters for an `(ε, δ)`. use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; -/// Confidence δ assumed when the target carries only an ε -/// (`AccuracyTarget::epsilon`): the (ε, δ)-parameterised sketches (CMS) need -/// one. `ln(1/0.01) → depth 5`, matching the conventional CMS sizing. -pub(crate) const DEFAULT_DELTA: f64 = 0.01; - /// The statistic a summary is read for to answer `intent`, or `None` when /// no sketch statistic serves the intent. pub fn sketch_statistic(intent: &AggIntent) -> Option { @@ -23,7 +14,7 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { AggIntent::FrequencyL2 { .. } => SketchStatistic::FrequencyL2, AggIntent::FrequencyEntropy { .. } => SketchStatistic::FrequencyEntropy, AggIntent::TopK { k, .. } => SketchStatistic::TopK { k: *k }, - AggIntent::Count { .. } => SketchStatistic::PointCount { + AggIntent::Count => SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, }, @@ -31,57 +22,6 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { }) } -/// The [`AccuracyTarget`] an approximate-capable intent's summary is sized -/// and checked against, or `None` for every other intent (no sketch -/// candidate applies): the query's target restated in the intent's own metric -/// (#690 §4): an L1-scaled absolute error for counts and top-k frequencies, -/// a relative error for distinct counts and L2 frequency moments, an -/// absolute error for entropy, and for a quantile the query's metric or a -/// rank error when none is set. The query's own metric, set or not, is -/// checked at its root in Stage 3. -pub fn sizing_target(intent: &AggIntent) -> Option { - let (target, metric) = match intent { - AggIntent::Quantile { accuracy, .. } => ( - accuracy, - accuracy.accuracy_metric.unwrap_or(AccuracyMetric::Rank), - ), - AggIntent::Cardinality { accuracy, .. } | AggIntent::FrequencyL2 { accuracy, .. } => { - (accuracy, AccuracyMetric::Relative) - } - AggIntent::FrequencyEntropy { accuracy, .. } => (accuracy, AccuracyMetric::ABSOLUTE), - AggIntent::Count { accuracy } | AggIntent::TopK { accuracy, .. } => { - (accuracy, AccuracyMetric::ABSOLUTE_STREAM_L1) - } - _ => return None, - }; - Some(target_in(target, metric)) -} - -/// `target`'s ε, δ and scope stated in `metric`. An exact target stays -/// exact with no metric: exact is exact in every metric. -fn target_in(target: &AccuracyTarget, metric: AccuracyMetric) -> AccuracyTarget { - if target.is_exact() { - return AccuracyTarget::exact().accuracy_scope(target.accuracy_scope); - } - target.clone().accuracy_metric(metric) -} - -/// Resolve an [`AccuracyTarget`] into the `(eps, delta)` budget -/// `default_size_params` needs — one place this resolution happens, so -/// nothing can drift apart on it. -/// -/// An exact target degrades to the tightest parameters for a caller that -/// resolves it anyway. -pub fn accuracy_budget(accuracy: &AccuracyTarget) -> (f64, f64) { - if accuracy.is_exact() { - return (f64::MIN_POSITIVE, DEFAULT_DELTA); - } - ( - accuracy.max_error, - accuracy.failure_probability.unwrap_or(DEFAULT_DELTA), - ) -} - /// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved /// `(eps, delta)` accuracy budget. /// @@ -103,13 +43,10 @@ pub(crate) fn default_size_params( mod tests { use super::*; - /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`, with an ε-only target - /// taking `DEFAULT_DELTA` (depth 5). + /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. #[test] fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.001).failure_probability(0.001), - }; + let intent = AggIntent::Count; assert_eq!( default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), SketchParams::Cms { @@ -117,12 +54,8 @@ mod tests { depth: 7 } // ⌈e/0.001⌉, ⌈ln 1000⌉ ); - let target = AccuracyTarget::max_error(0.001); - let (eps, delta) = accuracy_budget(&target); - assert_eq!((eps, delta), (0.001, DEFAULT_DELTA)); - let intent = AggIntent::Count { accuracy: target }; assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, eps, delta), + default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), SketchParams::Cms { width: 2719, depth: 5 diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs index 9a206a27f..672ee25ae 100644 --- a/crates/accuracy/src/test_support.rs +++ b/crates/accuracy/src/test_support.rs @@ -7,7 +7,6 @@ use std::rc::Rc; use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::guarantee::AccuracyGuarantee; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{ FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, }; @@ -72,19 +71,6 @@ pub fn source_distinct_guarantee( propagation::source_distinct_guarantee(family, query, input) } -/// `crate::satisfies`. -pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - crate::satisfies(guarantee, target) -} - -/// `crate::estimators::local_guarantee`. -pub fn local_guarantee( - family: &FieldDataType, - query: &SketchStatistic, -) -> Option { - crate::estimators::local_guarantee(family, query) -} - /// `crate::estimators::cms::cms_width`. pub fn cms_width(eps: f64) -> u32 { crate::estimators::cms::cms_width(eps) @@ -110,9 +96,6 @@ pub fn default_size_params( crate::sizing::default_size_params(kind, intent, eps, delta) } -/// `crate::sizing::DEFAULT_DELTA`. -pub const DEFAULT_DELTA: f64 = crate::sizing::DEFAULT_DELTA; - /// `crate::composition::QUOTIENT_OPERAND_DOMAINS`. pub const QUOTIENT_OPERAND_DOMAINS: &str = crate::composition::QUOTIENT_OPERAND_DOMAINS; diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 27e7f5ade..77c92aba8 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -35,21 +35,17 @@ mod usage; use analytical::AnalyticalCostError; pub(crate) use measured::aggregate_lookup; pub use measured::{ - lookup, CostTableProvenance, MeasuredOperationCost, MeasuredOperationCosts, NodePriceSource, + lookup, CostTableProvenance, MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, StateFamily, SummaryOperation, }; pub use operator_statistics::EdgeStatistics; pub use ranking::{cheaper, worst_latency_ms, PlanRanking}; pub use usage::{ - family_name, maintained_window_ms, materialized_tumbling_window_chains, node_map, node_usage, - plan_cost, query_reach, retained_tumbling_windows, trailing_window, - tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, tumbling_windows_of, - NodeMap, NodeUsage, ProducerOutput, + maintained_window_ms, node_map, node_usage, plan_cost, query_reach, retained_tumbling_windows, + summary_type_name, tumbling_window_retention_ms, tumbling_window_retention_ms_of_plan, + tumbling_windows_of, NodeMap, NodeUsage, ProducerOutput, }; -/// Cost per second of wall time; one unit is one CPU-millisecond under -/// [`ResourceUnitCost::illustrative`]. -pub const COST_UNIT: &str = "cost_per_second"; /// [`PlanCost::source`] when every node is priced from the formulas. pub const COST_SOURCE: &str = "analytical-cost-v2 (illustrative statistics)"; /// [`PlanCost::source`] when every node is priced from the measured @@ -382,9 +378,9 @@ pub struct NodeCost { /// ... bytes": one computation's work, the times it is computed per /// second and what it keeps. pub detail: String, - /// Whether its CPU operations were priced from the measured table or + /// Whether its CPU operations were costed from the measured table or /// from the formulas. - pub price_source: NodePriceSource, + pub cost_source: NodeCostSource, } /// One candidate's usage and price over the whole workload: `usage` is @@ -397,9 +393,7 @@ pub struct PlanCost { /// sum of its nodes' charges plus request overhead for queries without /// operator nodes); `None` under cost model 3. pub cost_per_second: Option, - /// The unit of `cost_per_second`: always [`COST_UNIT`]. - pub cost_unit: &'static str, - /// Where its prices came from, from its nodes' [`NodeCost::price_source`]: + /// Where its prices came from, from its nodes' [`NodeCost::cost_source`]: /// [`COST_SOURCE`] (all formulas), [`MEASURED_COST_SOURCE`] (all /// measured) or [`MIXED_COST_SOURCE`]. pub source: &'static str, diff --git a/crates/cost/src/measured.rs b/crates/cost/src/measured.rs index 1fabd322c..e7915fff3 100644 --- a/crates/cost/src/measured.rs +++ b/crates/cost/src/measured.rs @@ -3,9 +3,9 @@ //! from instead of its formulas. //! //! The Planner defines the table's structure; the deployment's runtime -//! measures the numbers (e.g. ASAPQuery-backend's `measure_costs`, or -//! sketch-bench's `rqe_atomic_costs.json` converted by `asap-devtools`' -//! `convert_rqe_atomic_costs`) and puts the table in +//! measures the numbers (e.g. sketch-bench's `rqe_atomic_costs.json` +//! converted by `asap-devtools`' `convert_rqe_atomic_costs`) and puts the +//! table in //! [`CostInputs::measured_operation_costs`]. The planner reads //! only this table, never a benchmark file. //! @@ -503,14 +503,14 @@ struct TableFields { provenance: Option, } -/// Which prices one node's CPU operations came from. A node is one operator +/// Which costs one node's CPU operations came from. A node is one operator /// node of the candidate's physical plan DAG (`PhysicalASAPDAG`), keyed by /// its `PhysicalASAPNodeId` in [`PlanCost::per_node`]; not a machine. /// /// [`PlanCost::per_node`]: crate::PlanCost::per_node #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -pub enum NodePriceSource { +pub enum NodeCostSource { /// Plan selection's formulas (`analytical_cost` and the sketch shapes), /// scaled by the calibration factor when the table has entries. Analytical, diff --git a/crates/cost/src/ranking.rs b/crates/cost/src/ranking.rs index 2c1308b6c..42322c75f 100644 --- a/crates/cost/src/ranking.rs +++ b/crates/cost/src/ranking.rs @@ -8,7 +8,7 @@ use crate::{ClusterObjective, CostModel, PlanCost, QueryLatency, ResourceUsage}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PlanRanking { /// Cost models 1 and 2: by cost per second. - Price, + Cost, /// Latency, then peak memory, then disk. Latency, /// Peak memory, then disk, then latency. @@ -18,7 +18,7 @@ pub enum PlanRanking { impl PlanRanking { pub fn of(model: &CostModel) -> Self { match model { - CostModel::UsageOverTime(_) | CostModel::ProvisionedCapacity(_) => Self::Price, + CostModel::UsageOverTime(_) | CostModel::ProvisionedCapacity(_) => Self::Cost, CostModel::ClusterCapacity(limits) => match limits.objective { ClusterObjective::LowestLatency => Self::Latency, ClusterObjective::LeastMemory => Self::Memory, @@ -40,7 +40,7 @@ impl PlanRanking { let memory = || a.peak_memory_bytes.cmp(&b.peak_memory_bytes); let disk = || a.disk_bytes.cmp(&b.disk_bytes); let order = match self { - Self::Price => rounded( + Self::Cost => rounded( cost.cost_per_second.unwrap_or(f64::INFINITY), best.cost_per_second.unwrap_or(f64::INFINITY), ), diff --git a/crates/cost/src/state_bytes.rs b/crates/cost/src/state_bytes.rs index 4b7d297e6..fb18c5431 100644 --- a/crates/cost/src/state_bytes.rs +++ b/crates/cost/src/state_bytes.rs @@ -274,7 +274,7 @@ const DDSKETCH_ASSUMED_VALUE_RANGE: f64 = 1e9; #[cfg(test)] mod tests { use super::*; - use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm}; + use asap_types::ir::schema::SketchAlgorithm; /// An exact accumulator's state is priced at the executor's per-group /// state, 56 bytes for every kind (a counter's state alone is 48), so a @@ -376,10 +376,7 @@ mod tests { }; let model_bytes = |rows_per_state| { sketch_shape( - &SketchParams::Kll { - k, - failure_probability: None, - }, + &SketchParams::Kll { k }, StateInputRows { rows_per_state, non_negative: true, @@ -478,10 +475,7 @@ mod tests { let families = [ ( A::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, + SketchParams::Kll { k: 200 }, Rows::Values, Priced::Kll { k: 200 }, 0, @@ -543,10 +537,7 @@ mod tests { ), ( A::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, + SketchParams::Hll { precision: 12 }, Rows::Values, Priced::Exact, hll_precision_tag, @@ -554,10 +545,7 @@ mod tests { ), ( A::Kmv, - SketchParams::Kmv { - k: 1_024, - relative_error: None, - }, + SketchParams::Kmv { k: 1_024 }, Rows::Values, Priced::Exact, 0, @@ -572,7 +560,6 @@ mod tests { sketch_rows: 1, sketch_cols: 16, layers: 4, - distinct_failure_probability: None, }, Rows::Labels, Priced::Bound, @@ -616,19 +603,13 @@ mod tests { assert_eq!(key.to_bytes().unwrap().len(), ASSUMED_HEAP_KEY_BYTES); for (algorithm, params, rows, priced_as, fields, full) in &families { for n in [1, 100, *full] { - let family = FieldDataType::Sketch(*algorithm, GroupingStrategy::default()); + let family = FieldDataType::Sketch(*algorithm); let input = match rows { Rows::Values => &value, Rows::Labels => &labeled, Rows::Keyed => &keyed, }; - let mut updater = create_planner_accumulator( - &family, - Some(params), - input, - &GroupingStrategy::default(), - ) - .unwrap(); + let mut updater = create_planner_accumulator(&family, Some(params), input).unwrap(); for i in 0..n { match rows { // Distinct magnitudes over the assumed range, in a diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 02dce919e..8bfa7afec 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -18,17 +18,15 @@ use crate::state_bytes::{ }; use crate::{aggregate_lookup, lookup, StateFamily}; use crate::{ - CalibrationFit, CostInputs, CostModel, NodeCost, NodePriceSource, OperatorKind, PlanCost, - QueryLatency, ResourceUsage, COST_SOURCE, COST_UNIT, MEASURED_COST_SOURCE, MIXED_COST_SOURCE, + CalibrationFit, CostInputs, CostModel, NodeCost, NodeCostSource, OperatorKind, PlanCost, + QueryLatency, ResourceUsage, COST_SOURCE, MEASURED_COST_SOURCE, MIXED_COST_SOURCE, }; use asap_types::ir::operator::{GroupKeys, Reduction, Source}; use asap_types::ir::physical_export::{ - PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, ScanWindow, + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload as Payload, }; use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, Schema, SketchParams, SketchStatistic, -}; +use asap_types::ir::schema::{FieldDataType, Schema, SketchParams, SketchStatistic}; use asap_types::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; use asap_types::physical::{Lifetime, StorageMedium}; use asap_types::workload::{DataWorkload, QueryRecurrence, RangeEvaluation}; @@ -46,15 +44,14 @@ use asap_types::workload::bounds::{ /// ([`computations`]); an ingestion-time node over one second of arriving /// rows. A materialized node's state is counted in byte-seconds per /// second; a tumbling-window chain materialized beyond one evaluation keeps the tumbling windows its -/// retention covers ([`tumbling_window_retention_ms`]). `nodes` and `materialized_chains` are -/// `dag`'s ([`node_map`], [`materialized_tumbling_window_chains`]); `recurrence[q]` and +/// retention covers ([`tumbling_window_retention_ms`]). `nodes` is `dag`'s +/// ([`node_map`]); `recurrence[q]` and /// `range_evaluations[q]` say how often query `q` runs and how many steps /// each run evaluates (one when missing). #[allow(clippy::too_many_arguments)] pub fn plan_cost( dag: &PhysicalASAPDAG, nodes: &NodeMap<'_>, - materialized_chains: &HashSet, recurrence: &[QueryRecurrence], range_evaluations: &[Option], data: &DataWorkload, @@ -77,14 +74,12 @@ pub fn plan_cost( } = each_node_usage( dag, nodes, - materialized_chains, recurrence, range_evaluations, data, settings, formula_scale, None, - None, )?; let mut output: HashMap = HashMap::new(); let mut per_node = BTreeMap::new(); @@ -339,11 +334,10 @@ pub fn plan_cost( PlanCost { usage, cost_per_second: cost, - cost_unit: COST_UNIT, source: if has_unattached_request_overhead { if per_node .values() - .all(|node| node.price_source == NodePriceSource::Measured) + .all(|node| node.cost_source == NodeCostSource::Measured) { MEASURED_COST_SOURCE } else { @@ -368,24 +362,18 @@ struct EachNodeUsage<'a> { } /// [`node_usage`] over every node of `dag`, children first, with the -/// arguments of [`plan_cost`]. `root_scan_lookback_ms` is the -/// look-back of a query-time scan no window above it in `dag` bounds (a -/// scan priced without its consumers); `None` when the scan has no -/// window, so it reads the source's `rows_held` or fails with -/// `MissingSourceFact`, as Stage 3 does. `root_rates`, when given, are +/// arguments of [`plan_cost`]. `root_rates`, when given, are /// the computations per second of each root's consumer in place of its /// query's own rate ([`computations`]). #[allow(clippy::too_many_arguments)] fn each_node_usage<'a>( dag: &'a PhysicalASAPDAG, nodes: &NodeMap<'a>, - materialized_chains: &HashSet, recurrence: &[QueryRecurrence], range_evaluations: &[Option], data: &DataWorkload, settings: &'a CostInputs, formula_scale: f64, - root_scan_lookback_ms: Option, root_rates: Option<&[f64]>, ) -> Result, (PhysicalASAPNodeId, AnalyticalCostError)> { let first = dag.nodes.first().map(|n| n.id).unwrap_or_default(); @@ -421,25 +409,10 @@ fn each_node_usage<'a>( } }) .collect(); - let scan_window = match (&node.payload, node.output_state.timing) { - (Payload::NonASAP(NonASAPOp::Scan { .. }), ExecutionTiming::QueryTime) => { - match dag.scan_window(node.id, materialized_chains) { - // Priced as before #955 decides `@` anchors and - // unwindowed queries: the widest window read. - ScanWindow::Unbounded => trailing_window( - widest_window_read_ms(dag, nodes, node.id, materialized_chains) - .or(root_scan_lookback_ms), - ), - window => window, - } - } - _ => ScanWindow::Unbounded, - }; let node_priced = node_usage( node, &views, multipliers[&node.id], - scan_window, retained_tumbling_windows(node, &retention_ms), (node.output_state.timing == ExecutionTiming::IngestionTime) .then(|| maintained_window_ms(node.id, nodes, &producers_of)) @@ -469,15 +442,15 @@ fn state_family<'a>( match payload { Payload::ASAP( ASAPOp::SummaryAgg { - family, grouping, .. + summary_type: family, + .. } | ASAPOp::SummaryTumblingWindowChain { - family, grouping, .. + summary_type: family, + .. }, ) => match family { - FieldDataType::Sketch(..) - if *grouping == GroupingStrategy::PerSubpopulationInstance => - { + FieldDataType::Sketch(..) => { todo!("#1207 PR 6: the state family of θ from the candidate's LogicalProperties::parameters") } FieldDataType::ExactAggregate(kind, _) => StateFamily::Exact(kind), @@ -542,10 +515,8 @@ pub struct NodeUsage<'a> { /// Measure and price one node of a physical DAG (one step of /// [`plan_cost`]): `producers` are its producers in edge order, -/// `rate` its `(multiplier, per_second)` ([`computations`]), `scan_window` -/// the raw rows a query-time scan reads ([`PhysicalASAPDAG::scan_window`]; -/// read only for scans: it reads `width_ms` of rows and keeps them back to -/// [`ScanWindow::lookback_ms`]), `retained_tumbling_window_count` the tumbling windows a tumbling-window chain keeps +/// `rate` its `(multiplier, per_second)` ([`computations`]), +/// `retained_tumbling_window_count` the tumbling windows a tumbling-window chain keeps /// completed (`retained_tumbling_windows`), and `ingestion_window_ms` the window an /// ingestion-time node maintains (`maintained_window_ms`). #[allow(clippy::too_many_arguments)] @@ -553,7 +524,6 @@ pub fn node_usage<'a>( node: &'a asap_types::ir::physical_export::PhysicalASAPDAGNode, producers: &[ProducerOutput<'a>], rate: (u64, f64), - scan_window: ScanWindow, retained_tumbling_window_count: u64, ingestion_window_ms: Option, data: &DataWorkload, @@ -572,7 +542,7 @@ pub fn node_usage<'a>( let mut held = None; let mut maintained = None; let mut raw_scan = None; - // A query-time scan's raw rows retained ([`ScanWindow::lookback_ms`]). + // A query-time scan's raw rows retained. let mut raw_retained_rows = None; let ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; let inputs: Vec = producers.iter().map(|p| p.output).collect(); @@ -619,7 +589,7 @@ pub fn node_usage<'a>( let measured_state_bytes = unit_price .and_then(|p| p.memory_bytes_per_state) .map(|bytes| bytes.round() as u64); - let mut price_source = NodePriceSource::Analytical; + let mut cost_source = NodeCostSource::Analytical; let input = inputs .first() .copied() @@ -707,12 +677,11 @@ pub fn node_usage<'a>( // without a window). let source = source_name(source); let rows_held = data.sources.get(source).and_then(|facts| facts.rows_held); - let window_width_ms = match scan_window { - ScanWindow::Within { width_ms, .. } => { - Some(u64::try_from(width_ms).unwrap_or(0)) - } - ScanWindow::Unbounded => None, - }; + let (window_width_ms, retained_lookback_ms): (Option, Option) = + match ingestion { + true => (None, None), + false => todo!("#1207 PR 7: a query-time Scan's read per edge (the TimeRange above it) and its retained rows from retention_ms (h_v)"), + }; let lookback = match ingestion { true => Some(1_000), false => window_width_ms, @@ -726,7 +695,7 @@ pub fn node_usage<'a>( // oldest row it reads, `end_offset + width` (an offset // window keeps more than it reads), at most those held. if !ingestion { - let retained_rows = match (by_rate(scan_window.lookback_ms()), rows_held) { + let retained_rows = match (by_rate(retained_lookback_ms), rows_held) { (Some(by_rate), Some(rows_held)) => Some(by_rate.min(rows_held)), (by_rate, rows_held) => by_rate.or(rows_held), }; @@ -906,7 +875,9 @@ pub fn node_usage<'a>( } }, Payload::ASAP(ASAPOp::SummaryAgg { - family, reduction, .. + summary_type: family, + reduction, + .. }) => { let group_count = groups(reduction)?; // At query time, each state sees its group's input rows; at @@ -933,7 +904,7 @@ pub fn node_usage<'a>( )), format!( "build {} into {group_count} states: {} rows x depth {depth}", - family_name(family), + summary_type_name(family), input.rows ), ) @@ -1020,7 +991,7 @@ pub fn node_usage<'a>( // non-negative whatever the factor). let estimate = match unit_price { Some(unit) => { - price_source = NodePriceSource::Measured; + cost_source = NodeCostSource::Measured; let measured_ms = (units as f64 * unit.ns_per_unit + unit.ns_per_invocation) * 1e-6; fit.record(measured_ms * per_second, formula_ms * per_second); detail += &format!( @@ -1198,14 +1169,14 @@ pub fn node_usage<'a>( cpu_core_seconds_per_second: node_ms * per_second / 1_000.0, multiplier, detail, - price_source, + cost_source, }, }) } /// [`PlanCost::source`] for nodes priced as `per_node` says. fn cost_source(per_node: &BTreeMap) -> &'static str { - let measured = |node: &NodeCost| node.price_source == NodePriceSource::Measured; + let measured = |node: &NodeCost| node.cost_source == NodeCostSource::Measured; if !per_node.values().any(measured) { COST_SOURCE } else if per_node.values().all(measured) { @@ -1453,15 +1424,6 @@ pub fn maintained_window_ms( } } -/// The window of a scan reading the `lookback_ms` up to the evaluation -/// (`Unbounded` for `None`). -pub fn trailing_window(lookback_ms: Option) -> ScanWindow { - lookback_ms.map_or(ScanWindow::Unbounded, |width_ms| ScanWindow::Within { - end_offset_ms: 0, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }) -} - /// The nodes of `dag` by id. pub fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { dag.nodes.iter().map(|n| (n.id, n)).collect() @@ -1471,79 +1433,11 @@ pub fn node_map(dag: &PhysicalASAPDAG) -> NodeMap<'_> { pub type NodeMap<'a> = HashMap; -/// The query-time tumbling-window chains of `dag` materialized across evaluations -/// (lifetime `AcrossEvaluations` at query time, which Stage 2 offers only when the -/// deployment keeps query-time results and every query builds one new -/// tumbling window per evaluation). Each evaluation builds the newest tumbling window, and the -/// older ones, which complete the queries' look-back, are retained from -/// earlier evaluations. -pub fn materialized_tumbling_window_chains(dag: &PhysicalASAPDAG) -> HashSet { - dag.nodes - .iter() - .filter(|node| { - matches!( - (&node.payload, node.output_state.timing, node.lifetime), - ( - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }), - ExecutionTiming::QueryTime, - Lifetime::AcrossEvaluations(_), - ) - ) - }) - .map(|node| node.id) - .collect() -} - -/// The widest window, in milliseconds, read above the scan `scan` at query -/// time, ignoring queries that read it outside a window and the anchors of -/// shifts: each range `TimeRange` and `SummaryTumblingWindowChain` (its whole window, or -/// one tumbling window for a chain in `materialized_chains`) reached through time shifts. -/// Prices a scan whose [`PhysicalASAPDAG::scan_window`] is unbounded as -/// Stage 3 did before; #955 decides `@` anchors. -fn widest_window_read_ms( - dag: &PhysicalASAPDAG, - nodes: &NodeMap<'_>, - scan: PhysicalASAPNodeId, - materialized_chains: &HashSet, -) -> Option { - let mut widest = None; - let mut stack = vec![scan]; - let mut seen = HashSet::new(); - while let Some(id) = stack.pop() { - if !seen.insert(id) { - continue; - } - for edge in dag.edges.iter().filter(|e| e.producer == id) { - let width_ms = match &nodes[&edge.consumer].payload { - Payload::NonASAP(NonASAPOp::TimeRange { range, .. }) => range.as_millis() as u64, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) if materialized_chains.contains(&edge.consumer) => *tumbling_window_width_ms, - Payload::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( - "#1207 PR 7: the window a chain that is not materialized reads, from the TimeRange over it" - ), - Payload::NonASAP(NonASAPOp::TimeShift { .. }) => { - stack.push(edge.consumer); - continue; - } - _ => continue, - }; - widest = widest.max(Some(width_ms)); - } - } - widest -} - -/// A summary family's name in reports: a shared (Hydra) structure by its -/// Hydra kind, e.g. `HydraCms`; a sketch by its algorithm; an exact -/// accumulator by its kind. -pub fn family_name(family: &FieldDataType) -> String { - match family { - FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { - format!("{kind:?}") - } - FieldDataType::Sketch(algorithm, _) => format!("{algorithm:?}"), +/// A summary type's name in reports: a sketch by its algorithm, e.g. +/// `HydraCms`; an exact accumulator by its kind. +pub fn summary_type_name(summary_type: &FieldDataType) -> String { + match summary_type { + FieldDataType::Sketch(algorithm) => format!("{algorithm:?}"), FieldDataType::ExactAggregate(kind, _) => format!("exact {kind:?} accumulator"), other => format!("{other:?}"), } diff --git a/crates/devtools/examples/canonical_examples.rs b/crates/devtools/examples/canonical_examples.rs index fbfb65be6..c618a2e26 100644 --- a/crates/devtools/examples/canonical_examples.rs +++ b/crates/devtools/examples/canonical_examples.rs @@ -6,7 +6,6 @@ use asap_devtools::corpora::{bgp_catalog, dqc_catalog}; use asap_devtools::lower_promql_with_sample_interval; use asap_frontend_sql::lower_sql_dialect; -use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::SqlDialect; #[tokio::main] @@ -35,7 +34,7 @@ async fn main() { ]; for (label, q) in promql_examples { println!("=== {label} === promql> {q}"); - match lower_promql_with_sample_interval(q, AccuracyTarget::exact(), 1_000) { + match lower_promql_with_sample_interval(q, 1_000) { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } @@ -63,14 +62,7 @@ async fn main() { &packets }; println!("=== {label} === sql> {q}"); - match lower_sql_dialect( - q, - catalog, - SqlDialect::DataFusionSQL, - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(q, catalog, SqlDialect::DataFusionSQL).await { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } diff --git a/crates/devtools/examples/topk_ir.rs b/crates/devtools/examples/topk_ir.rs index 5464fcae4..1765854de 100644 --- a/crates/devtools/examples/topk_ir.rs +++ b/crates/devtools/examples/topk_ir.rs @@ -4,7 +4,6 @@ // resulting pre-ASAP IR. Used for interactive exploration; not a test. use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; fn col(name: &str, dtype: DataType) -> Field { @@ -39,7 +38,7 @@ fn catalog() -> SqlCatalog { async fn show_sql(label: &str, query: &str) { println!("━━━ {label} ━━━"); println!("{query}"); - match lower_sql(query, &catalog(), AccuracyTarget::exact()).await { + match lower_sql(query, &catalog()).await { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } @@ -49,7 +48,7 @@ async fn show_sql(label: &str, query: &str) { fn show_promql(label: &str, query: &str) { println!("━━━ {label} ━━━"); println!("{query}"); - match lower_promql_with_sample_interval(query, AccuracyTarget::exact(), 1_000) { + match lower_promql_with_sample_interval(query, 1_000) { Ok(qe) => println!("{qe:#?}"), Err(e) => println!("ERR: {e}"), } diff --git a/crates/devtools/src/bin/analyze_corpora.rs b/crates/devtools/src/bin/analyze_corpora.rs index 5841cfab9..8a50f6698 100644 --- a/crates/devtools/src/bin/analyze_corpora.rs +++ b/crates/devtools/src/bin/analyze_corpora.rs @@ -9,7 +9,6 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_types::ir::properties::AccuracyTarget; use asap_types::workload::SqlDialect; use serde::Serialize; use serde_json::Value; @@ -124,7 +123,7 @@ fn run_corpus(name: &str, source: &str, interval_ms: u64) -> CorpusResult { for (index, expression) in promql_lines(source).enumerate() { let normalized_expression = normalize(expression); let structural_shape = structural_shape(expression); - match lower_promql_with_sample_interval(expression, AccuracyTarget::exact(), interval_ms) { + match lower_promql_with_sample_interval(expression, interval_ms) { Ok(ir) => result.lowered.push(DumpRecord { corpus: name.to_string(), query_number: index + 1, @@ -392,14 +391,7 @@ async fn run_sql_corpora(out_dir: PathBuf) { for (index, expression) in queries.into_iter().enumerate() { let normalized_expression = normalize(&expression); let structural_shape = structural_shape(&expression); - match lower_sql_dialect( - &expression, - &catalog, - dialect.clone(), - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(&expression, &catalog, dialect.clone()).await { Ok(ir) => lowered.push(DumpRecord { corpus: name.to_string(), query_number: index + 1, diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index a236e5ecf..ca50f72e8 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -53,23 +53,17 @@ // - stage0_query_language_frontends: the frontends' DAG, one root per query; // - stage1_logical_asap_optimization (from `stage1_pass1`): `pruned_candidates`, // the ways to compute a measure Pass 1 pruned (each on its aggregate's -// memo `group`, with `candidate` and `reason`); with `stage3`, -// `candidates` holds `L1`, the selected plan's roots (as Stage 2 built -// them, with Stage 3's weight proofs); `error` when Pass 1 failed; +// memo `group`, with `candidate` and `reason`); `error` when Pass 1 +// failed; // - stage2_physical_asap_optimization (from `stage2`): `pruned_candidates`, // the physical candidates Stage 2 pruned (each with its memo `group`, // `operator_node`, `operator`, `implementation_root`, `timing`, -// `lifetime` and `reason`); with `stage3`, `candidates` holds -// `P1`, the selected plan as Stage 2 built it; -// - stage3_plan_selection (from `stage3`): Stage 3's `Selection`: -// `selected_dag` (`candidate_of_node`, `edges`, -// `node_of_output_reference`), `nodes_with_proven_non_negative_weights`, -// the plan's `cost` (its `usage`, `cost_per_second` and per-node -// charges), `accepted_accuracy_per_query`, `pruned_candidates` and -// `search_report`; `error` when no plan was selected. -// `tumbling_window_retention_ms` maps each tumbling-window chain the plan -// keeps beyond one evaluation (by node id in `P1`) to how far back its -// tumbling windows are retained. +// `lifetime` and `reason`); +// - stage3_plan_selection (from `stage3`): the `plan` Stage 3 built (a +// `PhysicalASAPDAG`) and its `Selection`: `accepted_accuracy_per_query`, +// `pruned_candidates`, `milp_variables`, `milp_constraints`, +// `milp_construction_ms` and `milp_solve_ms`; `error` when no plan was +// selected. // `materialized_views` are the plan's materialized views as // `plan_installable` derives them, each with the node of `P1` it is // installed from (`stage2_node_id`), its position among that node's @@ -99,7 +93,7 @@ use std::collections::btree_map::Entry; use std::collections::BTreeSet; -use asap_cost::{CostInputs, ResourceUsage}; +use asap_cost::CostInputs; use asap_devtools::candidate_labels::node_label; use asap_devtools::planning_stages::{ lower_promql_roots, lower_sql_roots, run_stages_until, sql_catalog, PlanningStagesError, @@ -112,6 +106,7 @@ use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; +use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::properties::{default_metric_for_root, AccuracyTarget}; use asap_types::ir::QueryRoot; use asap_types::stages::PhysicalASAPMemoCandidates; @@ -238,11 +233,11 @@ fn run(args: Vec) -> Result<(), Failure> { .as_ref() .ok() .and_then(|run| run.run.selection.as_ref().ok()) - .and_then(|selection| { - large_peak_memory_warning( - selection.cost.usage.peak_memory_bytes, - &cost_inputs.cost_model, - ) + .map(|(_plan, _selection)| -> u64 { + todo!("#1207 PR 6: the selected plan's peak memory, from plan_cost(&plan)") + }) + .and_then(|peak_memory_bytes| { + large_peak_memory_warning(peak_memory_bytes, &cost_inputs.cost_model) }), ); let text = if json_format { @@ -305,10 +300,7 @@ fn document_of( "workload": { "queries": queries }, "stage0_query_language_frontends": { "dag": export(roots) }, "stage1_logical_asap_optimization": { - "combinations": 0, - "capped": false, "pruned_candidates": [], - "candidates": [], "error": format!("Pass 1: {error}"), }, "stage2_physical_asap_optimization": null, @@ -322,30 +314,8 @@ fn document_of( /// section of a stage `run` ran, the others null. fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningStagesRun) -> Value { let until = run.run_until; - let labels = asap_devtools::candidate_labels::candidate_labels(run); - let logical: Vec<_> = run - .logical_candidates() - .iter() - .map(|c| match &c.roots { - Ok(roots) => { - json!({ "id": c.id, "display_name": labels[&c.id], "dag": export(roots) }) - } - Err(error) => { - json!({ "id": c.id, "display_name": labels[&c.id], "error": error }) - } - }) - .collect(); - let physical: Vec<_> = run - .physical_candidates() - .iter() - .map(|p| json!({ "id": p.id, "from_logical": p.from_logical, "display_name": labels[&p.id], "dag": p.dag })) - .collect(); let stage1 = (until >= RunUntil::Pass1).then(|| { - let mut stage1 = json!({ - "combinations": logical.len(), - "capped": false, - "candidates": logical, - }); + let mut stage1 = json!({}); // Pass 1 or Pass 2 failed, or the deployment's capabilities // contradict the data workload. if run.run.logical_asap_candidates.is_none() { @@ -359,7 +329,6 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS }); let stage2 = (until >= RunUntil::Stage2).then(|| { let mut stage2 = json!({ - "candidates": physical, "pruned": [], "pruned_candidates": stage2_pruned_candidates_json(run.run.physical_asap_candidates.as_ref()), }); @@ -384,27 +353,19 @@ fn document(queries: Vec, entries: &[QueryWorkloadEntry], run: &PlanningS }) } -/// Stage 3's section: the selection, the selected plan's tumbling-window retention and -/// its materialized views. +/// Stage 3's section: the plan, the selection and the plan's materialized +/// views. fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value { let mut stage3 = selection_json(&run.run.selection); - match (&run.run.selection, &run.run.selected_physical_plan) { - (Ok(selection), Some(realized)) => { - stage3["tumbling_window_retention_ms"] = realized - .tumbling_window_retention_ms - .iter() - .map(|(node, ms)| (node.to_string(), json!(ms))) - .collect::>() - .into(); - match materialized_views_json(realized, selection, entries) { - Ok(views) => stage3["materialized_views"] = Value::Array(views), - Err(error) => { - stage3["materialized_views"] = json!([]); - stage3["materialized_views_error"] = json!(error); - } + match &run.run.selection { + Ok((plan, _)) => match materialized_views_json(plan, entries) { + Ok(views) => stage3["materialized_views"] = Value::Array(views), + Err(error) => { + stage3["materialized_views"] = json!([]); + stage3["materialized_views_error"] = json!(error); } - } - _ => stage3["materialized_views"] = json!([]), + }, + Err(_) => stage3["materialized_views"] = json!([]), } stage3 } @@ -623,12 +584,10 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option Result, String> { - let (plan, report) = - installable_plan_of(realized, Some(selection), entries).map_err(|e| e.to_string())?; + let (plan, report) = installable_plan_of(plan, None, entries).map_err(|e| e.to_string())?; let views = plan.materialized_views().map_err(|e| e.to_string())?; let mut stage2_of = std::collections::HashMap::new(); for node in report @@ -669,56 +628,14 @@ fn stage2_pruned_candidates_json(stage2: Option<&PhysicalASAPMemoCandidates>) -> todo!("#1207 PR 9 (#1205): each pruned PhysicalCandidate with its reason") } -/// One plan's cost: per second, its predicted CPU, unit and source, its -/// usage and each node's charge. -fn cost_json(cost: &asap_cost::PlanCost) -> Value { - let per_node: serde_json::Map<_, _> = cost - .per_node - .iter() - .map(|(node, c)| { - ( - node.to_string(), - json!({ - "operator": c.operator, - "timing": c.timing, - "cpu_core_seconds_per_second": c.cpu_core_seconds_per_second, - "cost_per_second": c.cost_per_second, - "multiplier": c.multiplier, - "detail": c.detail, - "price_source": c.price_source, - }), - ) - }) - .collect(); - json!({ - "cost_per_second": cost.cost_per_second, - // Report-only: the nodes' predicted CPU, summed as - // `plan_installable`'s predicted usage sums it. - "cpu_core_seconds_per_second": cost - .per_node - .values() - .map(|node| node.cpu_core_seconds_per_second) - .sum::(), - "cost_unit": cost.cost_unit, - "source": cost.source, - "usage": usage_json(&cost.usage), - "per_node": per_node, - }) -} - -/// Per query, the accuracy Stage 3 accepted: the target's metric, the -/// accepted metric, whether it was accepted regardless of its guarantee, -/// why the guarantee check would have rejected it, the sketches read and a -/// one-line summary. +/// Per query, the accuracy Stage 3 accepted: the accepted metric, the +/// sketches read and a one-line summary. fn accepted_json(queries: &[AcceptedAccuracy]) -> Value { queries .iter() .map(|accepted| { json!({ - "target_metric": accepted.target_metric, "accepted_metric": accepted.accepted_metric, - "accepted_regardless_of_guarantee": accepted.accepted_regardless_of_guarantee, - "guarantee_check_rejection": accepted.guarantee_check_rejection, "sketches": accepted.sketches, "summary": accepted.to_string(), }) @@ -726,57 +643,26 @@ fn accepted_json(queries: &[AcceptedAccuracy]) -> Value { .collect() } -/// Stage 3's selection ([`Selection`]): the selected DAG over the selected -/// round's Stage 2 candidates (one node per run of a candidate), the nodes -/// whose weights it proved non-negative, the plan's cost, the accuracy accepted per query, the -/// candidates it pruned and its search report; `error` when the facade -/// selected no plan. -fn selection_json(selection: &Result) -> Value { - let selection = match selection { - Ok(selection) => selection, - Err(error) => return json!({ "selected_dag": null, "error": error.to_string() }), - }; - let dag = &selection.selected_dag; - let nodes = |nodes: &[asap_types::stages::SelectedNodeId]| -> Vec { - nodes.iter().map(|node| node.0).collect() +/// Stage 3's plan and selection ([`Selection`]): the plan it built, the +/// accuracy accepted per query, the candidates it pruned and the selection +/// MILP's size and time; `error` when the facade selected no plan. +fn selection_json(selection: &Result<(PhysicalASAPDAG, Selection), SelectionError>) -> Value { + let (plan, selection) = match selection { + Ok(selected) => selected, + Err(error) => return json!({ "plan": null, "error": error.to_string() }), }; json!({ - "selected_dag": { - "candidate_of_node": dag.candidate_of_node.iter().map(|id| id.0).collect::>(), - "edges": dag - .edges - .iter() - .map(|e| json!({ "producer": e.producer.0, "consumer": e.consumer.0, "input": e.input_index_of_consumer })) - .collect::>(), - "node_of_output_reference": nodes(&dag.node_of_output_reference), - }, - "nodes_with_proven_non_negative_weights": - nodes(&selection.nodes_with_proven_non_negative_weights), - "cost": cost_json(&selection.cost), + "plan": plan, "accepted_accuracy_per_query": accepted_json(&selection.accepted_accuracy_per_query), "pruned_candidates": selection .pruned_candidates .iter() - .map(|p| json!({ "candidate": p.candidate.0, "reason": format!("{:?}", p.reason) })) - .collect::>(), - "search_report": selection.search_report, - }) -} - -/// A candidate's resource usage, in the units its field names give. -fn usage_json(usage: &ResourceUsage) -> Value { - json!({ - "cpu_ops_per_second": usage.cpu_ops_per_second, - "scanned_bytes_per_second": usage.scanned_bytes_per_second, - "average_memory_bytes": usage.average_memory_bytes, - "peak_memory_bytes": usage.peak_memory_bytes, - "disk_bytes": usage.disk_bytes, - "peak_cpu_cores": usage.peak_cpu_cores, - "latency_per_query": usage - .latency_per_query - .iter() - .map(|l| json!({ "compaction_ms": l.compaction_ms, "query_ms": l.query_ms })) + .map(|p| json!({ "candidate": format!("{:?}", p.candidate), "reason": format!("{:?}", p.reason) })) .collect::>(), + "milp_variables": selection.milp_variables, + "milp_constraints": selection.milp_constraints, + "milp_construction_ms": selection.milp_construction_ms, + "milp_solve_ms": selection.milp_solve_ms, }) } diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index e11557d33..4af160d1e 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -1,20 +1,15 @@ -// cargo run -p asap-devtools --bin sketch_coverage -- --sample-interval-ms [--max-error ] +// cargo run -p asap-devtools --bin sketch_coverage -- --sample-interval-ms // // Lowers every query in every corpus we have (mirrors `variant_coverage`'s -// corpus list exactly, so the two reports are directly comparable) with an -// *approximate* `AccuracyTarget`, runs Pass 1 of logical ASAP-aware -// optimization (`pass1_memo`, every -// summary supported, no accuracy pruning) over each query, and +// corpus list exactly, so the two reports are directly comparable), runs +// Pass 1 of logical ASAP-aware optimization (`pass1_memo`, every summary +// supported) over each query, and // reports which summary candidates Pass 1 offers for it, plus the fraction of // lowered queries with at least one. // // `--sample-interval-ms ` is the sample interval declared for every // metric a query reads, which its bare instant selectors look back. -// `--max-error ` (ε, default 0.01) sets the `AccuracyTarget` every query in -// every corpus lowers with. Without an approximate target, Pass 1 offers no -// summary candidate. -use asap_accuracy::{IgnoreAccuracyGuarantees, NoAccuracyEvidence}; use asap_devtools::corpora::{ bgp_catalog, dqc_catalog, netflow_catalog, parse_sample_interval, promql_lines, sql_statements, }; @@ -22,10 +17,9 @@ use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; use asap_logical_asap_optimization::memo_of_queries; use asap_logical_asap_optimization::pass1_memo; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::workload::{DataWorkload, SqlDialect}; +use asap_types::workload::SqlDialect; use std::collections::BTreeSet; use std::rc::Rc; @@ -47,31 +41,23 @@ fn pct(n: usize, total: usize) -> String { } } -/// Run Pass 1 over each of one corpus's already-lowered queries (each -/// lowered with `accuracy`) and collect the summary candidates it offers +/// Run Pass 1 over each of one corpus's already-lowered queries and collect the summary candidates it offers /// for any of the query's targets. fn analyze_corpus( name: &'static str, roots: Vec<(String, Rc)>, failed: usize, - accuracy: &AccuracyTarget, ) -> CorpusCoverage { let lowered = roots.len(); let mut rejected = 0; let mut summaries = Vec::new(); for (id, root) in roots { - let Ok(memo) = memo_of_queries(&[QueryRoot::Operator(root)]) else { + let Ok(memo) = memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()) + else { rejected += 1; continue; }; - let Ok(pass1) = pass1_memo( - memo, - std::slice::from_ref(accuracy), - &IgnoreAccuracyGuarantees, - &NoAccuracyEvidence, - &DataWorkload::default(), - None, - ) else { + let Ok(pass1) = pass1_memo(memo, None) else { rejected += 1; continue; }; @@ -99,10 +85,11 @@ fn analyze_corpus( /// Adds the family of the summary `operator` builds, if any, to `families`. fn summary_family(operator: &Operator, families: &mut BTreeSet) { if let Operator::ASAP( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, ) = operator { - if let FieldDataType::Sketch(kind, _) = family { + if let FieldDataType::Sketch(kind) = summary_type { families.insert(format!("{:?}", kind)); } } @@ -133,24 +120,9 @@ fn report(r: &CorpusCoverage) { println!(); } -fn parse_max_error() -> f64 { - let mut args = std::env::args().skip(1); - while let Some(arg) = args.next() { - if arg == "--max-error" { - let raw = args.next().expect("--max-error requires a value"); - return raw - .parse() - .unwrap_or_else(|_| panic!("--max-error must be a float, got {raw:?}")); - } - } - 0.01 -} - #[tokio::main] async fn main() { - let max_error = parse_max_error(); let interval_ms = parse_sample_interval(); - let accuracy = AccuracyTarget::max_error(max_error); let mut results = Vec::new(); // ── PromQL corpora — same set as `variant_coverage` ── @@ -182,12 +154,12 @@ async fn main() { let mut roots = Vec::new(); let mut failed = 0; for (i, q) in promql_lines(corpus).enumerate() { - match lower_promql_with_sample_interval(q, accuracy.clone(), interval_ms) { + match lower_promql_with_sample_interval(q, interval_ms) { Ok(qe) => roots.push((format!("q{i}"), qe)), Err(_) => failed += 1, } } - results.push(analyze_corpus(name, roots, failed, &accuracy)); + results.push(analyze_corpus(name, roots, failed)); } // ── SQL corpora ── @@ -217,12 +189,12 @@ async fn main() { let mut roots = Vec::new(); let mut failed = 0; for (i, q) in sql_statements(corpus).into_iter().enumerate() { - match lower_sql_dialect(&q, &catalog, dialect.clone(), accuracy.clone()).await { + match lower_sql_dialect(&q, &catalog, dialect.clone()).await { Ok(qe) => roots.push((format!("q{i}"), qe)), Err(_) => failed += 1, } } - results.push(analyze_corpus(name, roots, failed, &accuracy)); + results.push(analyze_corpus(name, roots, failed)); } for r in &results { @@ -234,7 +206,7 @@ async fn main() { let total_rejected: usize = results.iter().map(|r| r.rejected).sum(); let total_covered: usize = results.iter().map(covered).sum(); - println!("=== global (max_error = {max_error}) ==="); + println!("=== global ==="); println!( "total lowered: {total_lowered}, total failed: {total_failed}, rejected by Pass 1: {total_rejected}" ); diff --git a/crates/devtools/src/bin/variant_coverage.rs b/crates/devtools/src/bin/variant_coverage.rs index 5c0458e1c..0555af167 100644 --- a/crates/devtools/src/bin/variant_coverage.rs +++ b/crates/devtools/src/bin/variant_coverage.rs @@ -11,7 +11,6 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{OperatorNode, ScalarExpr}; use asap_types::workload::SqlDialect; use std::collections::BTreeSet; @@ -201,7 +200,7 @@ async fn main() { let mut lowered = 0; let mut failed = 0; for q in promql_lines(corpus) { - match lower_promql_with_sample_interval(q, AccuracyTarget::exact(), interval_ms) { + match lower_promql_with_sample_interval(q, interval_ms) { Ok(qe) => { walk(&qe, &mut variants); lowered += 1; @@ -229,14 +228,7 @@ async fn main() { let mut lowered = 0; let mut failed = 0; for q in sql_statements(corpus) { - match lower_sql_dialect( - &q, - &catalog, - SqlDialect::DataFusionSQL, - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(&q, &catalog, SqlDialect::DataFusionSQL).await { Ok(qe) => { walk(&qe, &mut variants); lowered += 1; @@ -262,14 +254,7 @@ async fn main() { let mut lowered = 0; let mut failed = 0; for q in sql_statements(corpus) { - match lower_sql_dialect( - &q, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - { + match lower_sql_dialect(&q, &catalog, SqlDialect::ClickhouseSQL).await { Ok(qe) => { walk(&qe, &mut variants); lowered += 1; diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 6351d71e0..5bfe456b4 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -4,101 +4,11 @@ //! are pure functions of the candidates' planning data (the logical roots //! and the physical properties assignment). -use std::collections::BTreeMap; -use std::rc::Rc; - use serde_json::Value; -use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::NonASAPOp; -use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; -use asap_types::physical::Lifetime; -use asap_types::physical::PhysicalWorkloadPlan; +use asap_types::ir::schema::SketchAlgorithm; use asap_types::stages::HowToComputeAggregateMeasure; -use crate::planning_stages::PlanningStagesRun; - -/// The label of every logical and physical candidate the document shows -/// of `run` ([`PlanningStagesRun::logical_candidates`], -/// [`PlanningStagesRun::physical_candidates`]), by id. -pub fn candidate_labels(run: &PlanningStagesRun) -> BTreeMap { - let mut labels = BTreeMap::new(); - for candidate in run.logical_candidates() { - let label = candidate - .roots - .as_deref() - .map(connected_candidate_label) - .unwrap_or_else(|_| "Unimplemented candidate".into()); - labels.insert(candidate.id.clone(), label); - } - for candidate in &run.physical_candidates() { - let logical = labels - .get(&candidate.from_logical) - .cloned() - .unwrap_or_default(); - labels.insert(candidate.id.clone(), physical_label(&logical, candidate)); - } - labels -} - -/// Joint selections carry their actual connected DAG rather than a Cartesian -/// choice vector. Derive labels from that DAG so a sketch is never called exact. -fn connected_candidate_label(roots: &[QueryRoot]) -> String { - let mut labels = Vec::new(); - let mut producers: BTreeMap)> = BTreeMap::new(); - for (query, root) in roots.iter().enumerate() { - let mut states = Vec::new(); - let mut seen = std::collections::HashSet::new(); - for operator in root.operator_refs() { - for node in OperatorNode::reachable(operator) { - let identity = Rc::as_ptr(&node) as usize; - if !seen.insert(identity) { - continue; - } - if matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) - ) { - let name = build_label(&node.operator); - if !states.contains(&name) { - states.push(name.clone()); - } - producers - .entry(identity) - .or_insert_with(|| (name, Vec::new())) - .1 - .push(query); - } else if matches!(node.non_asap(), Some(NonASAPOp::Scan { .. })) { - producers - .entry(identity) - .or_insert_with(|| ("Scan".into(), Vec::new())) - .1 - .push(query); - } - } - } - labels.push(format!( - "Q{} {}", - query + 1, - if states.is_empty() { - "exact".into() - } else { - states.join(" + ") - } - )); - } - let mut shared: Vec<_> = producers - .values() - .filter(|(_, queries)| queries.len() > 1) - .map(|(name, queries)| format!("shared {name} ({})", query_names(queries))) - .collect(); - shared.sort(); - labels.extend(shared); - labels.join(" · ") -} - /// A Pass 1 per-computation candidate, e.g. "exact", "Sum acc" or "Kll". pub fn computation_label(candidate: &HowToComputeAggregateMeasure) -> String { match candidate { @@ -111,142 +21,6 @@ pub fn computation_label(candidate: &HowToComputeAggregateMeasure) -> String { } } -/// `what` with the queries that read it, e.g. "Kll for Q1, Q3"; `what` -/// alone when none is found (a node later rules rebuilt). -fn for_queries(what: String, queries: &[usize]) -> String { - match queries.is_empty() { - true => what, - false => format!("{what} for {}", query_names(queries)), - } -} - -/// Query names for query indices, e.g. "Q1, Q3". -fn query_names(queries: &[usize]) -> String { - queries - .iter() - .map(|q| format!("Q{}", q + 1)) - .collect::>() - .join(", ") -} - -/// `ms` in the largest whole unit: "1m", "10s", "250ms". -fn duration_name(ms: u64) -> String { - match ms { - ms if ms % 3_600_000 == 0 => format!("{}h", ms / 3_600_000), - ms if ms % 60_000 == 0 => format!("{}m", ms / 60_000), - ms if ms % 1_000 == 0 => format!("{}s", ms / 1_000), - ms => format!("{ms}ms"), - } -} - -/// `logical` followed by what the candidate materializes, e.g. -/// "… · ingestion time: Kll for Q1, 5 × 1m tumbling windows · query time, shared: -/// DDSketch"; -/// just `logical` when nothing is. -pub fn physical_label(logical: &str, candidate: &PhysicalWorkloadPlan) -> String { - let mut parts: Vec<(&str, Vec)> = Vec::new(); - for node in &candidate.dag.nodes { - let id = &node.id; - let phase = match (node.output_state.timing, node.lifetime) { - (_, Lifetime::None) => continue, - (ExecutionTiming::IngestionTime, _) => "ingestion time", - (ExecutionTiming::QueryTime, Lifetime::OneEvaluation(_)) => "query time, shared", - (ExecutionTiming::QueryTime, Lifetime::AcrossEvaluations(_)) => { - "query time, materialized across evaluations" - } - }; - let label = match &node.payload { - // Name the chain: chains of one family differ by their queries - // and tumbling-window width. - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - }) => format!( - "{}, {} tumbling windows", - for_queries( - state_label(&node.payload), - &dag_readers(&candidate.dag, *id) - ), - duration_name(*tumbling_window_width_ms) - ), - payload => build_label(payload), - }; - match parts.iter_mut().find(|(p, _)| *p == phase) { - Some((_, labels)) => labels.push(label), - None => parts.push((phase, vec![label])), - } - } - std::iter::once(logical.to_string()) - .chain( - parts - .into_iter() - .map(|(phase, labels)| format!("{phase}: {}", labels.join(", "))), - ) - .collect::>() - .join(" · ") -} - -/// A summary build, e.g. "Kll tumbling windows" or "Sum acc". -fn build_label(payload: &Operator) -> String { - match payload { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => { - format!("{} tumbling windows", state_label(payload)) - } - _ => state_label(payload), - } -} - -/// The state a summary build or tumbling-window chain keeps, e.g. "Kll" or "Sum acc"; -/// any other operator's kind. -fn state_label(payload: &Operator) -> String { - match payload { - Operator::ASAP( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, - ) => match family { - FieldDataType::Sketch(algorithm, _) => sketch_name(algorithm), - FieldDataType::ExactAggregate(kind, _) => format!("{kind:?} acc"), - _ => payload.kind_name().to_string(), - }, - _ => payload.kind_name().to_string(), - } -} - -/// The queries (in workload order) whose root reaches node `id` of `dag`. -fn dag_readers(dag: &PhysicalASAPDAG, id: PhysicalASAPNodeId) -> Vec { - // Query `q` is the scalar root keyed `q` when there is one, and - // otherwise the next operator root. - let mut operator_roots = dag.roots.iter(); - let query_starts: Vec> = (0..dag.roots.len() + dag.scalar_roots.len()) - .map(|q| match dag.scalar_roots.iter().find(|(i, _)| *i == q) { - Some((_, expr)) => expr.operator_refs().into_iter().copied().collect(), - None => operator_roots.next().into_iter().copied().collect(), - }) - .collect(); - query_starts - .into_iter() - .enumerate() - .filter(|(_, starts)| { - let mut seen = std::collections::HashSet::new(); - let mut stack = starts.clone(); - while let Some(node) = stack.pop() { - if node == id { - return true; - } - if seen.insert(node) { - stack.extend( - dag.edges - .iter() - .filter(|e| e.consumer == node) - .map(|e| e.producer), - ); - } - } - false - }) - .map(|(q, _)| q) - .collect() -} - /// A sketch algorithm's display name, e.g. "CMS+heap" or "Kll". fn sketch_name(algorithm: &SketchAlgorithm) -> String { match algorithm { @@ -377,54 +151,4 @@ fn duration_text(value: &Value) -> String { } #[cfg(test)] -mod tests { - use super::*; - use crate::planner_layering_examples::example1; - use crate::planning_stages::lower_promql_roots; - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - - /// Selected DAG labels expose accumulator state without a legacy choice ID. - #[test] - fn connected_labels_describe_selected_states() { - let workload = example1(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &lower_promql_roots(&workload).unwrap(), - ) - .unwrap(), - ) - .unwrap(); - let choice = vec![0; inventory.targets.len()]; - let roots: Vec<_> = compose_logical_candidate(&inventory, &choice).unwrap(); - let label = connected_candidate_label(&roots); - assert!(label.contains("Q1 ") && label.contains("Q2 ")); - assert!(label.contains("acc"), "{label}"); - } - - /// A state reused by two query outputs is named once in the sharing suffix. - #[test] - fn connected_labels_name_actual_shared_state() { - let workload = example1(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &lower_promql_roots(&workload).unwrap(), - ) - .unwrap(), - ) - .unwrap(); - let roots = - compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); - let label = connected_candidate_label(&[roots[1].clone(), roots[1].clone()]); - assert!(label.contains("shared Sum acc (Q1, Q2)"), "{label}"); - assert_eq!(label.matches("shared Sum acc").count(), 1); - } - - /// Raw logical operators have an exact label, rather than an empty choice label. - #[test] - fn connected_labels_name_exact_results() { - let roots = lower_promql_roots(&example1()).unwrap(); - assert_eq!(connected_candidate_label(&roots), "Q1 exact · Q2 exact"); - } -} +mod tests {} diff --git a/crates/devtools/src/lib.rs b/crates/devtools/src/lib.rs index 6fe2ec83f..2b1e21f71 100644 --- a/crates/devtools/src/lib.rs +++ b/crates/devtools/src/lib.rs @@ -39,7 +39,6 @@ pub use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError}; /// default. pub fn lower_promql_with_sample_interval( query: &str, - accuracy: asap_types::ir::properties::AccuracyTarget, interval_ms: u64, ) -> Result, PromqlError> { use asap_types::workload::{ @@ -52,10 +51,7 @@ pub fn lower_promql_with_sample_interval( language: asap_types::workload::QueryLanguage::PromQL, query_batch: Some(vec![BatchEntry { query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, + requirements: QueryRequirements::default(), predictability: Predictability::Unknown, invocations: 1, execute_at: None, diff --git a/crates/devtools/src/planner_fuzz/README.md b/crates/devtools/src/planner_fuzz/README.md index 044dbf232..56ebb9825 100644 --- a/crates/devtools/src/planner_fuzz/README.md +++ b/crates/devtools/src/planner_fuzz/README.md @@ -137,7 +137,7 @@ Two rules decide whether a check counts: | `e-other-quantile-rank` | `q` at another rank | `q`'s estimate (or exact quantile aggregate) at the other rank, same input | | `f-exact-duplicate` | `q` | reading `q`'s node | | `g-outer-over-result` | an operator over `q`, by `q`'s outer operator: `topk(3, q)` (sum), `bottomk(3, q)` (count), `max(q)` (max), `quantile(0.5, q)` (avg), `q > 0` (min, quantile) | the operator's lowering grafted onto `q`'s node | -| `h-other-statistic` | SQL, ungrouped: another frequency statistic of `q`'s column (distinct count → row count, L2 → distinct count, entropy → L2) | a `SummaryEstimate` of it from `q`'s state, when its family answers it (`FamilyCapability::answers`); or `W ∪ {q'}` planned with UnivMon as the only sketch family, priced with every family | +| `h-other-statistic` | SQL, ungrouped: another frequency statistic of `q`'s column (distinct count → row count, L2 → distinct count, entropy → L2) | a `SummaryEstimate` of it from `q`'s state, when its family answers it (`SummaryTypeCapability::answers`); or `W ∪ {q'}` planned with UnivMon as the only sketch family, priced with every family | | `i-other-repeat-interval` | `q` at a longer repeat interval | reading `q`'s node | | `j-smaller-topk` | `q`'s top-k at k = 3 | `q`'s `Limit` / `TopK` at 3 over the same input | | `k-union-of-selections` | `q` over `label =~ "v1\|v2"`, with `q`'s `v2` twin added to `W` | `SummaryMerge` of the two states, then `q`'s estimate | @@ -204,7 +204,7 @@ status column, "conclusive" means the check either held or was violated. | Another quantile rank from the same sketch | planner-layering.md L631–634, L804–817 | `e-other-quantile-rank` | VIOLATED (39 of 465), exact percentiles only: new draft (unshared exact aggregates) | | Identical expressions (common-subexpression elimination) | planner-layering.md rule table L583–585, Algorithm 2 L655–658 | `f-exact-duplicate` | VIOLATED (92 of 2,915), exact query-time aggregates: new draft | | A query over another query's result (top-k over a shared sum) | joint-planning-search.md L177–187; planner-layering.md L583–585 | `g-outer-over-result` | holds (552 conclusive checks). With the operator following q's (ASAPWorkloads #4, #12, #17, #18): VIOLATED (6 of 49), `topk(3, …)` over an exact instant sum and `… > 0` over an approximate quantile | -| Summary capability: another statistic the family answers (`FamilyCapability::answers`) | planner-layering.md rule table L586, L611–617, Example 2 L1306–1370 | `h-other-statistic` | KLL and KMV answer one statistic, so h now checks SQL frequency statistics of a high-cardinality column (planner-layering Example 2; ASAPWorkloads #57, #107–#108): holds (5 conclusive checks) | +| Summary capability: another statistic the family answers (`SummaryTypeCapability::answers`) | planner-layering.md rule table L586, L611–617, Example 2 L1306–1370 | `h-other-statistic` | KLL and KMV answer one statistic, so h now checks SQL frequency statistics of a high-cardinality column (planner-layering Example 2; ASAPWorkloads #57, #107–#108): holds (5 conclusive checks) | | gcd tumbling-window grid across repeat intervals | planner-layering.md L791–809 | `i-other-repeat-interval` | VIOLATED (49 of 2,717): new draft; compare #731 | | Exact top-k reuse (smaller k) | planner-layering.md rule table L589; joint-planning-search.md L325 | `j-smaller-topk` | holds (62 conclusive checks) | | Merge of disjoint selections | asap-primitive-schema.md §4.2 L194–232, §5.6 Example B L721–752 | `k-union-of-selections` | never conclusive: Stage 1 composes no guarantee for the merged estimate | diff --git a/crates/devtools/src/planner_fuzz/generate.rs b/crates/devtools/src/planner_fuzz/generate.rs index d831c9894..b76168078 100644 --- a/crates/devtools/src/planner_fuzz/generate.rs +++ b/crates/devtools/src/planner_fuzz/generate.rs @@ -376,7 +376,7 @@ pub fn source_facts( let mut facts = json!({ "series_count": series_count, "sample_interval": sample_interval_ms, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "distinct_values_per_label": distinct, }); if let Some(metric_type) = metric_type { diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index da1b33e5c..eee772af7 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -437,21 +437,12 @@ impl Planner { ), } .map_err(|e| e.to_string())?; - let selection = run + let (_plan, _selection) = run .run .selection .as_ref() .map_err(|e| format!("selection: {e}"))?; - let cost_per_second = selection - .cost - .cost_per_second - .ok_or("no cost per second (cost model 3)")?; - Ok(Planned { - selected: "selected".into(), - cost_per_second, - stage0_roots: roots, - logical_roots: run.run.selected_plan.as_ref().cloned(), - }) + todo!("#1207 PR 9: the selected plan's cost (plan_cost) and roots from its PhysicalASAPDAG") } /// Stage 2 and Stage 3 over the logical plan `roots` of `workload`'s diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index f1ede01a7..ddc2b2aaf 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -17,8 +17,8 @@ use serde_json::{json, Value}; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::family_capability::FamilyCapability; use asap_types::ir::schema::sketch::{SketchAlgorithm, SketchStatistic}; +use asap_types::ir::schema::summary_type_capability::SummaryTypeCapability; use asap_types::ir::schema::{DataType, FieldDataType}; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, QueryRoot, @@ -990,7 +990,14 @@ fn with_root_estimate( fn state_family(state: &Rc) -> Result { match state.summary_build() { Some( - ASAPOp::SummaryAgg { family, .. } | ASAPOp::SummaryTumblingWindowChain { family, .. }, + ASAPOp::SummaryAgg { + summary_type: family, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: family, + .. + }, ) => Ok(family), _ => Err("q's estimate reads no single summary build".into()), } @@ -1118,7 +1125,7 @@ pub fn witness_root( Witness::SketchRollup { labels } => { let (state, query) = root_estimate(q_root)?; let family = state_family(&state)?; - if !FamilyCapability::merges_groups(&family) { + if !SummaryTypeCapability::merges_groups(&family) { return Err(format!("q's family {family:?} does not merge groups")); } let group_by = labels @@ -1192,7 +1199,7 @@ pub fn witness_root( Witness::OtherStatistic { statistic } => { let answers = |state: &Rc| -> Result<(), String> { let family = state_family(state)?; - if FamilyCapability::of(&family).answers(statistic) { + if SummaryTypeCapability::of(&family).answers(statistic) { Ok(()) } else { Err(format!( @@ -1242,7 +1249,7 @@ pub fn witness_root( return Err("both selections read one state".into()); } let family = state_family(&state)?; - if FamilyCapability::of(&family).merge_inputs.is_none() { + if SummaryTypeCapability::of(&family).merge_inputs.is_none() { return Err(format!("q's family {family:?} has no merge")); } // The merge's coverage check refuses states of different diff --git a/crates/devtools/src/planner_fuzz/report.rs b/crates/devtools/src/planner_fuzz/report.rs index 330c8de94..703543743 100644 --- a/crates/devtools/src/planner_fuzz/report.rs +++ b/crates/devtools/src/planner_fuzz/report.rs @@ -81,7 +81,7 @@ pub fn documented_patterns() -> Vec { note: "the operator follows q's own: topk, bottomk, max, quantile(0.5) or a threshold `> 0` (ASAPWorkloads #4, #12, #17, #18)", }, DocumentedPattern { - pattern: "Summary capability: another statistic the family answers from one build (FamilyCapability::answers)", + pattern: "Summary capability: another statistic the family answers from one build (SummaryTypeCapability::answers)", doc_reference: "planner-layering.md Pass 2 rule table L586, examples L611–617, Example 2 L1306–1370", relation: Some(Relation::OtherStatistic), note: "UnivMon's count, distinct count, L2 and entropy (ASAPWorkloads #57, #107–#108); the witness reads q's own state, or is W ∪ {q'} planned with UnivMon as the only family", diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 5c5c65dbf..dbf051f9a 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -7,23 +7,17 @@ use asap_logical_asap_optimization::{ memo_of_queries, pass1_memo, pass2_after_local_window_composition, pass2_local_window_composition, Pass1Error, }; -use asap_physical_asap_optimization::{ - generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, -}; +use asap_physical_asap_optimization::generate_physical_asap_memo_candidates; use asap_plan_selection::select_plan; use asap_planner::{ run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError, }; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::physical::PhysicalWorkloadPlan; -use asap_types::stages::{AssignmentRealizer, Memo}; -use asap_types::workload::bounds::with_confirmed_weights_at; +use asap_types::ir::QueryRoot; +use asap_types::stages::Memo; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; use serde_json::Value; -use std::collections::HashSet; -use std::rc::Rc; /// The last stage a run runs (`asap_planner --run-until`); the run's /// fields for later stages are left empty. @@ -83,43 +77,6 @@ pub struct PlanningStagesRun { pub run: PlanningRun, } -/// A logical candidate the document shows: its id and composed roots, or -/// why they do not compose. -#[derive(Debug, Clone)] -pub struct ShownLogicalCandidate { - pub id: String, - pub roots: Result, String>, -} - -impl PlanningStagesRun { - /// The logical candidates the document shows: the selected plan (`L1`; - /// its roots as Stage 2 built them, with Stage 3's weight proofs). - pub fn logical_candidates(&self) -> Vec { - self.run - .selected_plan - .iter() - .map(|plan| ShownLogicalCandidate { - id: "L1".into(), - roots: Ok(plan.clone()), - }) - .collect() - } - - /// The physical candidates the document shows: the selected plan as - /// Stage 2 built it (`P1`, from `L1`). - pub fn physical_candidates(&self) -> Vec { - self.run - .selected_physical_plan - .iter() - .map(|realized| PhysicalWorkloadPlan { - id: "P1".into(), - from_logical: "L1".into(), - ..realized.plan.clone() - }) - .collect() - } -} - /// Lower a PromQL workload, each series' identity as a column (the row /// representation per-series state needs at runtime). pub fn lower_promql_roots(workload: &PlanningWorkload) -> Result, String> { @@ -157,7 +114,6 @@ pub fn lower_sql_roots( &entry.query.0, catalog, dialect.clone(), - entry.requirements.accuracy.clone(), )) .map(QueryRoot::Operator) .map_err(|e| format!("lowering: {e}")) @@ -242,9 +198,6 @@ pub fn run_search_over( validate(&roots)?; let run = run_planning(roots.clone(), entries, data, models, settings) .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; - if let Some(plan) = &run.selected_plan { - validate(plan)?; - } Ok(PlanningStagesRun { run_until: RunUntil::Stage3, stage0_memo_groups: None, @@ -266,8 +219,16 @@ pub fn run_stages_until( until: RunUntil, ) -> Result { validate(&roots)?; - let stage0_dag = - memo_of_queries(&roots).map_err(|e| PlanningStagesError::Validation(e.to_string()))?; + // As `run_planning`: each query with its accuracy target, or none when + // accuracy is not enforced. + let accuracy_targets: Vec<_> = entries + .iter() + .map(|entry| { + (!models.ignore_accuracy_guarantees).then(|| entry.requirements.accuracy.clone()) + }) + .collect(); + let stage0_dag = memo_of_queries(&roots, &accuracy_targets, data) + .map_err(|e| PlanningStagesError::Validation(e.to_string()))?; let mut stages = PlanningStagesRun { run_until: until, stage0_memo_groups: Some(stage0_dag.clone()), @@ -278,8 +239,6 @@ pub fn run_stages_until( logical_asap_candidates: None, physical_asap_candidates: None, selection: Err(SelectionError::NotRun), - selected_physical_plan: None, - selected_plan: None, report: PlanningReport::default(), }, }; @@ -287,17 +246,8 @@ pub fn run_stages_until( return Ok(stages); } let run = &mut stages.run; - let accuracy_targets: Vec<_> = entries - .iter() - .map(|entry| entry.requirements.accuracy.clone()) - .collect(); - let accuracy = models.accuracy_check_inputs(); let pass1 = pass1_memo( stage0_dag, - &accuracy_targets, - accuracy.model, - accuracy.evidence, - data, models .deployment_capabilities .supported_summaries @@ -331,7 +281,7 @@ pub fn run_stages_until( } }; stages.pass2_local_window_composition = Some(composed.clone()); - let logical = match pass2_after_local_window_composition(composed) { + let logical = match pass2_after_local_window_composition(composed, models.evidence) { Ok(logical) => logical, Err(error) => { run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); @@ -350,44 +300,8 @@ pub fn run_stages_until( run.physical_asap_candidates = Some(physical); return Ok(stages); } - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries, - data, - query_time_results_retained: models.deployment_capabilities.query_time_results_retained, - }; - match select_plan( - &physical, - entries, - data, - &accuracy, - models.cost_inputs, - &realizer, - ) { - Ok(selection) => { - match realizer.realize(&physical, &selection.selected_dag) { - Ok(realized) => { - // As the facade does: the plan to run carries the weight - // proofs Stage 3 made. - let proven: HashSet<*const OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - let plan = with_confirmed_weights_at(&realized.plan.roots, data, &|node| { - proven.contains(&Rc::as_ptr(node)) - }); - validate(&plan)?; - run.selected_plan = Some(plan); - run.selected_physical_plan = Some(realized); - run.selection = Ok(selection); - } - Err(error) => run.selection = Err(SelectionError::Realization(error)), - } - } - Err(error) => run.selection = Err(SelectionError::Selection(error)), - } + run.selection = select_plan(&physical, entries, data, models.cost_inputs) + .map_err(SelectionError::Selection); run.physical_asap_candidates = Some(physical); Ok(stages) } diff --git a/crates/devtools/src/rqe_atomic_costs.rs b/crates/devtools/src/rqe_atomic_costs.rs index 05681f699..9c7195e0e 100644 --- a/crates/devtools/src/rqe_atomic_costs.rs +++ b/crates/devtools/src/rqe_atomic_costs.rs @@ -135,10 +135,7 @@ fn sketch_params(sketch_name: &str, params: &Value) -> Result Result 0.0)); @@ -763,7 +762,7 @@ fn a_quantile_target_without_a_metric_accepts_rank_or_relative() { }]}, // Relative DDS interpolation requires this explicit domain proof. "data_workload": {"arrival": "unknown", "sources": {"m": { - "sample_interval": 1000, "series_count": 1, "values_non_negative": true + "sample_interval": 1000, "series_count": 1, "column_signs": {"value": "non_negative"} }}} }); document(&workload, &["--run-until", "stage3"]) @@ -1025,7 +1024,7 @@ fn milp_lower_bound_is_consistent_with_the_selected_cost() { "sources": {"data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5, }}, }, diff --git a/crates/devtools/tests/cross_language.rs b/crates/devtools/tests/cross_language.rs index 3309dda27..4d28c0b8d 100644 --- a/crates/devtools/tests/cross_language.rs +++ b/crates/devtools/tests/cross_language.rs @@ -15,7 +15,6 @@ use asap_devtools::{lower_promql_with_sample_interval, lower_sql, SqlCatalog}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use std::rc::Rc; @@ -42,13 +41,13 @@ fn catalog() -> SqlCatalog { } async fn sql(q: &str) -> Rc { - lower_sql(q, &catalog(), AccuracyTarget::exact()) + lower_sql(q, &catalog()) .await .unwrap_or_else(|e| panic!("SQL {q:?} failed to lower: {e:?}")) } fn promql(q: &str) -> Rc { - lower_promql_with_sample_interval(q, AccuracyTarget::exact(), 1_000) + lower_promql_with_sample_interval(q, 1_000) .unwrap_or_else(|e| panic!("PromQL {q:?} failed to lower: {e:?}")) } @@ -75,8 +74,7 @@ fn heavy_hitter(qe: &OperatorNode) -> Option<(usize, GroupKeys)> { else { return None; }; - matches!(inner.as_slice(), [AggIntent::Count { .. }]) - .then(|| (*k, reduction.expect_reduce().clone())) + matches!(inner.as_slice(), [AggIntent::Count]).then(|| (*k, reduction.expect_reduce().clone())) } #[tokio::test] diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json index 656dabd73..1dd0566c5 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-grouped.json @@ -32,7 +32,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json index 65bc2ffbe..0455fd361 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/quantile-single.json @@ -32,7 +32,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json index 33c2c2a3c..d0006c2da 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-long-w01-q3.json @@ -34,7 +34,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json index 4e14096a2..1bd44805f 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w01-q3-q8.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json index 9ade0f5be..35784b296 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/random-mixed-seed-862-w02-q2-q7.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json index 85976d570..24f83d95c 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/ratio-of-quantiles-1h.json @@ -34,7 +34,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json b/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json index 43fc749f2..33bd85a34 100644 --- a/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json +++ b/crates/devtools/tests/sketch_plan_snapshots/workloads/topk-rate.json @@ -32,7 +32,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index df73a824d..1b5c97ffd 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -15,7 +15,7 @@ use asap_types::ir::operator::{ TimeShift, WindowFuncKind, }; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, ScalarValue}; -use asap_types::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; +use asap_types::ir::schema::sketch::{SketchStatistic, SummaryUpdate}; use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::ir::{ ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, @@ -171,10 +171,9 @@ fn asap_samples() -> Vec> { let samples = vec![ ASAPOp::SummaryAgg { child: (), - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, }, ASAPOp::SummaryEstimate { @@ -204,10 +203,9 @@ fn asap_samples() -> Vec> { }, ASAPOp::SummaryTumblingWindowChain { child: (), - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: 60_000, }, @@ -223,7 +221,7 @@ fn asap_samples() -> Vec> { outer: (), inner: (), key: 0, - family, + summary_type: family, }, ASAPOp::Extension { child: (), @@ -343,7 +341,6 @@ fn viewer_knows_every_view_refresh_and_rejection_cause() { let causes = [ C::AccuracyTargetMissed, - C::CountMinWeightsNotProvenNonNegative, C::NoAccuracyGuarantee, C::LatencyBoundExceeded, C::LatencyOverRepeatInterval, @@ -358,7 +355,6 @@ fn viewer_knows_every_view_refresh_and_rejection_cause() { for cause in causes { match cause { C::AccuracyTargetMissed - | C::CountMinWeightsNotProvenNonNegative | C::NoAccuracyGuarantee | C::LatencyBoundExceeded | C::LatencyOverRepeatInterval diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index 00aa39b12..279139cfb 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -11,8 +11,8 @@ use crate::Error; use planner_types::ir::schema::{ - ExactKind, ExactParams, FamilyCapability, FieldDataType as SummaryFamilyType, GroupingStrategy, - SketchAlgorithm, SketchStatistic, StatisticKind, SummaryUpdate, + ExactKind, ExactParams, FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchStatistic, + StatisticKind, SummaryTypeCapability, SummaryUpdate, }; /// Check the same contract used by `create_planner_accumulator` before a plan @@ -20,10 +20,9 @@ use planner_types::ir::schema::{ pub fn validate_summary_kernel( family: &SummaryFamilyType, input: &SummaryUpdate, - grouping: &GroupingStrategy, ) -> Result<(), String> { - if grouping != &GroupingStrategy::PerSubpopulationInstance { - return validate_hydra_cms(family, input, grouping); + if SummaryTypeCapability::shares_one_state_across_groups(family) { + return validate_hydra_cms(family, input); } let keyed = match family { SummaryFamilyType::ExactAggregate(kind, params) => { @@ -42,11 +41,7 @@ pub fn validate_summary_kernel( } input.item.is_some() } - SummaryFamilyType::Sketch(kind, layout) => { - if layout != grouping { - return Err("Planner family and operator grouping disagree".into()); - } - let _ = kind; + SummaryFamilyType::Sketch(_) => { todo!("#1207 PR 6: check θ, which the exported node carries") } _ => return Err(format!("unsupported summary kernel {family:?}")), @@ -57,20 +52,13 @@ pub fn validate_summary_kernel( Ok(()) } -/// The only shared grouping with a kernel: Hydra over Count-Min. Each update +/// The only Hydra algorithm with a kernel: Hydra over Count-Min. Each update /// adds one item column's value to its group, with weight 1 (a row count, /// `UnitCount`), a positive integer constant (`PositiveConstant`) or a weight /// column proven non-negative (a Count-Min point count only overestimates /// when no weight is negative). -fn validate_hydra_cms( - family: &SummaryFamilyType, - input: &SummaryUpdate, - grouping: &GroupingStrategy, -) -> Result<(), String> { +fn validate_hydra_cms(family: &SummaryFamilyType, input: &SummaryUpdate) -> Result<(), String> { use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; - if !matches!(family, SummaryFamilyType::Sketch(_, layout) if layout == grouping) { - return Err("Planner family and operator grouping disagree".into()); - } hydra_cms_shape(family)?; if !matches!(input.item, Some(SummaryInputExpr::Column(_))) { return Err("HydraCms requires one item column".into()); @@ -113,25 +101,13 @@ pub(crate) fn constant_row_weight(weight: f64) -> Result Result<(usize, usize, usize, usize), String> { - use planner_types::ir::schema::HydraKind; - let SummaryFamilyType::Sketch( - kind, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - }, - ) = family - else { - return Err("a shared-grouping (Hydra) family has no registered kernel".into()); + let SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms) = family else { + return Err("a Hydra algorithm other than HydraCms has no registered kernel".into()); }; - if kind != &SketchAlgorithm::Cms { - return Err("HydraCms family must be Count-Min with the Hydra width and depth".into()); - } todo!("#1207 PR 6: the Hydra grid's sizes are θ, which the exported node carries") } @@ -186,14 +162,14 @@ pub(crate) fn unit_frequency_column( pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { use planner_types::ir::schema::SketchAlgorithm as A; - if let SummaryFamilyType::Sketch(kind, grouping) = family { + if let SummaryFamilyType::Sketch(kind) = family { // Plain Count-Min is native as stored state only: it merges and reads // its bare count, but the DAG does not build it from rows. HydraCms // is built by the all-groups summary build. + if let A::HydraCms = kind { + return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); + } if let A::Cms = kind { - if grouping != &GroupingStrategy::PerSubpopulationInstance { - return hydra_cms_shape(family).map(|_| ()).map_err(Error::Invalid); - } todo!("#1207 PR 6: check the Count-Min θ, which the exported node carries") } if matches!(kind, A::CmsWithHeap | A::CountSketchWithHeap) { @@ -202,7 +178,7 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { } match family { SummaryFamilyType::ExactAggregate(..) => {} - SummaryFamilyType::Sketch(A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon, _) => {} + SummaryFamilyType::Sketch(A::Kll | A::DDSketch | A::Hll | A::Kmv | A::UnivMon) => {} _ => { return Err(Error::Invalid( "summary family has no native DAG state implementation".into(), @@ -214,7 +190,6 @@ pub fn validate_native_family(family: &SummaryFamilyType) -> Result<(), Error> { &planner_types::ir::schema::SummaryUpdate::column( planner_types::ir::scalar::ColumnRef::SampleValue, ), - &Default::default(), ) .map_err(Error::Invalid) } @@ -248,16 +223,11 @@ pub(crate) const EVALUATED_READOUTS: &[(SketchAlgorithm, &[StatisticKind])] = &[ /// Whether the executor reads `readout` from a state of `family` whose /// family-capability table row answers it. fn evaluates(family: &SummaryFamilyType, readout: StatisticKind) -> bool { - use planner_types::ir::schema::HydraKind; match family { - SummaryFamilyType::Sketch(kind, GroupingStrategy::PerSubpopulationInstance) => { - EVALUATED_READOUTS - .iter() - .any(|(family, readouts)| family == kind && readouts.contains(&readout)) - } - SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { - matches!(kind, HydraKind::HydraCms | HydraKind::HydraKll) - } + SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms | SketchAlgorithm::HydraKll) => true, + SummaryFamilyType::Sketch(kind) => EVALUATED_READOUTS + .iter() + .any(|(family, readouts)| family == kind && readouts.contains(&readout)), _ => false, } } @@ -275,16 +245,7 @@ pub fn validate_sketch_evaluation( // HydraCms answers a group's item frequency. A group's total needs a // group-mass query asap_sketchlib's Hydra lacks. if readout == StatisticKind::Total - && matches!( - family, - SummaryFamilyType::Sketch( - _, - GroupingStrategy::SharedMultiSubpopulation { - kind: planner_types::ir::schema::HydraKind::HydraCms, - .. - } - ) - ) + && matches!(family, SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms)) { return Err(Error::Invalid( "HydraCms does not answer a group total: asap_sketchlib's Hydra has no group-mass \ @@ -293,7 +254,7 @@ pub fn validate_sketch_evaluation( )); } if readout == StatisticKind::TopK - || !FamilyCapability::of(family).answers(query) + || !SummaryTypeCapability::of(family).answers(query) || !evaluates(family, readout) { return Err(Error::Invalid( diff --git a/crates/executor/src/operators/aggregate/mod.rs b/crates/executor/src/operators/aggregate/mod.rs index 0feb88ddc..e914f171e 100644 --- a/crates/executor/src/operators/aggregate/mod.rs +++ b/crates/executor/src/operators/aggregate/mod.rs @@ -155,7 +155,7 @@ impl Operator { | AggIntent::Deriv | AggIntent::PredictLinear { .. } | AggIntent::Increase - | AggIntent::Count { .. } + | AggIntent::Count | AggIntent::Sum { col: None } | AggIntent::Avg { col: None } | AggIntent::Min { col: None } @@ -187,7 +187,7 @@ impl Operator { .collect::>(); fields.push(result_field( "value", - if matches!(intent, AggIntent::Count { .. }) { + if matches!(intent, AggIntent::Count) { DataType::Int64 } else { DataType::Float64 diff --git a/crates/executor/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs index 0ea8cedfd..b913fad41 100644 --- a/crates/executor/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -122,7 +122,7 @@ pub(in crate::operators) fn window_value( .map(|v| Value::Float64(v * (end as f64 - start as f64) / 1000.)), AggIntent::Increase => rate(points, start, end, true) .map(|v| Value::Float64(v * (end as f64 - start as f64) / 1000.)), - AggIntent::Count { .. } => Some(Value::Int64( + AggIntent::Count => Some(Value::Int64( i64::try_from(points.len()).map_err(|_| Error::Invalid("count overflow".into()))?, )), AggIntent::Sum { .. } | AggIntent::Avg { .. } => { @@ -326,7 +326,7 @@ mod tests { values::Batch, }; use futures::StreamExt; - use planner_types::ir::properties::AccuracyTarget; + use planner_types::ir::schema::{ DataType, Field as SummaryField, FieldDataType as SummaryFamilyType, Schema, }; @@ -366,12 +366,7 @@ mod tests { for (intent, expected) in [ (AggIntent::Rate, Value::Float64(2.)), (AggIntent::Increase, Value::Float64(4.)), - ( - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - Value::Int64(3), - ), + (AggIntent::Count, Value::Int64(3)), ] { let batch = Batch::try_new( schema.clone(), diff --git a/crates/executor/src/operators/series_window.rs b/crates/executor/src/operators/series_window.rs index 6d81f48df..b4452438d 100644 --- a/crates/executor/src/operators/series_window.rs +++ b/crates/executor/src/operators/series_window.rs @@ -98,7 +98,7 @@ impl Operator { | AggIntent::PredictLinear { .. } | AggIntent::Increase | AggIntent::Delta - | AggIntent::Count { .. } + | AggIntent::Count | AggIntent::Sum { col: None } | AggIntent::Avg { col: None } | AggIntent::Min { col: None } diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 4f40f2a06..38a77dd6b 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -4,7 +4,7 @@ use crate::values::arrow_columns::ColumnDepth; use crate::values::StateColumn; use crate::StateColumnUpdater; use arrow::array::{Array, BooleanArray, Float64Array, TimestampMillisecondArray, UInt32Array}; -use planner_types::ir::schema::{FamilyCapability, MergeInputRule}; +use planner_types::ir::schema::{MergeInputRule, SummaryTypeCapability}; use std::num::NonZeroU32; /// What each input row adds to its item's count in a keyed summary build @@ -166,7 +166,6 @@ impl Operator { crate::capability::validate_summary_kernel( &family, &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), ) .map_err(Error::Invalid)?; let mut fields = groups @@ -286,7 +285,7 @@ impl Operator { /// takes Utf8, and UnivMon Utf8 and Bool. fn accepts_update_type(family: &SummaryFamilyType, dtype: &DataType) -> bool { use planner_types::ir::schema::{ExactKind, SketchAlgorithm as A}; - let SummaryFamilyType::Sketch(kind, _) = family else { + let SummaryFamilyType::Sketch(kind) = family else { return *dtype == DataType::Float64 || matches!( family, @@ -329,7 +328,7 @@ impl Operator { /// output is Int64 and whose updates have unit weight; execution rejects a /// non-integral total rather than rounding it. fn integral_count(family: &SummaryFamilyType, query: &SummaryEvaluation) -> bool { - matches!(family, SummaryFamilyType::Sketch(kind, _) + matches!(family, SummaryFamilyType::Sketch(kind) if kind == &planner_types::ir::schema::SketchAlgorithm::Cms) && matches!( query, @@ -846,7 +845,7 @@ async fn merge_summary( }; // Counter states join consecutive windows pairwise, which is exact only // in time order: order the inputs by their window start. - if FamilyCapability::of(family).merge_inputs == Some(MergeInputRule::DisjointInTime) { + if SummaryTypeCapability::of(family).merge_inputs == Some(MergeInputRule::DisjointInTime) { entries.sort_by_cached_key(|&entry| { let (states, index) = state_of(entry); match states.storage() { diff --git a/crates/executor/src/physical_planner/compiled.rs b/crates/executor/src/physical_planner/compiled.rs index 6e884928b..d3013c61a 100644 --- a/crates/executor/src/physical_planner/compiled.rs +++ b/crates/executor/src/physical_planner/compiled.rs @@ -24,7 +24,7 @@ pub(super) struct MaterializedViewSlot { pub(super) view: MaterializedView, /// The view node's schema. pub(super) schema: SchemaRef, - /// The data its state summarizes ([`PhysicalASAPDAG::view_window`]). + /// The data its state summarizes (`crate::sources::view_windows`). pub(super) window: ScanWindow, } @@ -37,7 +37,7 @@ pub(super) struct RawDataSlot { pub(super) schema: SchemaRef, /// The `Scan`'s predicates, compiled over `schema`. pub(super) predicates: Vec, - /// The rows its readers read ([`PhysicalASAPDAG::scan_window`]). + /// The rows its readers read (`crate::sources::scan_windows`). pub(super) window: ScanWindow, } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 5c527992f..d7369f00f 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -3,6 +3,7 @@ //! A node without a native lowering fails compilation; the deployment runtime //! must execute it. use crate::operators::{RowWeight, SummaryEvaluation}; +use crate::sources::{scan_windows, time_range_window, view_windows, ScanWindow}; use crate::summary_kernels::exact::ExactEvaluation; use crate::{ operators::{Expression, Operator, Reduction, SortKey}, @@ -15,12 +16,12 @@ use planner_types::ir::operator::{ agg_has_exact_per_entity_kernel, AggIntent, GroupKeys, Reduction as PlannerReduction, }; use planner_types::ir::physical_export::{ - time_range_window, PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, - PhysicalASAPOperatorPayload as Payload, ScanWindow, + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPNodeId, + PhysicalASAPOperatorPayload as Payload, }; use planner_types::ir::scalar::{ColumnRef, CompareOpKind}; use planner_types::ir::schema::{ - DataType, FamilyCapability, FieldDataType, SketchStatistic, SummaryInputExpr, + DataType, FieldDataType, SketchStatistic, SummaryInputExpr, SummaryTypeCapability, }; // `promql_rows` reads `OperatorNode` through `super::*` only in its // test-support helpers. @@ -159,13 +160,13 @@ fn compile_internal( } // Each view's and each `Scan`'s window, the DAG indexed once for all: // a materialized tumbling-window chain has a view per tumbling window. - let view_windows = dag.view_windows(views.iter().map(|view| view.node_id)); - let scan_windows = dag.scan_windows( + let view_windows = view_windows(dag, views.iter().map(|view| view.node_id)); + let scan_windows = scan_windows( + dag, dag.nodes .iter() .filter(|node| matches!(node.payload, Payload::NonASAP(NonASAPOp::Scan { .. }))) .map(|node| node.id), - &Default::default(), ); let compile_roots: BTreeSet = roots.iter().copied().collect(); let mut physical_dag = CompiledPhysicalDAG::new(roots.to_vec()); @@ -377,7 +378,7 @@ fn compile_internal( ))); } let reduction = match measure { - AggIntent::Count { .. } => Reduction::Count, + AggIntent::Count => Reduction::Count, AggIntent::Sum { .. } => Reduction::Sum(value), AggIntent::Avg { .. } => Reduction::Avg(value), AggIntent::Min { .. } => Reduction::Min(value), @@ -448,10 +449,9 @@ fn compile_internal( // A closed row must include either all source labels or the explicit // complete-label identity. Projected labels alone are insufficient. if let Payload::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input: update, reduction: PlannerReduction::PerEntity, - grouping, filter: None, .. }) = &node.payload @@ -479,7 +479,7 @@ fn compile_internal( "per-entity summary requires complete source identity", )); } - crate::capability::validate_summary_kernel(family, update, grouping) + crate::capability::validate_summary_kernel(family, update) .map_err(Error::Invalid)?; let value = match (frequency_item, &update.weight) { (Some(item), _) => item, @@ -879,7 +879,7 @@ fn temporal_evaluation_drops_name(node: &PhysicalASAPDAGNode) -> bool { /// one only when the family-capability table gives the family a sound merge. pub(crate) fn require_sound_merge(input: &SchemaRef) -> Result<(), Error> { let state = &input.fields[summary_column(input)?].dtype; - if FamilyCapability::of(state).merge_inputs.is_none() { + if SummaryTypeCapability::of(state).merge_inputs.is_none() { return Err(invalid(format!( "summary merge over {state:?} is unsupported: the family has no sound merge" ))); @@ -1137,7 +1137,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Reduction::Count, + AggIntent::Count => Reduction::Count, AggIntent::Cardinality { cols, .. } => { Reduction::Cardinality(if cols.is_empty() { vec![column(None)?] @@ -1200,10 +1200,9 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result Err(invalid("value operation has no native implementation")), }, Payload::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input: update, reduction, - grouping, filter, .. }) => { @@ -1213,7 +1212,7 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result build.with_row_filter(expression(&filter.0, input)?), None => Ok(build), @@ -1228,9 +1227,9 @@ fn bind_operation(node: &PhysicalASAPDAGNode, inputs: &[SchemaRef]) -> Result { let family = &input.fields[state].dtype; - if !FamilyCapability::merges_groups(family) { + if !SummaryTypeCapability::merges_groups(family) { return Err(invalid(format!( - "a rollup of {family:?} states: the family cannot merge different groups" + "a rollup of {family:?} states: the summary type cannot merge different groups" ))); } keys.clone() @@ -1282,11 +1281,12 @@ fn summary_agg_build( family: &FieldDataType, update: &planner_types::ir::schema::SummaryUpdate, reduction: &PlannerReduction, - grouping: &planner_types::ir::schema::GroupingStrategy, ) -> Result { - if grouping != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance { - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; + if matches!( + family, + FieldDataType::Sketch(planner_types::ir::schema::SketchAlgorithm::HydraCms) + ) { + crate::capability::validate_summary_kernel(family, update).map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("an all-groups summary requires explicit groups")); }; @@ -1309,8 +1309,7 @@ fn summary_agg_build( crate::capability::unit_frequency_column(update), unkeyed_frequency_family(family), ) { - crate::capability::validate_summary_kernel(family, update, grouping) - .map_err(Error::Invalid)?; + crate::capability::validate_summary_kernel(family, update).map_err(Error::Invalid)?; let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid( "summary construction requires explicit grouping columns", @@ -1328,7 +1327,7 @@ fn summary_agg_build( let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); }; - if matches!(family, FieldDataType::Sketch(kind, _) if kind == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind) if kind == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) && !matches!( update.weight_domain, planner_types::ir::schema::WeightDomain::NonNegative { .. } @@ -1362,7 +1361,7 @@ fn summary_agg_build( groups(input, keys)?, ); } - crate::capability::validate_summary_kernel(family, update, grouping).map_err(Error::Invalid)?; + crate::capability::validate_summary_kernel(family, update).map_err(Error::Invalid)?; let SummaryInputExpr::Column(column) = &update.weight else { return Err(invalid( "summary update expression must be projected to a column", @@ -1397,7 +1396,7 @@ fn row_weight(input: &SchemaRef, weight: &SummaryInputExpr) -> Result bool { - matches!(family, FieldDataType::Sketch(kind, _) if matches!( + matches!(family, FieldDataType::Sketch(kind) if matches!( kind, planner_types::ir::schema::SketchAlgorithm::UnivMon | planner_types::ir::schema::SketchAlgorithm::Hll diff --git a/crates/executor/src/sources/mod.rs b/crates/executor/src/sources/mod.rs index d9ff969a9..d3b8aba3d 100644 --- a/crates/executor/src/sources/mod.rs +++ b/crates/executor/src/sources/mod.rs @@ -12,13 +12,14 @@ use crate::{ use futures::{stream, StreamExt}; use planner_types::ir::materialized_view::MaterializedView; use planner_types::ir::operator::Source; -use planner_types::ir::physical_export::ScanWindow; use planner_types::ir::schema::DataType; use planner_types::ir::ScalarExpr; use std::sync::Arc; mod data_store; pub use data_store::{DataStore, MaterializedViewRequest, RawDataRequest}; +mod window; +pub(crate) use window::{scan_windows, time_range_window, view_windows, ScanWindow}; mod memory; pub use memory::InMemoryDataStore; diff --git a/crates/executor/src/sources/window.rs b/crates/executor/src/sources/window.rs new file mode 100644 index 000000000..5bc616a5a --- /dev/null +++ b/crates/executor/src/sources/window.rs @@ -0,0 +1,195 @@ +//! The rows a slot of a compiled DAG reads, back from the evaluation time: +//! the executor derives each `Scan`'s and each materialized view's window +//! from the `TimeRange` and `TimeShift` nodes above or below it, and pushes +//! it into the data-store read. + +use std::collections::{HashMap, HashSet}; + +use planner_types::ir::operator::asap::ASAPOp; +use planner_types::ir::operator::node::Operator; +use planner_types::ir::operator::non_asap::NonASAPOp; +use planner_types::ir::physical_export::{ + PhysicalASAPDAG, PhysicalASAPNodeId, PhysicalASAPOperatorPayload, +}; + +/// Which rows a slot reads, back from an evaluation time `T`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +pub(crate) enum ScanWindow { + /// The rows with time in `(T - end_offset_ms - width_ms, T - + /// end_offset_ms]`. + Within { end_offset_ms: i64, width_ms: i64 }, + /// Rows at any time. + Unbounded, +} + +impl ScanWindow { + /// `(start, end]` at evaluation time `evaluation_time_ms`; `None` when + /// unbounded. + pub(crate) fn bounds_ms(self, evaluation_time_ms: i64) -> Option<(i64, i64)> { + match self { + Self::Within { + end_offset_ms, + width_ms, + } => { + let end = evaluation_time_ms.saturating_sub(end_offset_ms); + Some((end.saturating_sub(width_ms), end)) + } + Self::Unbounded => None, + } + } + + /// The smallest window holding both. + fn union(self, other: Self) -> Self { + let ( + Self::Within { + end_offset_ms: a_end, + width_ms: a_width, + }, + Self::Within { + end_offset_ms: b_end, + width_ms: b_width, + }, + ) = (self, other) + else { + return Self::Unbounded; + }; + let end_offset_ms = a_end.min(b_end); + let start_back_ms = a_end + .saturating_add(a_width) + .max(b_end.saturating_add(b_width)); + Self::Within { + end_offset_ms, + width_ms: start_back_ms.saturating_sub(end_offset_ms), + } + } +} + +/// The rows the `TimeRange` `time_range` keeps, back from the evaluation +/// time `T`: `(T - s - w, T - s]` for a `TimeRange(w)` over `TimeShift`s +/// whose offsets sum to `s`. `Unbounded` when one of those shifts has an `@` +/// anchor, or `time_range` is not a `TimeRange`. +pub(crate) fn time_range_window<'a>( + time_range: PhysicalASAPNodeId, + payload_of: impl Fn(PhysicalASAPNodeId) -> Option<&'a PhysicalASAPOperatorPayload>, +) -> ScanWindow { + let Some(Operator::NonASAP(NonASAPOp::TimeRange { range, child, .. })) = payload_of(time_range) + else { + return ScanWindow::Unbounded; + }; + let mut end_offset_ms = 0i64; + let mut below = payload_of(*child); + while let Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, child })) = below { + if shift.at.is_some() { + return ScanWindow::Unbounded; + } + end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); + below = payload_of(*child); + } + ScanWindow::Within { + end_offset_ms, + width_ms: i64::try_from(range.as_millis()).unwrap_or(i64::MAX), + } +} + +fn payloads_by_id( + dag: &PhysicalASAPDAG, +) -> HashMap { + dag.nodes + .iter() + .map(|node| (node.id, &node.payload)) + .collect() +} + +/// Each of `scans`' window, by id: the union of the windows its consumers +/// read through `TimeShift`s without an `@` anchor, each a `TimeRange(w)` +/// (`(T - s - w, T - s]`). `Unbounded` when a root, any other consumer, or +/// a shift with an `@` anchor reads the `Scan`, or nothing reads it. +pub(crate) fn scan_windows( + dag: &PhysicalASAPDAG, + scans: impl IntoIterator, +) -> HashMap { + let payloads = payloads_by_id(dag); + let mut consumers: HashMap> = HashMap::new(); + for edge in &dag.edges { + consumers + .entry(edge.producer) + .or_default() + .push(edge.consumer); + } + let roots: HashSet = dag.roots.iter().copied().collect(); + let window_of = |scan: PhysicalASAPNodeId| { + let mut window: Option = None; + let mut pending = vec![(scan, 0i64)]; + while let Some((id, end_offset_ms)) = pending.pop() { + if roots.contains(&id) { + return ScanWindow::Unbounded; + } + for &consumer in consumers.get(&id).into_iter().flatten() { + let width_ms = match payloads.get(&consumer) { + Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, .. })) + if shift.at.is_none() => + { + pending.push((consumer, end_offset_ms.saturating_add(shift.offset_ms))); + continue; + } + Some(Operator::NonASAP(NonASAPOp::TimeRange { range, .. })) => { + range.as_millis() + } + Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( + "#1207 PR 6: the scan window of a chain read through a TimeRange over it" + ), + _ => return ScanWindow::Unbounded, + }; + let read = ScanWindow::Within { + end_offset_ms, + width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), + }; + window = Some(window.map_or(read, |window| window.union(read))); + } + } + window.unwrap_or(ScanWindow::Unbounded) + }; + scans + .into_iter() + .map(|scan| (scan, window_of(scan))) + .collect() +} + +/// Each of `nodes`' window, by id: the union of the windows of every +/// `TimeRange` below it. `Unbounded` when no `TimeRange` is below it or one +/// of them is unbounded (a whole SQL table, an `@` anchor). +pub(crate) fn view_windows( + dag: &PhysicalASAPDAG, + nodes: impl IntoIterator, +) -> HashMap { + let payloads = payloads_by_id(dag); + let window_of = |node: PhysicalASAPNodeId| { + let mut seen = HashSet::new(); + let mut pending = vec![node]; + let mut window: Option = None; + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + let read = match payloads.get(&id) { + Some(Operator::NonASAP(NonASAPOp::TimeRange { .. })) => { + time_range_window(id, |id| payloads.get(&id).copied()) + } + Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( + "#1207 PR 6: the window a chain read through a TimeRange over it summarizes" + ), + Some(payload) => { + pending.extend(payload.children().into_iter().copied()); + continue; + } + None => continue, + }; + window = Some(window.map_or(read, |window| window.union(read))); + } + window.unwrap_or(ScanWindow::Unbounded) + }; + nodes + .into_iter() + .map(|node| (node, window_of(node))) + .collect() +} diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs index 890a16fb8..1bd273372 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -7,7 +7,7 @@ use crate::summary_kernels::{ }; use crate::AggregateCore; use planner_types::ir::schema::{ - FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchKind, SketchParams, + FieldDataType as SummaryFamilyType, SketchAlgorithm, SketchParams, }; /// Generate the clone-based `AccumulatorUpdater` methods for updaters whose @@ -469,7 +469,6 @@ pub fn create_planner_accumulator( family: &SummaryFamilyType, parameters: Option<&SketchParams>, input: &planner_types::ir::schema::SummaryUpdate, - grouping: &planner_types::ir::schema::GroupingStrategy, ) -> Result, String> { if input.item.is_some() && matches!( @@ -483,11 +482,10 @@ pub fn create_planner_accumulator( return Err("window-weighted summaries require typed DAG binding; integer heap updaters cannot consume rates".into()); } - crate::capability::validate_summary_kernel(family, input, grouping)?; - use planner_types::ir::schema::GroupingStrategy; - if grouping != &GroupingStrategy::PerSubpopulationInstance { + crate::capability::validate_summary_kernel(family, input)?; + if matches!(family, SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms)) { return Err( - "shared grouping has one state for all groups; build it with `create_hydra_cms`".into(), + "HydraCms has one state for all groups; build it with `create_hydra_cms`".into(), ); } if matches!(family, SummaryFamilyType::ExactAggregate(..)) { @@ -498,12 +496,9 @@ pub fn create_planner_accumulator( )?, })); } - let SummaryFamilyType::Sketch(kind, family_grouping) = family else { + let SummaryFamilyType::Sketch(kind) = family else { return Err(format!("unsupported Planner summary family {family:?}")); }; - if family_grouping != grouping { - return Err("Planner family and operator grouping disagree".into()); - } let Some(params) = parameters else { return Err("a sketch summary needs its parameters θ".into()); }; @@ -532,8 +527,7 @@ pub fn create_planner_accumulator( } (SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, _) => { let (algorithm, width, depth, capacity) = - WeightedFrequency::configuration(&SketchKind::new(*kind, params.clone())) - .map_err(|e| e.to_string())?; + WeightedFrequency::configuration(kind, params).map_err(|e| e.to_string())?; Box::new( WeightedFrequencyUpdater::new(algorithm, width, depth, capacity) .map_err(|e| e.to_string())?, diff --git a/crates/executor/src/summary_kernels/hydra_cms.rs b/crates/executor/src/summary_kernels/hydra_cms.rs index 4f835d991..524b686fa 100644 --- a/crates/executor/src/summary_kernels/hydra_cms.rs +++ b/crates/executor/src/summary_kernels/hydra_cms.rs @@ -1,4 +1,4 @@ -//! Hydra over Count-Min Sketch (`HydraKind::HydraCms`) over `asap_sketchlib::Hydra`. +//! Hydra over Count-Min Sketch (`SketchAlgorithm::HydraCms`) over `asap_sketchlib::Hydra`. //! //! One `shared_rows × shared_columns` grid of Count-Min cells serves every //! group: a group hashes to one cell per row, and that cell's Count-Min diff --git a/crates/executor/src/summary_kernels/state_column_updater.rs b/crates/executor/src/summary_kernels/state_column_updater.rs index 358abed27..fb9d7b661 100644 --- a/crates/executor/src/summary_kernels/state_column_updater.rs +++ b/crates/executor/src/summary_kernels/state_column_updater.rs @@ -8,7 +8,7 @@ use super::weighted_frequency::WeightedFrequency; use crate::values::{StateColumn, Value}; use crate::{AggregateCore, Error}; -use planner_types::ir::schema::{ExactKind, FieldDataType, GroupingStrategy}; +use planner_types::ir::schema::{ExactKind, FieldDataType}; use std::sync::Arc; /// The open summary states of one family, one per group, updated one sample @@ -74,30 +74,24 @@ impl StateColumnUpdater { /// `SummaryAgg` the plan builds one state per group of: an exact /// accumulator, a value sketch (KLL, DDSketch, HLL, KMV, UnivMon), a /// keyed frequency sketch with a heap, or a HydraCms grid shared by all - /// groups. Bad input for any other family. + /// groups. Bad input for any other summary type. pub fn new(family: &FieldDataType) -> Result { crate::values::validate_family(family)?; - let hydra = matches!( - family, - FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }) - ); let states = match family { - FieldDataType::Sketch(..) if hydra => GroupStates::Hydra { - grid: Box::new(create_hydra_cms(family).map_err(Error::Invalid)?), - names: Vec::new(), - }, + FieldDataType::Sketch(planner_types::ir::schema::SketchAlgorithm::HydraCms) => { + GroupStates::Hydra { + grid: Box::new(create_hydra_cms(family).map_err(Error::Invalid)?), + names: Vec::new(), + } + } FieldDataType::ExactAggregate(..) => GroupStates::Exact { empty: ScalarState::empty(family).map_err(Error::Invalid)?, states: Vec::new(), }, - FieldDataType::Sketch(algorithm, grouping) - if grouping == &GroupingStrategy::PerSubpopulationInstance - && matches!( - algorithm, - planner_types::ir::schema::SketchAlgorithm::CmsWithHeap - | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap - ) => - { + FieldDataType::Sketch( + planner_types::ir::schema::SketchAlgorithm::CmsWithHeap + | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap, + ) => { todo!("#1207 PR 6: the keyed frequency state of θ, which the exported node carries") } FieldDataType::Sketch(..) => { diff --git a/crates/executor/src/summary_kernels/weighted_frequency.rs b/crates/executor/src/summary_kernels/weighted_frequency.rs index e0211c327..4c2aa9ef0 100644 --- a/crates/executor/src/summary_kernels/weighted_frequency.rs +++ b/crates/executor/src/summary_kernels/weighted_frequency.rs @@ -96,10 +96,11 @@ impl std::fmt::Debug for WeightedFrequency { impl WeightedFrequency { pub(crate) fn configuration( - kind: &planner_types::ir::schema::SketchKind, + algorithm: &planner_types::ir::schema::SketchAlgorithm, + params: &planner_types::ir::schema::SketchParams, ) -> Result<(FrequencyAlgorithm, usize, usize, usize), Error> { use planner_types::ir::schema::{SketchAlgorithm as A, SketchParams as P}; - let (algorithm, width, depth, capacity) = match (kind.algorithm(), kind.params()) { + let (algorithm, width, depth, capacity) = match (algorithm, params) { ( A::CmsWithHeap, P::CmsWithHeap { diff --git a/crates/executor/src/supported_summaries.rs b/crates/executor/src/supported_summaries.rs index 973ce0cfb..6aeed6b0c 100644 --- a/crates/executor/src/supported_summaries.rs +++ b/crates/executor/src/supported_summaries.rs @@ -28,8 +28,8 @@ mod tests { use planner_types::ir::operator::operator_properties::{Reduction, Source}; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{ - DataType, FamilyCapability, Field, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchStatistic, StatisticKind, SummaryInputExpr, SummaryUpdate, + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchStatistic, StatisticKind, + SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, }; use planner_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; @@ -56,7 +56,7 @@ mod tests { ]; fn per_instance(algorithm: &SketchAlgorithm) -> FieldDataType { - FieldDataType::Sketch(*algorithm, GroupingStrategy::PerSubpopulationInstance) + FieldDataType::Sketch(*algorithm) } /// A statistic of kind `readout`. @@ -84,7 +84,7 @@ mod tests { /// Whether the factory builds the per-instance `family` from rows, with /// a value column or keyed items. fn factory_builds(family: &FieldDataType) -> bool { - let FieldDataType::Sketch(_, grouping) = family else { + let FieldDataType::Sketch(_) = family else { return false; }; let value = SummaryUpdate::column(ColumnRef::SampleValue); @@ -92,8 +92,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - create_planner_accumulator(family, None, &value, grouping).is_ok() - || create_planner_accumulator(family, None, &keyed, grouping).is_ok() + create_planner_accumulator(family, None, &value).is_ok() + || create_planner_accumulator(family, None, &keyed).is_ok() } /// Whether the executor answers `readout` from `family`'s state: top-k @@ -101,7 +101,7 @@ mod tests { /// sketch evaluation, whose kernel must then read it from a built state. fn evaluation_answers(family: &FieldDataType, readout: StatisticKind) -> bool { match (readout, family) { - (StatisticKind::TopK, FieldDataType::Sketch(kind, _)) => matches!( + (StatisticKind::TopK, FieldDataType::Sketch(kind)) => matches!( kind, planner_types::ir::schema::SketchAlgorithm::CmsWithHeap | planner_types::ir::schema::SketchAlgorithm::CountSketchWithHeap @@ -119,7 +119,7 @@ mod tests { /// Whether the kernel of a per-instance `family` state built from one /// update reads `readout`. fn kernel_answers(family: &FieldDataType, readout: StatisticKind) -> bool { - let FieldDataType::Sketch(_, grouping) = family else { + let FieldDataType::Sketch(_) = family else { return false; }; let value = SummaryUpdate::column(ColumnRef::SampleValue); @@ -127,8 +127,8 @@ mod tests { item: Some(SummaryInputExpr::Column(ColumnRef::Named("item".into()))), ..value.clone() }; - let Ok(mut updater) = create_planner_accumulator(family, None, &value, grouping) - .or_else(|_| create_planner_accumulator(family, None, &keyed, grouping)) + let Ok(mut updater) = create_planner_accumulator(family, None, &value) + .or_else(|_| create_planner_accumulator(family, None, &keyed)) else { return false; }; @@ -160,10 +160,9 @@ mod tests { .unwrap(); let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: scan, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(); @@ -192,7 +191,7 @@ mod tests { .iter() .any(|s| &s.family == algorithm && s.readouts.contains(&readout)); assert_eq!(declared, executed, "{algorithm:?} {readout:?}"); - let answered = FamilyCapability::of(&family).answers(&statistic(readout)); + let answered = SummaryTypeCapability::of(&family).answers(&statistic(readout)); assert_eq!( ir_builds_estimate(&family, readout), answered, diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index 12f06cf90..b9b5144bc 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -772,7 +772,7 @@ mod tests { } fn dd_family() -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::DDSketch, Default::default()) + FieldDataType::Sketch(SketchAlgorithm::DDSketch) } // Rows sharing a state keep sharing one state; filtering rows shares the @@ -1130,7 +1130,7 @@ mod tests { let family = if exact { FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) } else { - FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) + FieldDataType::Sketch(SketchAlgorithm::Kll) }; let mut updater = StateColumnUpdater::new(&family).unwrap(); updater.ensure_groups(count as usize).unwrap(); diff --git a/crates/executor/src/values/mod.rs b/crates/executor/src/values/mod.rs index 40c273375..ccb924cc9 100644 --- a/crates/executor/src/values/mod.rs +++ b/crates/executor/src/values/mod.rs @@ -214,10 +214,7 @@ pub(super) fn validate_state( validate_family(family)?; let valid = match family { - SummaryFamilyType::Sketch(_, grouping) - if grouping - != &planner_types::ir::schema::GroupingStrategy::PerSubpopulationInstance => - { + SummaryFamilyType::Sketch(planner_types::ir::schema::SketchAlgorithm::HydraCms) => { let shape = crate::capability::hydra_cms_shape(family).map_err(Error::Invalid)?; state .as_any() @@ -289,17 +286,15 @@ mod weighted_state_tests { // A state cannot acquire a different family or shape merely by relabeling its batch. #[test] fn weighted_state_family_and_shape_must_match() { - let cms = SummaryFamilyType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); - let cs = - SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); + let cms = SummaryFamilyType::Sketch(SketchAlgorithm::CmsWithHeap); + let cs = SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap); let state = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 32, 5, 8).unwrap(); assert!(validate_state(&cs, &state).is_ok()); assert!(validate_state(&cms, &state).is_err()); let wrong_shape = WeightedFrequency::new(FrequencyAlgorithm::CountSketch, 64, 5, 8).unwrap(); assert!(validate_state(&cs, &wrong_shape).is_err()); - let even_depth = - SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); + let even_depth = SummaryFamilyType::Sketch(SketchAlgorithm::CountSketchWithHeap); assert!(validate_family(&even_depth).is_err()); } } diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index e397b15d8..956f83be8 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -222,7 +222,7 @@ fn weighted_summary_build_yields_within_a_batch() { Operator::source(input.clone(), vec![batch]).unwrap(), ) .unwrap(); - let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); let operator = Operator::per_group_keyed_summary_build( input, family, diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index a567eff37..ea0af3b3d 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -11,11 +11,11 @@ pub fn selected_dag( accuracy: planner_types::ir::properties::AccuracyTarget, ) -> Rc { let run = run_all_stages(root, accuracy); - let [QueryRoot::Operator(selected)] = run - .selected_plan - .as_deref() - .unwrap_or_else(|| panic!("a plan is selected: {:?}", run.selection)) - else { + let roots = match &run.selection { + Ok((plan, _)) => plan.query_roots().unwrap(), + Err(error) => panic!("a plan is selected: {error}"), + }; + let [QueryRoot::Operator(selected)] = roots.as_slice() else { panic!("one operator root"); }; Rc::clone(selected) @@ -29,7 +29,12 @@ pub fn computation_candidates(root: Rc) -> Vec> { compose_logical_candidate, generate_per_computation_candidates, }; let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .expect("Pass 1 accepts the query"); let mut choices = vec![vec![0; inventory.targets.len()]]; diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index 1368b006f..f53900a4b 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -172,7 +172,7 @@ fn spatial_heap_ranks_latest_values_in_independent_runs() { heap_size: 100, }, }; - let family = FieldDataType::Sketch(algorithm, Default::default()); + let family = FieldDataType::Sketch(algorithm); let build = Operator::per_group_keyed_summary_build( schema(), family, diff --git a/crates/executor/tests/current_timestamp.rs b/crates/executor/tests/current_timestamp.rs index bbd1c7a40..ccd974d05 100644 --- a/crates/executor/tests/current_timestamp.rs +++ b/crates/executor/tests/current_timestamp.rs @@ -70,6 +70,7 @@ fn scan_plan(predicates: Vec>) -> PhysicalASAPDAG { output_schema: schema(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index e79652f97..79b070cb6 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -1,22 +1,18 @@ //! Exercise the public library without a backend server, store, or scheduler. use asap_executor::planner::{ ir::scalar::ColumnRef, - ir::schema::{FieldDataType, GroupingStrategy, SketchAlgorithm, SketchParams, SummaryUpdate}, + ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SummaryUpdate}, }; use asap_executor::{factory::create_planner_accumulator, AggregateCore}; fn family(_k: u32) -> FieldDataType { - FieldDataType::Sketch( - SketchAlgorithm::Kll, - GroupingStrategy::PerSubpopulationInstance, - ) + FieldDataType::Sketch(SketchAlgorithm::Kll) } fn build(values: &[f64]) -> Box { let mut operator = create_planner_accumulator( &family(512), None, &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), ) .unwrap(); for (at, value) in values.iter().enumerate() { @@ -53,7 +49,6 @@ fn invalid_kll_parameters_are_rejected_at_binding() { &family(0), None, &SummaryUpdate::column(ColumnRef::SampleValue), - &Default::default(), ); assert!(result.is_err()); } @@ -63,11 +58,10 @@ fn invalid_kll_parameters_are_rejected_at_binding() { #[test] fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { use asap_executor::planner::ir::schema::SummaryInputExpr; - let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = - create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); + let mut operator = create_planner_accumulator(&family, None, &update).unwrap(); operator .update_keyed(&["a".to_string()], 7.0, 1000) .unwrap(); @@ -85,7 +79,7 @@ fn native_count_sketch_dimensions_are_not_packed_wire_dimensions() { } fn plain_count_sketch(_width: u32, _depth: u32) -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::CountSketch, Default::default()) + FieldDataType::Sketch(SketchAlgorithm::CountSketch) } fn host(i: usize) -> [String; 1] { @@ -108,8 +102,7 @@ fn build_weighted_frequency( use asap_executor::planner::ir::schema::SummaryInputExpr; let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); - let mut operator = - create_planner_accumulator(family, None, &update, &Default::default()).unwrap(); + let mut operator = create_planner_accumulator(family, None, &update).unwrap(); for (at, (i, weight)) in updates.into_iter().enumerate() { operator.validate_single_input(weight).unwrap(); operator.update_keyed(&host(i), weight, at as i64).unwrap(); @@ -213,7 +206,7 @@ fn heap_sketch( heap_size, }, }; - FieldDataType::Sketch(algorithm, Default::default()) + FieldDataType::Sketch(algorithm) } // The factory's top-k sketches use the same per-row-seeded kernel: past 20 @@ -319,7 +312,7 @@ fn count_min_heap_rejects_negative_inputs() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = heap_sketch(SketchAlgorithm::CmsWithHeap, 64, 3, 4); - let operator = create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); + let operator = create_planner_accumulator(&family, None, &update).unwrap(); assert!(operator.validate_single_input(-1.0).is_err()); assert!(operator.validate_single_input(0.5).is_ok()); } @@ -385,8 +378,7 @@ fn keyed_update_overflow_is_an_error_not_a_panic() { let mut update = SummaryUpdate::column(ColumnRef::SampleValue); update.item = Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))); let family = plain_count_sketch(64, 3); - let mut operator = - create_planner_accumulator(&family, None, &update, &Default::default()).unwrap(); + let mut operator = create_planner_accumulator(&family, None, &update).unwrap(); operator.validate_single_input(f64::MAX).unwrap(); operator.update_keyed(&host(0), f64::MAX, 0).unwrap(); let error = operator.update_keyed(&host(0), f64::MAX, 1).unwrap_err(); diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index 9fa2904b5..a9988947a 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -36,14 +36,8 @@ const ROWS: [(&str, &str, f64); 9] = [ ("db", "vacuum", 2.), ]; -fn grouping() -> GroupingStrategy { - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - } -} - fn family() -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::Cms, grouping()) + FieldDataType::Sketch(SketchAlgorithm::HydraCms) } fn schema(fields: Vec) -> Schema { @@ -65,6 +59,7 @@ fn node(id: usize, payload: Payload, output: Schema) -> PhysicalASAPDAGNode { output_schema: output, guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -109,10 +104,9 @@ fn summary_agg(update: SummaryUpdate, filter: Option>) -> Physi 1, Payload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family(), + summary_type: family(), input: update, reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: grouping(), filter, }), schema(vec![ @@ -405,7 +399,7 @@ fn constant_weight_above_i32_is_refused_at_validation() { }, ..unit_count() }; - let error = asap_executor::capability::validate_summary_kernel(&family(), &update, &grouping()) + let error = asap_executor::capability::validate_summary_kernel(&family(), &update) .expect_err("validation accepted a constant above i32::MAX"); assert!( error.contains("HydraCms") && error.contains("i32::MAX"), diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index 19ecf85b9..bb81ab532 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -31,10 +31,7 @@ const ROWS: [(&str, &str); 6] = [ ]; fn heap_family() -> FieldDataType { - FieldDataType::Sketch( - SketchAlgorithm::CmsWithHeap, - GroupingStrategy::PerSubpopulationInstance, - ) + FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap) } fn schema(fields: Vec) -> SchemaRef { @@ -137,7 +134,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { id: 1, payload: Payload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: heap_family(), + summary_type: heap_family(), input: SummaryUpdate { item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( ColumnRef::Named("service".into()), @@ -148,7 +145,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, @@ -166,6 +162,7 @@ fn query_time_count_heap_binds_a_constant_row_weight() { .clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let build = compile_node(&node, std::slice::from_ref(&input)).unwrap(); let wire = serde_json::to_value(&build).unwrap(); @@ -241,10 +238,7 @@ fn int64_weight_column_equals_float64_column() { ), counts(limit as f64) ); - let count_sketch = FieldDataType::Sketch( - SketchAlgorithm::CountSketchWithHeap, - GroupingStrategy::PerSubpopulationInstance, - ); + let count_sketch = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); let signed = Operator::per_group_keyed_summary_build( int64.clone(), count_sketch, diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 7a0595e9e..99d8dd0ea 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -350,13 +350,10 @@ fn binding_rejects_unsupported_operations() { // KLL is one family example: precomputation changes input sources, not operators. #[test] fn kll_raw_partial_and_precomputed_are_native_dags() { - use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm}; + use planner_types::ir::schema::SketchAlgorithm; let input = schema(&[("value", DataType::Float64, false)]); - let family = FieldDataType::Sketch( - SketchAlgorithm::Kll, - GroupingStrategy::PerSubpopulationInstance, - ); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let build = Operator::per_group_summary_build(input.clone(), family, 0, None, vec![]).unwrap(); let state = build.schema(); let build_range = |start: u32, end: u32| { @@ -538,6 +535,7 @@ fn bind_physical_asap_dag_before_execution() { output_schema: (*schema).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let mut dag = PhysicalASAPDAG { nodes: vec![ @@ -699,6 +697,7 @@ fn planner_semijoin_sort_limit_contract_at_both_phases() { parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer, role, schema: &asap_executor::values::SchemaRef| { PhysicalASAPDAGEdge::new( @@ -1010,14 +1009,11 @@ fn assert_weighted_rate_topk(count_sketch: bool) { Some((0, 60_000)), ) .unwrap(); - let family = FieldDataType::Sketch( - if count_sketch { - SketchAlgorithm::CountSketchWithHeap - } else { - SketchAlgorithm::CmsWithHeap - }, - Default::default(), - ); + let family = FieldDataType::Sketch(if count_sketch { + SketchAlgorithm::CountSketchWithHeap + } else { + SketchAlgorithm::CmsWithHeap + }); let build = Operator::per_group_keyed_summary_build( rates.schema(), family, @@ -1122,6 +1118,7 @@ fn grouped_temporal_schema_compiles_and_executes_topk() { output_schema: (*input).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -1223,6 +1220,7 @@ fn certified_pruning_rejects_missing_authoritative_values_after_recovery() { left: 0, right: 1, }), + selected_physical_candidate: None, }; let join = if certified { Operator::certified_semi_join(schema.clone(), schema.clone(), vec![(0, 0)]).unwrap() @@ -1329,6 +1327,7 @@ fn compiled_ingestion_binary_preserves_alignment_and_rejects_missing_updates() { lhs: 0, rhs: 1, }), + selected_physical_candidate: None, }; let binary = compile_node(&node, &[input.clone(), input.clone()]).unwrap(); for (right, expected) in [ diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 78bca3e36..b3dc5c0c3 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -390,6 +390,7 @@ fn global_extrema_bind_with_planner_derived_schema() { output_schema: (*output).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let operator = compile_node(&node, std::slice::from_ref(&input)) .expect("global extremum should bind to its Planner schema"); @@ -558,7 +559,7 @@ fn kll_partial_merge_and_multiple_evaluations_preserve_population() { use planner_types::ir::schema::SketchAlgorithm; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let mut dag = PhysicalDAG::default(); for (id, range) in [(0, 0..64), (1, 64..128), (2, 0..128)] { let rows = range.map(|n| vec![Value::Float64(n as f64)]).collect(); @@ -651,7 +652,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { let bound = 2.296 / f64::from(K).powf(0.9723); let n = (TUMBLING_WINDOW_COUNT * ROWS_PER_TUMBLING_WINDOW) as f64; let input = schema(&[("v", DataType::Float64, false)]); - let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let build = || { Operator::per_group_summary_build(input.clone(), family.clone(), 0, None, vec![]).unwrap() }; @@ -933,6 +934,7 @@ fn planner_node( ), guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -948,20 +950,16 @@ fn round_trip(operator: Operator) -> Operator { fn measure_filters_keep_groups_without_matching_rows() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::{AggIntent, GroupKeys, Reduction as Plan}; - use planner_types::ir::properties::AccuracyTarget; + let (input, rows) = filtered_input(); let plain = |t: DataType| FieldDataType::Plain(t); let node = planner_node( PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Aggregate { reduction: Plan::Reduce(GroupKeys::by(vec![0])), measures: vec![ - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, + AggIntent::Count, AggIntent::Sum { col: Some(1) }, - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, + AggIntent::Count, ], output_names: vec!["c".into(), "s".into(), "all".into()], filters: vec![keep(), keep(), None], @@ -1020,7 +1018,7 @@ fn filtered_summary_build_keeps_empty_groups() { }; use planner_types::ir::ASAPOp; let (input, rows) = filtered_input(); - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); let count = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let cases = [ ( @@ -1043,10 +1041,9 @@ fn filtered_summary_build_keeps_empty_groups() { let build = planner_node( PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Plan::Reduce(GroupKeys::by(vec![0])), - grouping: Default::default(), filter: keep(), }), &[ @@ -1096,10 +1093,9 @@ fn filtered_summary_build_is_rejected_at_ingestion_time() { let mut node = planner_node( PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Plan::Reduce(GroupKeys::by(vec![0])), - grouping: Default::default(), filter, }), &[ @@ -1171,6 +1167,7 @@ fn exact_aggregate_node( output_schema: (*schema(&[(name, dtype, false)])).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -1180,7 +1177,7 @@ fn exact_aggregate_node( fn exact_frequency_intents_execute_typed_keys_and_empty_input() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::AggIntent; - use planner_types::ir::properties::AccuracyTarget; + for (dtype, values) in [ ( DataType::Utf8, @@ -1202,18 +1199,12 @@ fn exact_frequency_intents_execute_typed_keys_and_empty_input() { let input = schema(&[("key", dtype, true)]); for (measure, name, expected) in [ ( - AggIntent::FrequencyL2 { - col: Some(0), - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyL2 { col: Some(0) }, "frequency_l2", 8.0_f64.sqrt(), ), ( - AggIntent::FrequencyEntropy { - col: Some(0), - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyEntropy { col: Some(0) }, "frequency_entropy", 1.0, ), @@ -1315,20 +1306,14 @@ fn exact_frequency_rejects_non_finite_identities() { fn exact_cardinality_binds_and_executes_typed_tuples() { use asap_executor::physical_planner::compile_node; use planner_types::ir::operator::AggIntent; - use planner_types::ir::properties::AccuracyTarget; + let input = schema(&[ ("key", DataType::Int64, true), ("tag", DataType::Utf8, true), ]); for (cols, expected) in [(vec![0], 2), (vec![0, 1], 3)] { - let node = exact_aggregate_node( - AggIntent::Cardinality { - cols, - accuracy: AccuracyTarget::exact(), - }, - "distinct", - DataType::Int64, - ); + let node = + exact_aggregate_node(AggIntent::Cardinality { cols }, "distinct", DataType::Int64); let operator = compile_node(&node, std::slice::from_ref(&input)).expect("exact distinct intent binds"); let rows = vec![ diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 5b39368fc..15c30098c 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -54,6 +54,7 @@ fn sorted_scan(schema: &Schema, identity: &Source) -> PhysicalASAPDAG { output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; PhysicalASAPDAG { nodes: vec![ @@ -137,9 +138,8 @@ fn summary_capability_levels_are_distinct() { planner::ir::schema::SketchStatistic, }; use planner_types::ir::scalar::ColumnRef; - use planner_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; - let grouping = GroupingStrategy::default(); - let cms = FieldDataType::Sketch(SketchAlgorithm::Cms, grouping.clone()); + use planner_types::ir::schema::{SketchAlgorithm, SummaryUpdate}; + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms); let update = SummaryUpdate { item: Some(planner_types::ir::schema::SummaryInputExpr::Column( ColumnRef::Named("host".into()), @@ -147,7 +147,7 @@ fn summary_capability_levels_are_distinct() { weight: planner_types::ir::schema::SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), }; - assert!(validate_summary_kernel(&cms, &update, &grouping).is_ok()); + assert!(validate_summary_kernel(&cms, &update).is_ok()); // Stored Count-Min state reads only its bare count natively. assert!(validate_native_family(&cms).is_ok()); let bare_count = SketchStatistic::PointCount { @@ -163,7 +163,7 @@ fn summary_capability_levels_are_distinct() { } ) .is_err()); - let kll = FieldDataType::Sketch(SketchAlgorithm::Kll, grouping); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); assert!(validate_native_family(&kll).is_ok()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Quantile { q: 1.5 }).is_err()); assert!(validate_sketch_evaluation(&kll, &SketchStatistic::Cardinality).is_err()); diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index cf72343ec..31135b390 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -33,11 +33,7 @@ fn logical_schema(family: FieldDataType) -> Schema { /// The stored state at node 0 (materialized at ingestion time): a global `family` /// `SummaryAgg` over a minute of raw `m` samples (`Scan` node `scan`, /// `TimeRange` node `scan + 1`). -fn view_over_samples( - family: FieldDataType, - grouping: GroupingStrategy, - scan: usize, -) -> [PhysicalASAPDAGNode; 3] { +fn view_over_samples(family: FieldDataType, scan: usize) -> [PhysicalASAPDAGNode; 3] { let samples = Schema::with_time_index( vec![ Field::plain("time", DataType::Timestamp, false), @@ -55,6 +51,7 @@ fn view_over_samples( output_schema, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; [ PhysicalASAPDAGNode { @@ -64,10 +61,9 @@ fn view_over_samples( ..node( 0, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::Reduce(GroupKeys::by(vec![])), - grouping, filter: None, child: scan + 1, }), @@ -126,8 +122,7 @@ fn try_state_dag( merge: bool, ) -> Result { let scan = 2 + usize::from(merge) + usize::from(target.is_some()); - let [view, scan_node, range_node] = - view_over_samples(family.clone(), GroupingStrategy::default(), scan); + let [view, scan_node, range_node] = view_over_samples(family.clone(), scan); let mut nodes = vec![view]; if merge { nodes.push(PhysicalASAPDAGNode { @@ -144,6 +139,7 @@ fn try_state_dag( output_schema: logical_schema(family.clone()), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); } let read_id = nodes.len(); @@ -158,15 +154,15 @@ fn try_state_dag( output_schema: logical_schema(FieldDataType::Plain(DataType::Float64)), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); if let Some(target) = target { nodes.push(PhysicalASAPDAGNode { id: nodes.len(), payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - family: target.clone(), + summary_type: target.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, child: read_id, }), @@ -178,6 +174,7 @@ fn try_state_dag( output_schema: logical_schema(target), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); } let root = nodes.last().unwrap().id; @@ -289,7 +286,7 @@ fn explicit_merge_changes_tumbling_window_cardinality() { #[test] fn precompute_rejects_nonfinite_and_out_of_range_dds_updates() { let source = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let target = FieldDataType::Sketch(SketchAlgorithm::DDSketch, GroupingStrategy::default()); + let target = FieldDataType::Sketch(SketchAlgorithm::DDSketch); let program = state_dag(source.clone(), Some(target), false); for value in [20., -20., 0.] { assert!( diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index 47a13aebd..b70a4d1be 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -86,6 +86,7 @@ fn program_for_bool(operator: BinaryOperator, return_bool: bool) -> Operator { output_schema: (*schema).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; compile_node(&node, &[schema.clone(), schema]).unwrap() } @@ -373,10 +374,9 @@ fn stored_series_evaluations_support_filters_and_sets() { payload: match id { 0 | 1 => PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 6, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, }), 5 => PhysicalASAPOperatorPayload::NonASAP( @@ -426,6 +426,7 @@ fn stored_series_evaluations_support_filters_and_sets() { }, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }) .collect::>(); let edges = [ diff --git a/crates/executor/tests/promql_values.rs b/crates/executor/tests/promql_values.rs index 57f56c020..f785e4a99 100644 --- a/crates/executor/tests/promql_values.rs +++ b/crates/executor/tests/promql_values.rs @@ -144,7 +144,7 @@ fn compile_aggregate( let reduction = match intent { AggIntent::Sum { .. } => Reduction::Sum(1), AggIntent::Avg { .. } => Reduction::Avg(1), - AggIntent::Count { .. } => Reduction::Count, + AggIntent::Count => Reduction::Count, AggIntent::Min { .. } => Reduction::Min(1), AggIntent::Max { .. } => Reduction::Max(1), _ => return Err(Error::Invalid("unsupported vector aggregate".into())), @@ -328,9 +328,7 @@ fn grouping_preserves_unenumerated_labels_and_empty_label_semantics() { equal_rows( run( compile_aggregate( - &AggIntent::Count { - accuracy: planner_types::ir::properties::AccuracyTarget::exact(), - }, + &AggIntent::Count, &GroupKeys::by(vec![ColumnRef::Named("job".into())]), ) .unwrap(), @@ -542,7 +540,6 @@ fn exact_state_evaluations_recover_and_finalize_tumbling_windows() { &family, None, &SummaryUpdate::column(ColumnRef::SampleValue), - &GroupingStrategy::PerSubpopulationInstance, ) .unwrap(); for sample in samples { @@ -591,7 +588,6 @@ fn recovered_exact_counter_uses_window_and_omits_insufficient_samples() { &family, None, &SummaryUpdate::column(ColumnRef::SampleValue), - &GroupingStrategy::PerSubpopulationInstance, ) .unwrap(); state.update_single(100., -50_000).unwrap(); diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index 7a2ff24bf..dfa473590 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -89,6 +89,7 @@ fn plan( output_schema: (**schema).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -510,6 +511,7 @@ fn windowed_plan(shift_ms: Option) -> PhysicalASAPDAG { output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let edge = |producer, consumer| { PhysicalASAPDAGEdge::new( @@ -844,20 +846,20 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { output_schema: output_schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, } }; dag.nodes.push(node( 3, PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 2, - family, + summary_type: family, input: SummaryUpdate { item: None, weight: SummaryInputExpr::Constant(1.0), weight_domain: Default::default(), }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), - grouping: GroupingStrategy::default(), filter: None, }), &state, diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index 3957844d8..05a57b1cc 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -6,8 +6,8 @@ use asap_executor::operators::{Operator, RowWeight}; use asap_executor::plan::PhysicalDAG; use asap_executor::planner::ir::scalar::ColumnRef; use asap_executor::planner::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, HydraKind, Schema, - SketchAlgorithm, SketchParams, SketchStatistic, + DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, + SketchStatistic, }; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::summary_kernels::weighted_frequency::WeightedFrequency; @@ -92,7 +92,7 @@ fn input_batch() -> Batch { } fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(algorithm, Default::default()) + FieldDataType::Sketch(algorithm) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { @@ -327,12 +327,7 @@ fn an_unknown_group_is_bad_input() { /// A HydraCms family: one shared grid for all groups. fn hydra_family() -> FieldDataType { - FieldDataType::Sketch( - SketchAlgorithm::Cms, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - }, - ) + FieldDataType::Sketch(SketchAlgorithm::HydraCms) } /// The per-sample HydraCms build: groups added with their key as series diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index 3b455e972..ea92264ca 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -25,7 +25,7 @@ fn tumbling_window() -> (Arc, Batch) { /// One tumbling window of `series` KLL states (k = 200), keyed by series. fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { - let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let schema = Arc::new(Schema { fields: vec![ Field { @@ -301,7 +301,7 @@ fn merge_over_a_stored_tumbling_window( }; use asap_executor::planner::ir::properties::ExecutionDataState; use asap_executor::planner::ir::scalar::ColumnRef; - use asap_executor::planner::ir::schema::{GroupingStrategy, SummaryUpdate}; + use asap_executor::planner::ir::schema::SummaryUpdate; use asap_executor::planner::ir::{ASAPOp, NonASAPOp, TimeRangeKind}; let samples = Schema::with_time_index( vec![ @@ -321,16 +321,16 @@ fn merge_over_a_stored_tumbling_window( parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let nodes = vec![ node( 0, Payload::ASAP(ASAPOp::SummaryAgg { child: 3, - family: schema.fields[1].dtype.clone(), + summary_type: schema.fields[1].dtype.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::Reduce(GroupKeys::by(vec![0])), - grouping: GroupingStrategy::default(), filter: None, }), schema, diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index a14d3c713..0b6a7d134 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -55,7 +55,7 @@ fn one_large_batch_fails_near_the_memory_limit() { .map(|i| vec![Value::Utf8(format!("s{i}").into()), Value::Float64(1.0)]) .collect(); let batch = Batch::try_new(input.clone(), rows).unwrap(); - let family = FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let mut dag = PhysicalDAG::default(); dag.add( 0, diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index e14f47ff7..7d5a95ec9 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -56,6 +56,7 @@ fn raw_node(id: usize, payload: NonASAPOp) -> PhysicalASAPDAGNode { parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, } } @@ -114,12 +115,11 @@ fn summary_projection_survives_recovery() { id: 0, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 3, - family: family.clone(), + summary_type: family.clone(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: planner_types::ir::operator::Reduction::Reduce( planner_types::ir::operator::GroupKeys::by(vec![2]), ), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, }), output_schema: (*schema).clone(), @@ -128,6 +128,7 @@ fn summary_projection_survives_recovery() { parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }, raw_node(2, scan()), raw_node( @@ -157,6 +158,7 @@ fn summary_projection_survives_recovery() { parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }, ], edges: vec![ @@ -196,7 +198,6 @@ fn summary_projection_survives_recovery() { &family, None, &SummaryUpdate::column(ColumnRef::SampleValue), - &GroupingStrategy::PerSubpopulationInstance, ) .unwrap(); accumulator.update_single(7., 1).unwrap(); @@ -266,6 +267,7 @@ fn summary_merge_without_inputs_is_rejected() { parameters: None, guarantee: None, retention_ms: None, + selected_physical_candidate: None, }], edges: vec![], roots: vec![0], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index cbc8e9128..7a7b083e4 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -15,14 +15,14 @@ use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, Schema}; use planner_types::ir::schema::{ - Field, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, - SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, + Field, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, + SummaryInputExpr, SummaryUpdate, WeightDomain, }; use planner_types::ir::ASAPOp; use std::sync::Arc; fn family() -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::UnivMon, GroupingStrategy::default()) + FieldDataType::Sketch(SketchAlgorithm::UnivMon) } fn context() -> RunContext { @@ -166,7 +166,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() id: 1, payload: PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { child: 0, - family: family(), + summary_type: family(), input: SummaryUpdate { item: Some(SummaryInputExpr::Column(ColumnRef::Named("src_ip".into()))), weight: SummaryInputExpr::Constant(1.0), @@ -175,7 +175,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), - grouping: GroupingStrategy::default(), filter: None, }), output_state: ExecutionDataState::QUERY_ROWS, @@ -184,6 +183,7 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() output_schema: (*state).clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }; let build = compile_node(&node, std::slice::from_ref(&input))?; let batch = Batch::try_new( diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index 4e878920a..bb97d9c47 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -3,7 +3,7 @@ use planner_types::ir::schema::*; #[test] fn rate_updates_cannot_enter_integer_heap_factory() { - let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, Default::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); let input = SummaryUpdate { item: Some(SummaryInputExpr::Column( planner_types::ir::scalar::ColumnRef::Named("service".into()), @@ -13,11 +13,5 @@ fn rate_updates_cannot_enter_integer_heap_factory() { proof: NonNegativeWeightProof::ResetAwareCounterDerivative, }, }; - assert!(asap_executor::factory::create_planner_accumulator( - &family, - None, - &input, - &Default::default() - ) - .is_err()); + assert!(asap_executor::factory::create_planner_accumulator(&family, None, &input).is_err()); } diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index 43495769f..ffd5b7e0d 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -43,7 +43,6 @@ use asap_types::ir::operator::non_asap::NonASAPOp; use asap_types::ir::operator::operator_properties::{ GroupKeys, JoinKind, Reduction, WindowFuncKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use asap_types::ir::scalar::{ExprSemantics, Predicate, ProjectItem, ScalarExpr, SortKey}; use asap_types::ir::SchemaDerivationError; @@ -277,19 +276,12 @@ fn try_promote_additive_top_ranking( { return Ok(None); } - // Count ranks unit updates; a direct Sum ranks weighted updates. - let accuracy = match ranked_agg { - AggIntent::Count { accuracy } => accuracy.clone(), - AggIntent::Sum { .. } => AccuracyTarget::exact(), - _ => unreachable!("additive ranking gate admitted a non-additive measure"), - }; - // Outer heavy-hitter `TopK`, grouped by the ranking's partition (empty for // a global `ORDER BY … LIMIT k`; the `by` labels for a partitioned `topk // by`), over the unchanged inner additive aggregate. OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::by(partition), - measures: vec![AggIntent::TopK { k: *k, accuracy }], + measures: vec![AggIntent::TopK { k: *k }], output_names: Vec::new(), filters: vec![], having: None, @@ -597,9 +589,7 @@ mod tests { } fn count() -> AggIntent { - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - } + AggIntent::Count } /// `Aggregate{ by: [1], [Count] }` over the scan — output cols `[service, count]`. @@ -672,7 +662,7 @@ mod tests { return false; }; matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]) - && matches!(self::measures(child), [AggIntent::Count { .. }]) + && matches!(self::measures(child), [AggIntent::Count]) } #[test] diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 1fb82f5a9..003b87680 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -628,9 +628,7 @@ fn resolve_agg_intent( .transpose() }; Ok(match intent { - AggIntent::Count { accuracy } => AggIntent::Count { - accuracy: accuracy.clone(), - }, + AggIntent::Count => AggIntent::Count, AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { left: resolve_column_ref(left, schema)?, right: resolve_column_ref(right, schema)?, @@ -647,34 +645,19 @@ fn resolve_agg_intent( col: col(c)?, population: *population, }, - AggIntent::Quantile { - col: c, - q, - accuracy, - } => AggIntent::Quantile { + AggIntent::Quantile { col: c, q } => AggIntent::Quantile { col: col(c)?, q: *q, - accuracy: accuracy.clone(), - }, - AggIntent::TopK { k, accuracy } => AggIntent::TopK { - k: *k, - accuracy: accuracy.clone(), }, - AggIntent::Cardinality { cols, accuracy } => AggIntent::Cardinality { + AggIntent::TopK { k } => AggIntent::TopK { k: *k }, + AggIntent::Cardinality { cols } => AggIntent::Cardinality { cols: cols .iter() .map(|c| resolve_column_ref(c, schema)) .collect::>()?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyL2 { col: c, accuracy } => AggIntent::FrequencyL2 { - col: col(c)?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyEntropy { col: c, accuracy } => AggIntent::FrequencyEntropy { - col: col(c)?, - accuracy: accuracy.clone(), }, + AggIntent::FrequencyL2 { col: c } => AggIntent::FrequencyL2 { col: col(c)? }, + AggIntent::FrequencyEntropy { col: c } => AggIntent::FrequencyEntropy { col: col(c)? }, AggIntent::Rate => AggIntent::Rate, AggIntent::IRate => AggIntent::IRate, AggIntent::Increase => AggIntent::Increase, @@ -853,7 +836,7 @@ mod tests { use asap_types::ir::operator::{ BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Source, VectorMatch, }; - use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::schema::{DataType, Field}; use asap_types::ir::BinaryOperator; @@ -936,18 +919,13 @@ mod tests { }; let intent = AggIntent::Cardinality { cols: vec![qualified("b"), qualified("a")], - accuracy: AccuracyTarget::exact(), }; assert_eq!( resolve_agg_intent(&intent, &schema).unwrap(), - AggIntent::Cardinality { - cols: vec![1, 0], - accuracy: AccuracyTarget::exact(), - } + AggIntent::Cardinality { cols: vec![1, 0] } ); let missing = AggIntent::Cardinality { cols: vec![qualified("a"), ColumnRef::Named("missing".into())], - accuracy: AccuracyTarget::exact(), }; assert!(resolve_agg_intent(&missing, &schema).is_err()); } diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs index 4a3172506..23df96169 100644 --- a/crates/frontend-metricsql/src/lib.rs +++ b/crates/frontend-metricsql/src/lib.rs @@ -8,7 +8,6 @@ use asap_frontend_common::{ use asap_types::ir::operator::{ AggIntent, BinaryOpKind, GroupKeys, PromQLVectorSetOpKind, Reduction, Source, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::{BinaryOperator, ExprSemantics, TimeRangeKind}; use asap_types::workload::{DurationMs, PlanningWorkload, QueryLanguage, WorkloadError}; @@ -63,13 +62,7 @@ pub fn lower_metricsql_query_workload( workload .query_workload .entries() - .map(|entry| { - lower_with_sample_intervals( - &entry.query.0, - entry.requirements.accuracy.clone(), - &sample_interval_of, - ) - }) + .map(|entry| lower_with_sample_intervals(&entry.query.0, &sample_interval_of)) .collect() } @@ -79,7 +72,6 @@ pub fn lower_metricsql_query_workload( /// [`WorkloadError::MissingSampleInterval`]. pub(crate) fn lower_with_sample_intervals( query: &str, - accuracy: AccuracyTarget, sample_interval_of: &dyn Fn(&str) -> Option, ) -> Result { let ast = parse_metricsql(query)?; @@ -88,18 +80,13 @@ pub(crate) fn lower_with_sample_intervals( asap_types::ir::ScalarExpr::literal_f64(number.value), )); } - let unresolved = Lowerer { - accuracy, - sample_interval_of, - } - .lower(&ast)?; + let unresolved = Lowerer { sample_interval_of }.lower(&ast)?; resolve_root(&unresolved) .map(asap_types::ir::QueryRoot::Operator) .map_err(|e| MetricsqlError::Resolve(e.to_string())) } struct Lowerer<'a> { - accuracy: AccuracyTarget, /// What a bare instant selector over a metric looks back; `None` when /// the metric declares no sample interval. sample_interval_of: &'a dyn Fn(&str) -> Option, @@ -220,9 +207,7 @@ impl Lowerer<'_> { RollupFunction::MinOverTime => AggIntent::Min { col: None }, RollupFunction::MaxOverTime => AggIntent::Max { col: None }, RollupFunction::SumOverTime => AggIntent::Sum { col: None }, - RollupFunction::CountOverTime => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, + RollupFunction::CountOverTime => AggIntent::Count, RollupFunction::StddevOverTime => AggIntent::StdDev { col: None, population: true, @@ -245,7 +230,6 @@ impl Lowerer<'_> { RollupFunction::QuantileOverTime => AggIntent::Quantile { col: None, q: number_arg(&function.args, 0)?, - accuracy: self.accuracy.clone(), }, _ => { return Err(unsupported(format!( @@ -294,9 +278,7 @@ impl Lowerer<'_> { AggregateFunction::Min => AggIntent::Min { col: None }, AggregateFunction::Max => AggIntent::Max { col: None }, // `count(v)` counts series, not distinct sample values. - AggregateFunction::Count => AggIntent::Count { - accuracy: self.accuracy.clone(), - }, + AggregateFunction::Count => AggIntent::Count, AggregateFunction::StdDev => AggIntent::StdDev { col: None, population: true, @@ -309,7 +291,6 @@ impl Lowerer<'_> { AggregateFunction::Quantile => AggIntent::Quantile { col: None, q: number_arg(&expr.args, 0)?, - accuracy: self.accuracy.clone(), }, _ => return Err(unsupported(format!("aggregate `{}`", expr.name()))), }; @@ -457,10 +438,9 @@ fn unsupported(message: impl Into) -> MetricsqlError { /// [`lower_metricsql_query`]). pub fn lower_metricsql( query: &str, - accuracy: AccuracyTarget, sample_interval: Duration, ) -> Result, MetricsqlError> { - match lower_metricsql_query(query, accuracy, sample_interval)? { + match lower_metricsql_query(query, sample_interval)? { asap_types::ir::QueryRoot::Operator(node) => Ok(node), _ => Err(MetricsqlError::UnsupportedFeature( "scalar root: use lower_metricsql_query".into(), @@ -473,8 +453,7 @@ pub fn lower_metricsql( /// back that long for its latest sample. pub fn lower_metricsql_query( query: &str, - accuracy: AccuracyTarget, sample_interval: Duration, ) -> Result { - lower_with_sample_intervals(query, accuracy, &|_| Some(sample_interval)) + lower_with_sample_intervals(query, &|_| Some(sample_interval)) } diff --git a/crates/frontend-metricsql/tests/external_corpus.rs b/crates/frontend-metricsql/tests/external_corpus.rs index 6aa8d1427..4e4f565dd 100644 --- a/crates/frontend-metricsql/tests/external_corpus.rs +++ b/crates/frontend-metricsql/tests/external_corpus.rs @@ -15,7 +15,6 @@ use std::path::Path; use std::time::Duration; use asap_frontend_metricsql::{lower_metricsql_query, MetricsqlError}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::QueryRoot; use crate::support::logical_plan; @@ -71,10 +70,9 @@ fn normalize(msg: &str) -> String { /// Tally `corpus`, counting each failure's normalized message in `errors`. fn tally(corpus: &str, errors: &mut HashMap) -> Tally { - let accuracy = AccuracyTarget::max_error(0.01); let mut t = Tally::default(); for q in queries(corpus) { - match lower_metricsql_query(q, accuracy.clone(), Duration::from_secs(15)) { + match lower_metricsql_query(q, Duration::from_secs(15)) { Ok(QueryRoot::Operator(dag)) => { t.lowered += 1; if logical_plan(&dag).is_err() { diff --git a/crates/frontend-metricsql/tests/lowering.rs b/crates/frontend-metricsql/tests/lowering.rs index 92d200272..c8f972ebd 100644 --- a/crates/frontend-metricsql/tests/lowering.rs +++ b/crates/frontend-metricsql/tests/lowering.rs @@ -6,14 +6,13 @@ use asap_frontend_metricsql::test_support::{ }; use asap_frontend_metricsql::MetricsqlError; use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; /// Every metric's sample interval, the lookback of a bare selector. const INTERVAL: Duration = Duration::from_secs(15); fn lower(query: &str) -> Rc { - lower_metricsql(query, AccuracyTarget::max_error(0.01), INTERVAL).unwrap() + lower_metricsql(query, INTERVAL).unwrap() } #[test] @@ -71,12 +70,8 @@ fn default_rollup_with_explicit_range_is_last_over_time() { fn keep_metric_names_is_preserved_in_ast_and_rejected_without_lineage() { let ast = parse_metricsql("rate(requests_total[5m]) keep_metric_names").unwrap(); assert!(ast.keep_metric_names()); - let error = lower_metricsql( - "rate(requests_total[5m]) keep_metric_names", - AccuracyTarget::exact(), - INTERVAL, - ) - .unwrap_err(); + let error = + lower_metricsql("rate(requests_total[5m]) keep_metric_names", INTERVAL).unwrap_err(); assert!( matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("metric-name lineage")) ); @@ -98,7 +93,7 @@ fn victoria_metrics_extension_corpus_parses_natively_and_fails_closed() { let ast = parse_metricsql(query) .unwrap_or_else(|error| panic!("native MetricsQL parser rejected {query:?}: {error}")); assert!(!ast.to_string().is_empty()); - let result = lower_metricsql(query, AccuracyTarget::exact(), INTERVAL); + let result = lower_metricsql(query, INTERVAL); assert!( result.is_err(), "extension semantics must be represented or routed to exact fallback: {query}" @@ -136,7 +131,7 @@ fn canonical_identity_recursively_formats_metricsql_extensions() { fn metricsql_multi_argument_aggregates_fail_closed() { for query in ["sum(foo, bar)", "avg(foo, bar)", "count(foo, bar)"] { parse_metricsql(query).expect("MetricsQL accepts multi-argument aggregates"); - let error = lower_metricsql(query, AccuracyTarget::exact(), INTERVAL).unwrap_err(); + let error = lower_metricsql(query, INTERVAL).unwrap_err(); assert!( matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("requires exactly 1")), "{query} must not silently discard an aggregate input" diff --git a/crates/frontend-metricsql/tests/lowering_semantics.rs b/crates/frontend-metricsql/tests/lowering_semantics.rs index 0db792586..37284b71b 100644 --- a/crates/frontend-metricsql/tests/lowering_semantics.rs +++ b/crates/frontend-metricsql/tests/lowering_semantics.rs @@ -3,7 +3,6 @@ use std::time::Duration; use asap_frontend_metricsql::{lower_metricsql_query_workload, MetricsqlError}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, @@ -12,11 +11,8 @@ use asap_types::workload::{ const INTERVAL: Duration = Duration::from_secs(15); -fn lower_metricsql( - query: &str, - accuracy: AccuracyTarget, -) -> Result, MetricsqlError> { - asap_frontend_metricsql::test_support::lower_metricsql(query, accuracy, INTERVAL) +fn lower_metricsql(query: &str) -> Result, MetricsqlError> { + asap_frontend_metricsql::test_support::lower_metricsql(query, INTERVAL) } fn workload(language: QueryLanguage, query: &str) -> PlanningWorkload { @@ -62,12 +58,12 @@ fn time_range_under_aggregate(node: &OperatorNode) -> (Duration, TimeRangeKind) /// `count(v)` counts series; it is not a distinct count of sample values. #[test] fn count_counts_series() { - let node = lower_metricsql("count(up)", AccuracyTarget::exact()).unwrap(); + let node = lower_metricsql("count(up)").unwrap(); let NonASAPOp::Aggregate { measures, .. } = node.expect_non_asap() else { panic!("expected an aggregate, got {node:?}"); }; assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "{measures:?}" ); } @@ -82,25 +78,25 @@ fn rollups_without_an_explicit_range_are_rejected() { "max_over_time(m)", "default_rollup(m)", ] { - let error = lower_metricsql(query, AccuracyTarget::exact()).unwrap_err(); + let error = lower_metricsql(query).unwrap_err(); assert!( matches!(&error, MetricsqlError::UnsupportedFeature(message) if message.contains("evaluation step")), "{query}: {error:?}" ); } - assert!(lower_metricsql("rate(m[5m])", AccuracyTarget::exact()).is_ok()); + assert!(lower_metricsql("rate(m[5m])").is_ok()); } /// A bare instant selector reads the latest sample within the sample /// interval, as in PromQL; an explicit range keeps its own window. #[test] fn instant_selectors_look_back_one_sample_interval() { - let node = lower_metricsql("sum(up)", AccuracyTarget::exact()).unwrap(); + let node = lower_metricsql("sum(up)").unwrap(); assert_eq!( time_range_under_aggregate(&node), (INTERVAL, TimeRangeKind::Instant) ); - let node = lower_metricsql("rate(m[5m])", AccuracyTarget::exact()).unwrap(); + let node = lower_metricsql("rate(m[5m])").unwrap(); assert_eq!( time_range_under_aggregate(&node), (Duration::from_secs(300), TimeRangeKind::Range) diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index 38f9db6b0..fa90b507d 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -104,7 +104,6 @@ pub fn lower_promql_query_workload( .map(|entry| { let root = promql::PromqlLowerer::lower_query( &entry.query.0, - &entry.requirements.accuracy.clone(), &sample_interval_of, &metric_type_of, )?; diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 6c8a33f2a..66aa42a1b 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -79,7 +79,6 @@ use asap_types::ir::operator::operator_properties::{ use asap_types::ir::{BinaryOperator, ExprSemantics, TimeRangeKind}; use asap_types::ir::operator::{InfoMatcher, SampleKind}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::workload::{PrometheusMetricType, WorkloadError}; @@ -92,8 +91,6 @@ type Result = std::result::Result; /// Parses and lowers (→ the canonical, unresolved DAG) a PromQL query string. pub(crate) struct PromqlLowerer<'a> { - /// The target threaded onto every approximate intent. - accuracy: AccuracyTarget, /// What a bare instant selector over a metric looks back; `None` when /// the metric declares no sample interval. sample_interval_of: &'a dyn Fn(&str) -> Option, @@ -138,7 +135,6 @@ impl PromqlLowerer<'_> { /// checked by [`check_histogram_quantile_input`]. pub(crate) fn lower_query( query: &str, - accuracy: &AccuracyTarget, sample_interval_of: &dyn Fn(&str) -> Option, metric_type_of: &dyn Fn(&str) -> Option, ) -> Result { @@ -146,10 +142,7 @@ impl PromqlLowerer<'_> { check_depth(&ast, MAX_DEPTH)?; let mut metrics = Vec::new(); collect_metric_names(&ast, &mut metrics); - let lowerer = PromqlLowerer { - accuracy: accuracy.clone(), - sample_interval_of, - }; + let lowerer = PromqlLowerer { sample_interval_of }; if metrics .iter() .any(|metric| metric_type_of(metric) == Some(PrometheusMetricType::NativeHistogram)) @@ -402,7 +395,6 @@ impl PromqlLowerer<'_> { AggIntent::Quantile { col: None, q: num_arg(call, 0)?, - accuracy: self.accuracy.clone(), }, 1, ))), @@ -418,21 +410,10 @@ impl PromqlLowerer<'_> { col: None, population: true, }), - "count_over_time" => at0(AggIntent::Count { - accuracy: self.accuracy.clone(), - }), - "distinct_over_time" => at0(AggIntent::Cardinality { - cols: vec![], - accuracy: self.accuracy.clone(), - }), - "entropy_over_time" => at0(AggIntent::FrequencyEntropy { - col: None, - accuracy: self.accuracy.clone(), - }), - "l2_over_time" => at0(AggIntent::FrequencyL2 { - col: None, - accuracy: self.accuracy.clone(), - }), + "count_over_time" => at0(AggIntent::Count), + "distinct_over_time" => at0(AggIntent::Cardinality { cols: vec![] }), + "entropy_over_time" => at0(AggIntent::FrequencyEntropy { col: None }), + "l2_over_time" => at0(AggIntent::FrequencyL2 { col: None }), // Counter-derivative range functions (issue #44). Each has its own // intent — `changes` (value-change count) and `resets` (counter-reset // count) are NOT sample counts, so they are not aliased to @@ -536,9 +517,7 @@ impl PromqlLowerer<'_> { } } else if op == token::T_COUNT { // Counts vector elements regardless of their sample values. - Outer::Reduce(AggIntent::Count { - accuracy: self.accuracy.clone(), - }) + Outer::Reduce(AggIntent::Count) } else if op == token::T_SUM { Outer::Reduce(AggIntent::Sum { col: None }) } else if op == token::T_GROUP { @@ -581,7 +560,6 @@ impl PromqlLowerer<'_> { Outer::Reduce(AggIntent::Quantile { col: None, q: num_param(agg)?, - accuracy: self.accuracy.clone(), }) } else { return Err(LoweringError::UnsupportedAggregateOp(format!( @@ -632,7 +610,7 @@ impl PromqlLowerer<'_> { ) || (matches!(reduction, Reduction::PerEntity) && matches!( measures.as_slice(), - [AggIntent::Sum { .. } | AggIntent::Count { .. }] + [AggIntent::Sum { .. } | AggIntent::Count] ))) ); let direct_counter_ranking = matches!(&child, Unresolved::Aggregate { @@ -644,10 +622,7 @@ impl PromqlLowerer<'_> { if descending && (weighted_group_ranking || direct_counter_ranking) { return Ok(outer_aggregate( keys, - AggIntent::TopK { - k: k as usize, - accuracy: self.accuracy.clone(), - }, + AggIntent::TopK { k: k as usize }, child, )); } @@ -1089,7 +1064,7 @@ impl PromqlLowerer<'_> { // operator pair. The descending-plus-measure rule is shared with the // canonicalize-pass promotion so the two cannot drift (issue #38). let measure = match func { - Some(AggIntent::Count { .. }) => topk::Ranking::Frequency, + Some(AggIntent::Count) => topk::Ranking::Frequency, Some(AggIntent::Sum { .. }) => topk::Ranking::WeightedSum, _ => topk::Ranking::NonAdditive, }; @@ -1119,10 +1094,7 @@ impl PromqlLowerer<'_> { // A ranking always reduces (a `by`-empty TopK ranks the // whole input into one ordering, never per-entity). reduction: Reduction::Reduce(keys.into()), - measures: vec![AggIntent::TopK { - k: k as usize, - accuracy: self.accuracy.clone(), - }], + measures: vec![AggIntent::TopK { k: k as usize }], output_names: vec![], filters: vec![], having: None, diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index 09a3b2797..d94b49e5c 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -4,8 +4,8 @@ use crate::support; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{ - ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, SketchAlgorithm, - SummaryInputExpr, WeightDomain, + ExactKind, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr, + WeightDomain, }; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; use std::rc::Rc; @@ -17,7 +17,13 @@ fn reads_hydra(node: &Rc) -> bool { matches!( &node.operator, Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }), + summary_type: FieldDataType::Sketch( + SketchAlgorithm::HydraCms + | SketchAlgorithm::HydraCountSketch + | SketchAlgorithm::HydraKll + | SketchAlgorithm::HydraHll + | SketchAlgorithm::HydraUnivMon + ), .. }) ) @@ -49,7 +55,7 @@ fn exact_counts_select_count_accumulators() { matches!( &node.operator, Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Count, _), + summary_type: FieldDataType::ExactAggregate(ExactKind::Count, _), .. }) ) @@ -82,7 +88,7 @@ fn frequency_count_candidates_use_unit_weights() { continue; }; let Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(kind), input, .. }) = &summary_input.operator @@ -157,7 +163,7 @@ fn aggregate_fixture(query: &str, series: &[Vec]) -> Vec { other => panic!("unsupported fixture input: {other:?}"), } let aggregate = |values: &[f64]| match measures.as_slice() { - [AggIntent::Count { .. }] => values.len() as f64, + [AggIntent::Count] => values.len() as f64, [AggIntent::Cardinality { .. }] => { let mut distinct = values.to_vec(); distinct.sort_by(f64::total_cmp); @@ -235,7 +241,7 @@ fn cms_count_updates_total_ten_for_zero_positive_and_negative_samples() { .iter() .any(|node| { matches!(&node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) + PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) if kind == &SketchAlgorithm::Cms) }) .then_some(dag) diff --git a/crates/frontend-promql/tests/planner_support.rs b/crates/frontend-promql/tests/planner_support.rs index 39f00e0ca..04aa13096 100644 --- a/crates/frontend-promql/tests/planner_support.rs +++ b/crates/frontend-promql/tests/planner_support.rs @@ -37,7 +37,8 @@ pub fn run_all_stages( /// at `accuracy`. pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { let run = run_all_stages(root, accuracy); - match run.selected_plan.as_deref().expect("a plan is selected") { + let (plan, _) = run.selection.as_ref().expect("a plan is selected"); + match plan.query_roots().unwrap().as_slice() { [asap_types::ir::QueryRoot::Operator(node)] => Rc::clone(node), other => panic!("one operator root: {other:?}"), } diff --git a/crates/frontend-promql/tests/promql_binding_regressions.rs b/crates/frontend-promql/tests/promql_binding_regressions.rs index 77066e474..3d7b681eb 100644 --- a/crates/frontend-promql/tests/promql_binding_regressions.rs +++ b/crates/frontend-promql/tests/promql_binding_regressions.rs @@ -38,5 +38,5 @@ fn count_is_row_count_not_distinct_sample_value_count() { let NonASAPOp::Aggregate { measures, .. } = tree.expect_non_asap() else { panic!("expected aggregate") }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index e661ee984..f96ecc8f4 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -289,10 +289,7 @@ fn sum_by_groups_via_positional_aggregate() { #[test] fn count_is_row_count() { - assert!(has(&ok("count(up)"), |i| matches!( - i, - AggIntent::Count { .. } - ))); + assert!(has(&ok("count(up)"), |i| matches!(i, AggIntent::Count))); } #[test] @@ -425,7 +422,7 @@ fn over_time_functions_reduce_over_time_range() { "max" => matches!(i, AggIntent::Max { .. }), "min" => matches!(i, AggIntent::Min { .. }), "sum" => matches!(i, AggIntent::Sum { .. }), - "count" => matches!(i, AggIntent::Count { .. }), + "count" => matches!(i, AggIntent::Count), _ => unreachable!(), }); assert!(matched, "{q}: missing {want} intent"); @@ -582,28 +579,14 @@ fn double_unary_negation_nests() { assert!(negates_via_scalar(child)); } +/// `count` lowers to a `Count` measure, not a distinct count. #[test] -fn count_maps_to_count_and_inherits_accuracy() { - // Counts preserve the workload accuracy target without counting distinct values. - let exact = lower_promql("count by (job) (up)", AccuracyTarget::exact()).unwrap(); - assert!( - has(&exact, |i| matches!( - i, - AggIntent::Count { accuracy } if accuracy.is_exact() - )), - "Count must stay exact under an exact target, got {:?}", - intents(&exact) - ); - - let approx = lower_promql("count by (job) (up)", AccuracyTarget::max_error(0.01)).unwrap(); +fn count_maps_to_count() { + let lowered = lower_promql("count by (job) (up)", AccuracyTarget::exact()).unwrap(); assert!( - has(&approx, |i| matches!( - i, - AggIntent::Count { accuracy } if (accuracy.max_error - 0.01).abs() < 1e-9 - && accuracy.accuracy_metric.is_none() - )), - "Count must carry the approximate target, got {:?}", - intents(&approx) + has(&lowered, |i| matches!(i, AggIntent::Count)), + "got {:?}", + intents(&lowered) ); } @@ -1939,7 +1922,7 @@ fn limitk_by_carries_the_grouping_and_composes_in_a_set_op() { // the PromqlSeriesSample must be preserved under the set op (it must lower, not reject). assert!(has( &ok("count(limitk(2, http_requests) and http_requests)"), - |i| matches!(i, AggIntent::Count { .. }) + |i| matches!(i, AggIntent::Count) )); } diff --git a/crates/frontend-promql/tests/promql_lowering.rs b/crates/frontend-promql/tests/promql_lowering.rs index 163fe82b6..cbe818130 100644 --- a/crates/frontend-promql/tests/promql_lowering.rs +++ b/crates/frontend-promql/tests/promql_lowering.rs @@ -30,17 +30,11 @@ fn frequency_extensions_lower_to_explicit_frequency_statistics() { for (query, expected) in [ ( "entropy_over_time(cpu_usage[5m])", - AggIntent::FrequencyEntropy { - col: None, - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyEntropy { col: None }, ), ( "l2_over_time(cpu_usage[5m])", - AggIntent::FrequencyL2 { - col: None, - accuracy: AccuracyTarget::exact(), - }, + AggIntent::FrequencyL2 { col: None }, ), ] { assert!(all_intents(&lower(query)) @@ -50,9 +44,9 @@ fn frequency_extensions_lower_to_explicit_frequency_statistics() { } #[test] -fn distinct_over_time_preserves_cardinality_accuracy_and_nested_windows() { - // Distinct counts sample values, not samples or series; all lowering routes - // retain the caller's accuracy requirement, including subquery arguments. +fn distinct_over_time_lowers_to_cardinality_over_nested_windows() { + // Distinct counts sample values, not samples or series, on every lowering + // route, subquery arguments included. for query in [ "distinct_over_time(cpu_usage{job=\"worker\"}[5m] offset 1h)", "distinct_over_time((cpu_usage + 1)[5m:1m])", @@ -63,14 +57,14 @@ fn distinct_over_time_preserves_cardinality_accuracy_and_nested_windows() { let mut intents = Vec::new(); collect_intents(&dag, &mut intents); assert!( - intents.iter().any(|intent| matches!( - intent, AggIntent::Cardinality { accuracy: actual, .. } if *actual == accuracy - )), + intents + .iter() + .any(|intent| matches!(intent, AggIntent::Cardinality { .. })), "{query}: {dag:?}" ); assert!(!intents .iter() - .any(|intent| matches!(intent, AggIntent::Count { .. }))); + .any(|intent| matches!(intent, AggIntent::Count))); } } } @@ -441,7 +435,7 @@ fn count_over_rate_keeps_both_levels() { else { panic!("expected outer Aggregate{{Count}}, got {qe:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); assert!(matches!( child.expect_non_asap(), NonASAPOp::Aggregate { measures, .. } if matches!(measures.as_slice(), [AggIntent::Rate]) @@ -473,7 +467,7 @@ fn count_over_distinct_over_time_preserves_both_aggregates() { panic!("expected outer Count: {dag:?}"); }; assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "{query}: {dag:?}" ); assert_eq!(actual, &reduction, "{query}"); @@ -513,7 +507,7 @@ fn count_never_lowers_to_distinct_sample_values() { let dag = lower(query); let intents = all_intents(&dag); assert!( - intents.iter().any(|i| matches!(i, AggIntent::Count { .. })), + intents.iter().any(|i| matches!(i, AggIntent::Count)), "{query}: {dag:?}" ); assert!( @@ -541,7 +535,7 @@ fn outer_count_counts_series() { panic!("expected outer Aggregate grouped by symbol, got {qe:?}"); }; assert_eq!(reduction, &Reduction::by(vec![2])); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); // Inner: Aggregate{Count} over TimeRange (per-series count_over_time). let NonASAPOp::Aggregate { measures, child, .. @@ -549,7 +543,7 @@ fn outer_count_counts_series() { else { panic!("expected Aggregate (count_over_time) under the outer count, got {child:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); assert!(matches!( child.expect_non_asap(), NonASAPOp::TimeRange { .. } @@ -584,7 +578,7 @@ fn topk_over_count_is_heavy_hitter_topk() { else { panic!("expected Aggregate (count_over_time) under TopK, got {child:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); let NonASAPOp::TimeRange { range, child, .. } = child.expect_non_asap() else { panic!("expected TimeRange under Count aggregate, got {child:?}"); }; @@ -686,7 +680,7 @@ fn bottomk_over_count_is_generic_sort_ascending() { }; assert!(keys[0].ascending, "bottomk ranks ascending"); // Count intent is still present (as the inner aggregate), no TopK. - assert!(has_intent(&qe, |i| matches!(i, AggIntent::Count { .. }))); + assert!(has_intent(&qe, |i| matches!(i, AggIntent::Count))); assert!(!has_intent(&qe, |i| matches!(i, AggIntent::TopK { .. }))); } @@ -737,7 +731,7 @@ fn topk_count_output_schema_carries_group_key() { else { panic!("expected inner Aggregate{{Count}}, got {child:?}"); }; - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); assert!(matches!( child.expect_non_asap(), NonASAPOp::TimeRange { .. } @@ -1040,22 +1034,6 @@ fn fill_modifiers_are_rejected_not_ignored() { // ── accuracy propagation ────────────────────────────────────────────────────── -#[test] -fn accuracy_target_flows_into_quantile_intent() { - let qe = lower_promql( - "quantile_over_time(0.9, m[5m])", - AccuracyTarget::max_error(0.01), - ) - .unwrap(); - let NonASAPOp::Aggregate { measures, .. } = qe.expect_non_asap() else { - panic!("expected Aggregate"); - }; - assert!(matches!( - &measures[0], - AggIntent::Quantile { accuracy, .. } if (accuracy.max_error - 0.01).abs() < 1e-12 - )); -} - // ── schema flow (positional, carried on Scan; derived on demand) ───────────────── #[test] diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 38be69839..816706b6c 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -150,7 +150,12 @@ fn local_computation_candidates(root: Rc) -> Result) -> Option> { let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &node.operator else { return false; }; - matches!(&summary_input.operator, Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) + matches!(&summary_input.operator, Operator::ASAP(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) if kind == &SketchAlgorithm::UnivMon) }) } @@ -41,11 +41,12 @@ fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { assert!(unknown.is_empty(), "{query}"); } else { assert!(!unknown.is_empty(), "{query}"); - let [asap_types::ir::QueryRoot::Operator(selected)] = run - .selected_plan - .as_deref() - .expect("a certified plan is selected") - else { + let (plan, _) = run + .selection + .as_ref() + .expect("a certified plan is selected"); + let roots = plan.query_roots().unwrap(); + let [asap_types::ir::QueryRoot::Operator(selected)] = roots.as_slice() else { panic!("one operator result"); }; assert!(selected.guarantee.is_some(), "{query}"); diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs index c95f4af1a..755e64514 100644 --- a/crates/frontend-sql/src/frequency.rs +++ b/crates/frontend-sql/src/frequency.rs @@ -4,8 +4,8 @@ //! unit counts. [`recognize_frequency_idioms`] names them as //! `FrequencyL2` / `FrequencyEntropy` intents, so Pass 1 can offer exact and //! summary candidates (UnivMon among them) for each. It preserves the -//! output schema, the predicates below the grouping, the query's accuracy -//! target, SQL's NULL result on empty input (decided by an exact population +//! output schema, the predicates below the grouping, SQL's NULL result on +//! empty input (decided by an exact population //! count, never by an estimate) and the entropy unit: the IR's entropy is in //! bits, the SQL idiom's `LN` gives nats. Integer products and nullable //! keys are refused, because SQL integer overflow and NULL groups are @@ -14,7 +14,6 @@ use asap_types::ir::operator::non_asap::any_measure_filtered; use asap_types::ir::operator::{ AggIntent, GroupKeys, JoinKind, Reduction, WindowFrameBound, WindowFrameOffset, WindowFuncKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::schema::DataType; use asap_types::ir::{ @@ -146,25 +145,18 @@ fn frequency_l2_rewrite(root: &Rc) -> Option> { ) { return None; } - let (key, accuracy, input) = grouped_unit_count(&inner)?; + let (key, input) = grouped_unit_count(&inner)?; sql_frequency_result( root, input, - AggIntent::FrequencyL2 { - col: Some(key), - // The count's target; Stage 1 sizes the norm's summary for it as - // a relative error. - accuracy, - }, + AggIntent::FrequencyL2 { col: Some(key) }, "frequency_l2", ScalarExpr::Column(0), ) } // Both frequency rules require a complete, unfiltered count per non-NULL identity. -fn grouped_unit_count( - node: &Rc, -) -> Option<(usize, AccuracyTarget, Rc)> { +fn grouped_unit_count(node: &Rc) -> Option<(usize, Rc)> { let NonASAPOp::Aggregate { reduction: Reduction::Reduce(keys), measures, @@ -179,7 +171,7 @@ fn grouped_unit_count( let [key] = keys.keys() else { return None; }; - let [AggIntent::Count { accuracy }] = measures.as_slice() else { + let [AggIntent::Count] = measures.as_slice() else { return None; }; let field = child.schema.fields.get(*key)?; @@ -193,7 +185,7 @@ fn grouped_unit_count( { return None; } - Some((*key, accuracy.clone(), Rc::clone(child))) + Some((*key, Rc::clone(child))) } fn probability_term(product: &ScalarExpr) -> Option<&ScalarExpr> { @@ -307,7 +299,7 @@ fn frequency_entropy_rewrite(root: &Rc) -> Option ) { return None; } - let (key, accuracy, input) = grouped_unit_count(child)?; + let (key, input) = grouped_unit_count(child)?; let nats = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Mul, left: Box::new(ScalarExpr::Column(0)), @@ -329,12 +321,7 @@ fn frequency_entropy_rewrite(root: &Rc) -> Option sql_frequency_result( root, input, - AggIntent::FrequencyEntropy { - col: Some(key), - // The count's target; Stage 1 sizes the entropy's summary for it - // as an absolute error in bits. - accuracy, - }, + AggIntent::FrequencyEntropy { col: Some(key) }, "frequency_entropy", nats, ) @@ -364,12 +351,7 @@ fn sql_frequency_result( .ok() }; let statistic = aggregate(measure, name)?; - let count = aggregate( - AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - "population_count", - )?; + let count = aggregate(AggIntent::Count, "population_count")?; let child = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Join { kind: JoinKind::Cross, pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), diff --git a/crates/frontend-sql/src/lib.rs b/crates/frontend-sql/src/lib.rs index 9a8d09766..daa923eb3 100644 --- a/crates/frontend-sql/src/lib.rs +++ b/crates/frontend-sql/src/lib.rs @@ -17,7 +17,6 @@ pub mod test_support; use std::rc::Rc; use asap_frontend_common::resolve_root; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::OperatorNode; use asap_types::workload::SqlDialect; @@ -30,13 +29,9 @@ use sql::SqlLowerer; /// /// The `catalog` supplies table schemas (used both to plan the SQL with /// DataFusion and to carry positional column identity into the resolved -/// DAG). `accuracy` is threaded onto every approximate intent as it's built. -pub async fn lower_sql( - query: &str, - catalog: &SqlCatalog, - accuracy: AccuracyTarget, -) -> Result, SqlError> { - lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL, accuracy).await +/// DAG). +pub async fn lower_sql(query: &str, catalog: &SqlCatalog) -> Result, SqlError> { + lower_sql_dialect(query, catalog, SqlDialect::DataFusionSQL).await } /// Lower a single SQL query string under an explicit [`SqlDialect`]. @@ -51,10 +46,9 @@ pub async fn lower_sql_dialect( query: &str, catalog: &SqlCatalog, dialect: SqlDialect, - accuracy: AccuracyTarget, ) -> Result, SqlError> { let unresolved = SqlLowerer::with_dialect(catalog, dialect) - .lower(query, &accuracy) + .lower(query) .await?; // Binding resolves names and derives every node's schema; result-type // checks (such as temporal subtraction, whose duration unit the IR cannot diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index 4121dd22e..3fd2debc7 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -67,7 +67,6 @@ use asap_types::ir::schema::{DataType, FieldDataType, Schema}; use asap_types::ir::TimeRangeKind; use asap_types::ir::operator::{JoinKind, RelationalSetOpKind, WindowFuncKind}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::workload::SqlDialect; @@ -89,9 +88,6 @@ use self::types::{arrow_to_dtype, scalar_value_to_asap, schema_to_arrow}; pub(crate) struct SqlLowerer<'a> { catalog: &'a SqlCatalog, dialect: SqlDialect, - /// The target [`lower`](Self::lower) threads onto every approximate - /// intent; exact until a `lower` call sets it. - accuracy: AccuracyTarget, } impl<'a> SqlLowerer<'a> { @@ -109,16 +105,10 @@ impl<'a> SqlLowerer<'a> { /// still unknown to DataFusion's planner and still fails there, and /// `ElasticSQL` has no vendored parser at all. pub(crate) fn with_dialect(catalog: &'a SqlCatalog, dialect: SqlDialect) -> Self { - Self { - catalog, - dialect, - accuracy: AccuracyTarget::exact(), - } + Self { catalog, dialect } } - /// Parse + lower a SQL query to the name-based tree, threading - /// `accuracy` onto every approximate intent (`Count`, `Quantile`, - /// `Cardinality`) as it is built. + /// Parse + lower a SQL query to the name-based tree. /// /// Runs `ApplyFunctionRewrites` — the single `AnalyzerRule` DataFusion's /// own `Analyzer` uses internally to apply `FunctionRewrite`s, called @@ -140,11 +130,7 @@ impl<'a> SqlLowerer<'a> { /// check would. Going straight to `ApplyFunctionRewrites` avoids that /// entirely. TypeCoercion then records implicit conversions explicitly, /// including timestamp literals in predicates, before IR validation. - pub(crate) async fn lower( - &mut self, - sql: &str, - accuracy: &AccuracyTarget, - ) -> Result { + pub(crate) async fn lower(&self, sql: &str) -> Result { let ctx = self.build_context()?; let state = ctx.state(); let statement = if matches!(self.dialect, SqlDialect::ClickhouseSQL) { @@ -201,7 +187,6 @@ impl<'a> SqlLowerer<'a> { let plan = datafusion::optimizer::analyzer::type_coercion::TypeCoercion::new() .analyze(plan, &ctx.state().options())?; let plan = drop_integer_decimal_widening(plan)?; - self.accuracy = accuracy.clone(); self.lower_plan(&plan) } @@ -866,7 +851,7 @@ impl<'a> SqlLowerer<'a> { .collect(); let measures = aggr_expr .iter() - .map(|e| lower_agg_intent(e, &self.accuracy)) + .map(lower_agg_intent) .collect::, LoweringError>>()?; // Empty when nothing is filtered — the one canonical unfiltered shape. let filters = if measure_filters.iter().any(Option::is_some) { @@ -1113,7 +1098,7 @@ impl<'a> SqlLowerer<'a> { .collect::, LoweringError>>()?; let measures = aggr_expr .iter() - .map(|e| lower_agg_intent(e, &self.accuracy)) + .map(lower_agg_intent) .collect::, LoweringError>>()?; let input = derived.wrap(input)?; @@ -1708,17 +1693,14 @@ fn conditional_count_arm(expr: &Expr) -> Option<(&Expr, &Expr)> { /// (issue #225) as flat data (`NATIVE_FUNCTIONS`); what stays here is /// call-site logic that isn't a function of the name alone — the DISTINCT /// modifier rule, the "reducer argument must be a bare column" rule -/// (`reducer_col`), φ extraction from a literal argument, and attaching -/// `accuracy` to approximate intents. `resolve_root` resolves `col` to a positional +/// (`reducer_col`) and φ extraction from a literal argument. `resolve_root` +/// resolves `col` to a positional /// `ColumnId`; the output name (DataFusion's own, e.g. /// `"sum(metrics.bytes)"`) is threaded separately as `Aggregate.output_names`, /// not carried here. -fn lower_agg_intent( - expr: &Expr, - accuracy: &AccuracyTarget, -) -> Result, LoweringError> { +fn lower_agg_intent(expr: &Expr) -> Result, LoweringError> { match expr { - Expr::Alias(a) => lower_agg_intent(&a.expr, accuracy), + Expr::Alias(a) => lower_agg_intent(&a.expr), Expr::AggregateFunction(agg_fn) => { let name = agg_fn.func.name().to_lowercase(); // ClickHouse's row-selecting `argMax`/`argMin` — `RewriteKind:: @@ -1782,12 +1764,9 @@ fn lower_agg_intent( } args => AggIntent::Cardinality { cols: args.iter().map(distinct_col).collect::>()?, - accuracy: accuracy.clone(), }, }, - AggSemantic::Count => AggIntent::Count { - accuracy: accuracy.clone(), - }, + AggSemantic::Count => AggIntent::Count, AggSemantic::Sum => AggIntent::Sum { col: col(&agg_fn.params.args)?, }, @@ -1827,11 +1806,9 @@ fn lower_agg_intent( _ => extract_percentile_q(&agg_fn.params.args)?, }, }, - accuracy: accuracy.clone(), }, AggSemantic::Cardinality => AggIntent::Cardinality { cols: vec![reducer_col(&name, &agg_fn.params.args)?], - accuracy: accuracy.clone(), }, }) } diff --git a/crates/frontend-sql/src/test_support.rs b/crates/frontend-sql/src/test_support.rs index c8de29b9c..aa599d562 100644 --- a/crates/frontend-sql/src/test_support.rs +++ b/crates/frontend-sql/src/test_support.rs @@ -1,6 +1,5 @@ //! Lowering steps only tests call, exposed under the `test-support` feature. use asap_frontend_common::UnresolvedOp; -use asap_types::ir::properties::AccuracyTarget; use crate::sql::{SqlCatalog, SqlLowerer}; use crate::SqlError; @@ -8,10 +7,6 @@ use crate::SqlError; /// `query` lowered to the name-based tree, parsed as /// `SqlDialect::DataFusionSQL`, before `resolve_root` binds it and before /// frequency recognition. -pub async fn lower_unresolved( - query: &str, - catalog: &SqlCatalog, - accuracy: &AccuracyTarget, -) -> Result { - SqlLowerer::new(catalog).lower(query, accuracy).await +pub async fn lower_unresolved(query: &str, catalog: &SqlCatalog) -> Result { + SqlLowerer::new(catalog).lower(query).await } diff --git a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs index 7185e8f8a..73312a7c2 100644 --- a/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs +++ b/crates/frontend-sql/tests/bgp_analytics/bgp_analytics.rs @@ -39,7 +39,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::workload::SqlDialect; @@ -96,13 +95,7 @@ fn queries() -> Vec { } async fn lower(q: &str) -> Result, LoweringError> { - lower_sql_dialect( - q, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await + lower_sql_dialect(q, &catalog(), SqlDialect::ClickhouseSQL).await } /// Pinned outcome for one corpus query. @@ -258,7 +251,7 @@ async fn top_k_queries_are_count_grouped_by_prefix() { idx + 1 ); assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "q{} ({label}) expected a single Count aggregate, got {measures:?}", idx + 1 ); diff --git a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs index bc7353f9e..9975ebb19 100644 --- a/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs +++ b/crates/frontend-sql/tests/bgp_jan2024_workload/bgp_jan2024_workload.rs @@ -14,7 +14,6 @@ //! tally** by outcome category -- see the module doc on [`Category`] for why. use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::workload::SqlDialect; use datafusion::error::DataFusionError; @@ -71,13 +70,7 @@ fn catalog() -> SqlCatalog { } async fn lower(q: &str) -> Result, SqlError> { - lower_sql_dialect( - q, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await + lower_sql_dialect(q, &catalog(), SqlDialect::ClickhouseSQL).await } /// Coarse outcome bucket for a corpus query. Deliberately coarser than diff --git a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs index 53ffbbfed..aae70b05a 100644 --- a/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs +++ b/crates/frontend-sql/tests/data_quality_check/synthetic_packet_trace.rs @@ -23,7 +23,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; @@ -125,7 +124,7 @@ fn has_window_func(node: &OperatorNode) -> bool { } async fn lower(q: &str) -> Rc { - lower_sql(q, &catalog(), AccuracyTarget::exact()) + lower_sql(q, &catalog()) .await .unwrap_or_else(|e| panic!("expected {q:?} to lower, got error: {e}")) } @@ -145,7 +144,7 @@ async fn corpus_lowering_covers_every_query() { let mut t = Tally::default(); for q in queries() { // A panic here (not an `Err`) fails the test — the totality guarantee. - match lower_sql(&q, &cat, AccuracyTarget::exact()).await { + match lower_sql(&q, &cat).await { Ok(_) => t.lowered += 1, // DataFusion surfaces parse/plan failures as `DataFusion(_)`. Err(LoweringError::DataFusion(_)) => t.unparseable += 1, @@ -183,7 +182,7 @@ async fn packet_count_is_count_intent() { let qe = lower("SELECT COUNT(*) AS total_packets FROM packets").await; let (by, measures) = first_aggregate(&qe).expect("expected an Aggregate"); assert!(by.is_empty(), "global count has no grouping"); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } #[tokio::test] @@ -224,7 +223,7 @@ async fn five_tuple_flow_group_by_binds_five_keys() { .await; let (by, measures) = first_aggregate(&qe).expect("expected an Aggregate"); assert_eq!(by.len(), 5, "grouped by the 5-tuple"); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } #[tokio::test] diff --git a/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs b/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs index a7ad616f1..f7965edf5 100644 --- a/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs +++ b/crates/frontend-sql/tests/data_quality_check/tpch_deequ.rs @@ -23,7 +23,6 @@ //! which is the same narrowing sidra's own catalog file makes. use asap_frontend_sql::{lower_sql, SqlCatalog, SqlError as LoweringError}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; const CORPUS: &str = include_str!("data/tpch_deequ_queries.sql"); @@ -85,7 +84,7 @@ async fn lowers_the_warehouse_ingestion_check_set() { let mut rejected = Vec::new(); let mut lowered = 0; for (id, query) in queries { - match lower_sql(query, &cat, AccuracyTarget::exact()).await { + match lower_sql(query, &cat).await { Ok(_) => lowered += 1, Err(LoweringError::UnsupportedAggregate(reason)) => rejected.push((id, reason)), Err(error) => panic!("unexpected failure for U-{id}: {error}"), diff --git a/crates/frontend-sql/tests/frequency_entropy.rs b/crates/frontend-sql/tests/frequency_entropy.rs index 7bd9962db..3c8d66973 100644 --- a/crates/frontend-sql/tests/frequency_entropy.rs +++ b/crates/frontend-sql/tests/frequency_entropy.rs @@ -4,7 +4,6 @@ use asap_frontend_common::resolve_root; use asap_frontend_sql::test_support::lower_unresolved; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; @@ -32,15 +31,8 @@ fn has_entropy(node: &OperatorNode) -> bool { #[tokio::test] async fn recognizes_sql_entropy_in_nats() { let sql = "SELECT -SUM(p * LN(p)) AS entropy FROM (SELECT COUNT(*) * 1.0 / SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f"; - let rewritten = lower_sql(sql, &catalog(false), AccuracyTarget::exact()) - .await - .unwrap(); - let original = resolve_root( - &lower_unresolved(sql, &catalog(false), &AccuracyTarget::exact()) - .await - .unwrap(), - ) - .unwrap(); + let rewritten = lower_sql(sql, &catalog(false)).await.unwrap(); + let original = resolve_root(&lower_unresolved(sql, &catalog(false)).await.unwrap()).unwrap(); assert!(has_entropy(&rewritten)); assert_eq!(original.schema, rewritten.schema); assert!(!has_entropy(&original)); @@ -59,7 +51,7 @@ async fn declines_non_equivalent_entropy_shapes() { ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), ("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", true), ] { - let root = lower_sql(sql, &catalog(nullable), AccuracyTarget::exact()) + let root = lower_sql(sql, &catalog(nullable)) .await .unwrap(); assert!(!has_entropy(&root), "{sql}"); @@ -70,8 +62,7 @@ async fn declines_non_equivalent_entropy_shapes() { /// is an exact count. #[tokio::test] async fn entropy_accuracy_and_population_guard_are_separate() { - let target = AccuracyTarget::max_error(0.05).failure_probability(0.01); - let node = lower_sql("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", &catalog(false), target.clone()).await.unwrap(); + let node = lower_sql("SELECT -SUM(p*LN(p)) FROM (SELECT COUNT(*)*1.0/SUM(COUNT(*)) OVER () AS p FROM flows GROUP BY src_ip) f", &catalog(false)).await.unwrap(); let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { panic!("project"); }; @@ -81,14 +72,9 @@ async fn entropy_accuracy_and_population_guard_are_separate() { let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { panic!("entropy"); }; - assert!( - matches!(&measures[0], AggIntent::FrequencyEntropy { accuracy, .. } if *accuracy == target) - ); + assert!(matches!(&measures[0], AggIntent::FrequencyEntropy { .. })); let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { panic!("count"); }; - assert!(matches!( - measures.as_slice(), - [AggIntent::Count { accuracy }] if accuracy.is_exact() - )); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index 2cdbead77..bd8261de3 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -4,7 +4,6 @@ use asap_frontend_common::resolve_root; use asap_frontend_sql::test_support::lower_unresolved; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::stages::HowToComputeAggregateMeasure; @@ -34,12 +33,7 @@ fn has_l2(node: &OperatorNode) -> bool { /// The relational DAG the frontend lowers before frequency recognition. async fn relational(sql: &str, catalog: &SqlCatalog) -> Rc { - resolve_root( - &lower_unresolved(sql, catalog, &AccuracyTarget::exact()) - .await - .unwrap(), - ) - .unwrap() + resolve_root(&lower_unresolved(sql, catalog).await.unwrap()).unwrap() } const CAST_PRODUCT: &str = "SELECT SQRT(SUM(CAST(c AS DOUBLE) * CAST(c AS DOUBLE))) AS norm FROM (SELECT src_ip, COUNT(*) AS c FROM flows WHERE keep GROUP BY src_ip) f"; @@ -50,9 +44,7 @@ const CAST_COUNT: &str = "SELECT SQRT(SUM(c * c)) AS norm FROM (SELECT src_ip, C #[tokio::test] async fn recognizes_float_frequency_l2() { for sql in [CAST_PRODUCT, CAST_COUNT] { - let rewritten = lower_sql(sql, &catalog(false), AccuracyTarget::exact()) - .await - .unwrap(); + let rewritten = lower_sql(sql, &catalog(false)).await.unwrap(); let original = relational(sql, &catalog(false)).await; assert!(has_l2(&rewritten), "{sql}"); assert_eq!(original.schema, rewritten.schema); @@ -71,7 +63,7 @@ async fn declines_non_equivalent_frequency_shapes() { ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(COUNT(*) AS DOUBLE) AS c FROM flows GROUP BY src_ip HAVING COUNT(*) > 1) f", false), ("SELECT SQRT(SUM(c*c)) FROM (SELECT src_ip, CAST(SUM(CAST(keep AS BIGINT)) AS DOUBLE) AS c FROM flows GROUP BY src_ip) f", false), ] { - let root = lower_sql(sql, &catalog(nullable), AccuracyTarget::exact()) + let root = lower_sql(sql, &catalog(nullable)) .await .unwrap(); assert!(!has_l2(&root), "{sql}"); @@ -81,10 +73,7 @@ async fn declines_non_equivalent_frequency_shapes() { /// The L2 intent carries the query's accuracy target, not a default. #[tokio::test] async fn frequency_l2_preserves_accuracy_target() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let node = lower_sql(CAST_COUNT, &catalog(false), target.clone()) - .await - .unwrap(); + let node = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { panic!("project"); }; @@ -94,17 +83,14 @@ async fn frequency_l2_preserves_accuracy_target() { let NonASAPOp::Aggregate { measures, .. } = left.expect_non_asap() else { panic!("aggregate"); }; - assert!(matches!(&measures[0], AggIntent::FrequencyL2 { accuracy, .. } if *accuracy == target)); + assert!(matches!(&measures[0], AggIntent::FrequencyL2 { .. })); } /// An approximate L2 estimate never decides whether SQL returns NULL: the /// empty-input guard is an exact count. #[tokio::test] async fn frequency_empty_input_guard_uses_an_exact_count() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let node = lower_sql(CAST_COUNT, &catalog(false), target) - .await - .unwrap(); + let node = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let NonASAPOp::Project { child, .. } = node.expect_non_asap() else { panic!("project"); }; @@ -114,10 +100,7 @@ async fn frequency_empty_input_guard_uses_an_exact_count() { let NonASAPOp::Aggregate { measures, .. } = right.expect_non_asap() else { panic!("count"); }; - assert!(matches!( - measures.as_slice(), - [AggIntent::Count { accuracy }] if accuracy.is_exact() - )); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } /// Stage 1 offers the exact option and summary candidates for the @@ -126,12 +109,14 @@ async fn frequency_empty_input_guard_uses_an_exact_count() { async fn stage1_offers_exact_and_summary_l2_candidates() { use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; use asap_types::ir::QueryRoot; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_sql(CAST_COUNT, &catalog(false), target) - .await - .unwrap(); + let root = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let l2 = inventory @@ -177,11 +162,7 @@ async fn example2_queries_lower_to_frequency_intents() { let q3_int = format!("SELECT SQRT(SUM(c * c)) FROM (SELECT src_ip, COUNT(*) AS c FROM flows {WINDOW} GROUP BY src_ip)"); let lower = |sql: String| { let catalog = &catalog; - async move { - lower_sql(&sql, catalog, AccuracyTarget::exact()) - .await - .unwrap() - } + async move { lower_sql(&sql, catalog).await.unwrap() } }; let q2 = lower(q2).await; assert!( diff --git a/crates/frontend-sql/tests/netflow/netflow.rs b/crates/frontend-sql/tests/netflow/netflow.rs index 596f911dd..033dcf153 100644 --- a/crates/frontend-sql/tests/netflow/netflow.rs +++ b/crates/frontend-sql/tests/netflow/netflow.rs @@ -8,7 +8,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; @@ -113,7 +112,7 @@ async fn netflow_sql_corpus_lowers_to_expected_intents() { for (idx, (query, expected)) in queries.iter().zip(EXPECTED).enumerate() { // A successful `lower_sql` already derived every node's schema. - let qe = lower_sql(query, &catalog(), AccuracyTarget::exact()) + let qe = lower_sql(query, &catalog()) .await .unwrap_or_else(|err| panic!("q{} failed to lower:\n{query}\n{err}", idx + 1)); assert!( @@ -146,7 +145,7 @@ fn assert_expected(qe: &Rc, expected: Expected, case_no: usize) { "q{case_no} expected count-ranked TopK({k}), got {qe:?}" ); assert!( - aggregate_by_with(qe, inner_by, |agg| matches!(agg, AggIntent::Count { .. })), + aggregate_by_with(qe, inner_by, |agg| matches!(agg, AggIntent::Count)), "q{case_no} expected inner Count grouped by {inner_by:?}, got {qe:?}" ); } diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 167d03fcb..dce78d103 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -3,7 +3,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode, ScalarExpr}; @@ -19,9 +18,7 @@ fn catalog() -> SqlCatalog { } async fn lower(sql: &str) -> Rc { - lower_sql(sql, &catalog(), AccuracyTarget::exact()) - .await - .unwrap() + lower_sql(sql, &catalog()).await.unwrap() } fn aggregate(query: &OperatorNode) -> (&[AggIntent], &OperatorNode) { @@ -111,12 +108,7 @@ async fn corr_rejects_unrepresented_forms() { "SELECT corr(x, y) OVER () FROM a", "SELECT corr(x) FROM a", ] { - assert!( - lower_sql(sql, &catalog(), AccuracyTarget::exact()) - .await - .is_err(), - "{sql}" - ); + assert!(lower_sql(sql, &catalog()).await.is_err(), "{sql}"); } } diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index ad9ba1367..9197318c3 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -16,7 +16,6 @@ use asap_types::ir::operator::{ AggIntent, GroupKeys, JoinKind, Reduction, Source, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, }; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ExprSemantics, NonASAPOp, OperatorNode, ScalarExpr}; @@ -52,7 +51,7 @@ fn catalog() -> SqlCatalog { } async fn lower(sql: &str) -> Rc { - lower_sql(sql, &catalog(), AccuracyTarget::exact()) + lower_sql(sql, &catalog()) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } @@ -64,7 +63,7 @@ fn op(node: &OperatorNode) -> &NonASAPOp { #[tokio::test] async fn planning_subquery_bridge_rejects_a_relation_without_vector_conversion() { - let result = lower_sql("SELECT max(value) FROM (SELECT asap_promql_subquery(21600000, 60000) AS value FROM (SELECT sum(bytes) AS value FROM metrics))", &catalog(), AccuracyTarget::exact()).await; + let result = lower_sql("SELECT max(value) FROM (SELECT asap_promql_subquery(21600000, 60000) AS value FROM (SELECT sum(bytes) AS value FROM metrics))", &catalog()).await; assert!(result.is_err()); } @@ -98,7 +97,6 @@ async fn planning_relation_bridges_reject_ambiguous_shapes() { let missing_alias = lower_sql( "SELECT asap_promql_subquery(300000, 60000) FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -107,7 +105,6 @@ async fn planning_relation_bridges_reject_ambiguous_shapes() { let histogram_with_extra_column = lower_sql( "SELECT service, asap_histogram_quantile(0.95) AS value FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -118,7 +115,6 @@ async fn planning_relation_bridges_reject_ambiguous_shapes() { let invalid_q = lower_sql( "SELECT asap_histogram_quantile(1.5) AS value FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -314,7 +310,7 @@ async fn count_ranked_topk_is_heavy_hitter() { let (inner_by, inner_measures) = find_aggregate(child).expect("expected inner Count aggregate"); assert_eq!(inner_by, &vec![1], "inner Count grouped by service → col 1"); assert!( - matches!(inner_measures.as_slice(), [AggIntent::Count { .. }]), + matches!(inner_measures.as_slice(), [AggIntent::Count]), "inner aggregate is the explicit Count, got {inner_measures:?}" ); } @@ -366,12 +362,12 @@ async fn filtered_count_ranking_is_not_a_heavy_hitter() { GROUP BY service ORDER BY c DESC LIMIT 10", "SELECT service, COUNT(bytes) AS c FROM metrics GROUP BY service ORDER BY c DESC LIMIT 10", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); assert!( - matches!(measures.as_slice(), [AggIntent::Count { .. }]), + matches!(measures.as_slice(), [AggIntent::Count]), "{sql}: the filtered Count must stay, got {measures:?}" ); let NonASAPOp::Aggregate { filters, .. } = op(find_aggregate_node(&qe).unwrap()) else { @@ -404,12 +400,7 @@ async fn non_count_ranked_limit_keeps_the_aggregate() { async fn distinct_value_reducer_is_rejected_not_dropped() { // The canonical intent algebra has no distinct-Sum; SUM(DISTINCT x) must // be rejected, not silently lowered as SUM(x). - let res = lower_sql( - "SELECT SUM(DISTINCT bytes) FROM metrics", - &catalog(), - AccuracyTarget::exact(), - ) - .await; + let res = lower_sql("SELECT SUM(DISTINCT bytes) FROM metrics", &catalog()).await; assert!(res.is_err(), "SUM(DISTINCT ...) should be rejected"); } @@ -438,7 +429,7 @@ async fn count_star_is_count_intent() { let qe = lower("SELECT COUNT(*) FROM metrics").await; let (by, measures) = find_aggregate(&qe).expect("expected an Aggregate"); assert!(by.is_empty()); - assert!(matches!(measures.as_slice(), [AggIntent::Count { .. }])); + assert!(matches!(measures.as_slice(), [AggIntent::Count])); } #[tokio::test] @@ -737,7 +728,6 @@ async fn a_multi_column_in_subquery_is_rejected() { let err = lower_sql( "SELECT service FROM metrics WHERE service IN (SELECT service, region FROM hosts)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("IN must select one column"); @@ -962,7 +952,6 @@ async fn groups_frame_is_rejected() { "SELECT service, SUM(latency) OVER (PARTITION BY service ORDER BY ts \ GROUPS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM metrics", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("GROUPS frame unit must be rejected"); @@ -1167,7 +1156,6 @@ async fn a_correlated_in_subquery_is_rejected() { "SELECT service FROM metrics m WHERE service IN \ (SELECT h.service FROM hosts h WHERE h.region = m.service)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("correlated IN needs both a key match and a correlation"); @@ -1179,7 +1167,7 @@ async fn a_correlated_in_subquery_is_rejected() { /// The name-based tree, before `resolve_root`. async fn lower_unresolved(sql: &str) -> UnresolvedOp { let catalog = catalog(); - asap_frontend_sql::test_support::lower_unresolved(sql, &catalog, &AccuracyTarget::exact()) + asap_frontend_sql::test_support::lower_unresolved(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } @@ -1451,27 +1439,6 @@ async fn median_is_the_same_intent_as_an_explicit_half_percentile() { assert_eq!(m_measures, p_measures); } -#[tokio::test] -async fn median_threads_the_accuracy_target() { - // The `approx_` prefix does not decide: the AccuracyTarget does. - let qe = lower_sql( - "SELECT approx_median(latency) FROM metrics", - &catalog(), - AccuracyTarget::max_error(0.01), - ) - .await - .expect("approx_median should lower"); - let (_, measures) = find_aggregate(&qe).expect("expected an Aggregate"); - assert!( - matches!( - measures.as_slice(), - [AggIntent::Quantile { accuracy, .. }] - if (accuracy.max_error - 0.01).abs() < 1e-12 - ), - "median must carry the workload's accuracy target, got {measures:?}" - ); -} - #[tokio::test] async fn median_over_an_expression_binds_the_derived_column() { // Was rejected when filed (#111); supported since #110 materialized the @@ -1708,7 +1675,6 @@ async fn grouping_function_is_rejected() { let err = lower_sql( "SELECT service, SUM(bytes), GROUPING(service) FROM metrics GROUP BY ROLLUP(service)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("GROUPING() must be rejected while __grouping_id is dropped"); @@ -1722,7 +1688,6 @@ async fn a_non_column_key_inside_a_grouping_set_is_rejected() { let err = lower_sql( "SELECT date_trunc('minute', ts) AS m, SUM(bytes) FROM metrics GROUP BY ROLLUP(m)", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("expression key inside ROLLUP must be rejected"); @@ -1766,7 +1731,6 @@ async fn an_ambiguous_passthrough_column_is_rejected_only_when_projecting() { "SELECT m.service, h.service, SUM(m.bytes) FROM metrics m \ JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", &catalog(), - AccuracyTarget::exact(), ) .await; assert!( @@ -1778,7 +1742,6 @@ async fn an_ambiguous_passthrough_column_is_rejected_only_when_projecting() { "SELECT m.service, h.service, SUM(m.bytes * 2) FROM metrics m \ JOIN hosts h ON m.service = h.service GROUP BY m.service, h.service", &catalog(), - AccuracyTarget::exact(), ) .await .expect_err("ambiguous passthrough must be rejected, not silently resolved"); @@ -1798,13 +1761,9 @@ async fn array_agg_is_deliberately_rejected() { // feed the sketch path. // // This test exists so the rejection reads as a decision rather than a gap. - let err = lower_sql( - "SELECT array_agg(service) FROM metrics", - &catalog(), - AccuracyTarget::exact(), - ) - .await - .expect_err("array_agg must not lower to an intent"); + let err = lower_sql("SELECT array_agg(service) FROM metrics", &catalog()) + .await + .expect_err("array_agg must not lower to an intent"); assert!( format!("{err}").contains("unsupported aggregate: array_agg"), "expected a clean UnsupportedAggregate, got {err}" @@ -1860,14 +1819,9 @@ async fn clickhouse_identifiers_keep_their_case() { // uniqExact from #221) ─────────────────────────────────────────────────── async fn lower_clickhouse(sql: &str) -> Rc { - lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) + lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } fn temporal_aggregate(node: &OperatorNode) -> (&AggIntent, std::time::Duration, &OperatorNode) { @@ -1923,14 +1877,9 @@ async fn temporal_aggregate_rejects_non_timestamp_and_non_positive_window() { "SELECT asap_rate(latency, ts, 0) FROM metrics", "SELECT asap_rate(latency, ts, bytes) FROM metrics", ] { - let err = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("invalid temporal arguments must fail closed"); + let err = lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .expect_err("invalid temporal arguments must fail closed"); assert!( format!("{err}").contains("timestamp argument") || format!("{err}").contains("window_ms"), @@ -1945,7 +1894,6 @@ async fn temporal_aggregate_rejects_mixed_reducers() { "SELECT asap_rate(latency, ts, 300000), sum(bytes) FROM metrics", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("one child cannot carry temporal and ordinary aggregate semantics"); @@ -1958,7 +1906,6 @@ async fn last_fails_closed_until_an_executable_summary_exists() { "SELECT service, asap_last(latency, ts, 300000) FROM metrics GROUP BY service", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("last must not be advertised without an executable physical summary"); @@ -1984,14 +1931,9 @@ async fn temporal_grouping_requires_the_complete_declared_series_identity() { "SELECT asap_rate(value, ts, 300000) FROM samples", "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", ] { - let err = lower_sql_dialect( - sql, - &multi_series, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("partial identity must not merge counter series"); + let err = lower_sql_dialect(sql, &multi_series, SqlDialect::ClickhouseSQL) + .await + .expect_err("partial identity must not merge counter series"); assert!(format!("{err}").contains("declared series identity")); } @@ -2000,7 +1942,6 @@ async fn temporal_grouping_requires_the_complete_declared_series_identity() { FROM samples GROUP BY service, instance", &multi_series, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect("the complete declared series identity is safe"); @@ -2021,7 +1962,6 @@ async fn temporal_grouping_requires_the_complete_declared_series_identity() { "SELECT service, asap_rate(value, ts, 300000) FROM samples GROUP BY service", &row_id_only, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("a row key without time does not prove a series identity"); @@ -2035,14 +1975,9 @@ async fn temporal_grouping_rejects_value_time_and_duplicate_resolved_columns() { "SELECT m.service, asap_rate(m.latency, m.ts, 300000) \ FROM metrics m GROUP BY m.service, service", ] { - let err = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("unsafe or duplicate resolved grouping must fail closed"); + let err = lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .expect_err("unsafe or duplicate resolved grouping must fail closed"); let message = format!("{err}"); assert!( message.contains("timestamp or value") @@ -2231,7 +2166,6 @@ async fn arg_max_rejects_a_non_column_argument() { "SELECT argMax(service, latency * 2) FROM metrics", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect_err("argMax over a non-column expression must be rejected"); @@ -2370,7 +2304,7 @@ async fn count_null_semantics_become_a_measure_filter() { "SELECT count(value) FROM samples", "SELECT count(value + 1) FROM samples", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|error| panic!("{sql}: {error}")); assert!( @@ -2383,7 +2317,7 @@ async fn count_null_semantics_become_a_measure_filter() { "SELECT count(NULL) FROM samples", "SELECT count(nullable_value + 1) FROM samples", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|error| panic!("{sql}: {error}")); let [Some(Predicate(cond))] = aggregate_filters(&qe) else { @@ -2398,7 +2332,6 @@ async fn count_null_semantics_become_a_measure_filter() { let qe = lower_sql( "SELECT count(*), count(nullable_value) FROM samples", &catalog, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2406,7 +2339,6 @@ async fn count_null_semantics_become_a_measure_filter() { let error = lower_sql( "SELECT count(nullable_value) FROM samples GROUP BY ROLLUP(value)", &catalog, - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -2435,7 +2367,6 @@ async fn grouped_map_column_preserves_map_type() { "SELECT labels, max(value) AS value FROM raw_samples GROUP BY labels ORDER BY labels", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2464,7 +2395,6 @@ async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { &format!("SELECT {call} AS value FROM numbers"), &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2472,7 +2402,6 @@ async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { &format!("SELECT {native} AS value FROM numbers"), &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2483,7 +2412,6 @@ async fn clickhouse_modulo_uses_native_arithmetic_types_and_nullability() { "SELECT modulo(n, 3) AS value FROM numbers", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap() @@ -2519,14 +2447,9 @@ async fn original_o11y_map_queries_lower_with_typed_results() { include_str!("data/o11y_q09.sql"), include_str!("data/o11y_q12.sql"), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap_or_else(|e| panic!("{sql}: {e}")); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap_or_else(|e| panic!("{sql}: {e}")); let schema = &query.schema; assert!( schema @@ -2551,14 +2474,9 @@ async fn clickhouse_modulo_preserves_projection_names_and_outer_references() { "modulo(bytes, 3)", ), ] { - let query = lower_sql_dialect( - sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .unwrap(); assert_eq!(query.schema.fields[0].name, name); } } @@ -2585,7 +2503,6 @@ async fn clickhouse_map_access_keeps_generated_names_and_rejects_variant_coercio "SELECT labels['job'] FROM t", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap(); @@ -2597,15 +2514,13 @@ async fn clickhouse_map_access_keeps_generated_names_and_rejects_variant_coercio "SELECT map()['a'] FROM t", &catalog, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .is_err()); assert!(lower_sql_dialect( "SELECT map('a', integer, 'b', floating) FROM t", &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact() + SqlDialect::ClickhouseSQL ) .await .is_err()); @@ -2633,14 +2548,9 @@ async fn arg_selector_result_schema_tracks_selected_argument() { true, ), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap(); let schema = &query.schema; assert_eq!(schema.fields[0].dtype, dtype); assert_eq!(schema.fields[0].nullable, nullable); @@ -2667,14 +2577,9 @@ async fn clickhouse_list_element_uses_canonical_typed_access() { ("SELECT arrayElement(samples, -1) AS selected FROM t", false), ("SELECT samples[index] AS selected FROM t", true), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap(); let output = &query.schema; assert_eq!(output.fields[0].dtype, DataType::Int64); assert_eq!(output.fields[0].nullable, nullable); @@ -2683,14 +2588,9 @@ async fn clickhouse_list_element_uses_canonical_typed_access() { } for sql in ["SELECT samples[0] FROM t", "SELECT samples['bad'] FROM t"] { assert!( - lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact() - ) - .await - .is_err(), + lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .is_err(), "{sql}" ); } @@ -2726,14 +2626,9 @@ async fn clickhouse_tuple_element_preserves_declared_field_metadata() { true, ), ] { - let query = lower_sql_dialect( - sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let query = lower_sql_dialect(sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .unwrap(); let output = &query.schema; assert_eq!(output.fields[0].dtype, dtype); assert_eq!(output.fields[0].nullable, nullable); @@ -2744,14 +2639,9 @@ async fn clickhouse_tuple_element_preserves_declared_field_metadata() { for selector in ["0", "-1", "3", "'missing'", "index"] { let sql = format!("SELECT tupleElement(sample, {selector}) FROM t"); assert!( - lower_sql_dialect( - &sql, - &catalog, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact() - ) - .await - .is_err(), + lower_sql_dialect(&sql, &catalog, SqlDialect::ClickhouseSQL) + .await + .is_err(), "{sql}" ); } @@ -2778,13 +2668,9 @@ async fn composite_distinct_counts_tuples() { Field::plain("b", DataType::Int64, false), ]), ); - let composite = lower_sql( - "SELECT COUNT(DISTINCT a, b) FROM t", - &cat, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let composite = lower_sql("SELECT COUNT(DISTINCT a, b) FROM t", &cat) + .await + .unwrap(); let NonASAPOp::Aggregate { measures, .. } = op(find_aggregate_node(&composite).expect("expected an Aggregate")) else { @@ -2795,13 +2681,9 @@ async fn composite_distinct_counts_tuples() { "{measures:?}" ); - let single = lower_sql( - "SELECT COUNT(DISTINCT a) FROM t", - &cat, - AccuracyTarget::exact(), - ) - .await - .unwrap(); + let single = lower_sql("SELECT COUNT(DISTINCT a) FROM t", &cat) + .await + .unwrap(); let NonASAPOp::Aggregate { measures, .. } = op(find_aggregate_node(&single).expect("expected an Aggregate")) else { @@ -2824,13 +2706,9 @@ async fn composite_distinct_rejects_expression_arguments() { Field::plain("b", DataType::Int64, false), ]), ); - let error = lower_sql( - "SELECT COUNT(DISTINCT a, b + 1) FROM t", - &cat, - AccuracyTarget::exact(), - ) - .await - .unwrap_err(); + let error = lower_sql("SELECT COUNT(DISTINCT a, b + 1) FROM t", &cat) + .await + .unwrap_err(); assert!( matches!(&error, LoweringError::UnsupportedAggregate(reason) if reason.contains("non-column expression")), @@ -2853,7 +2731,7 @@ async fn distinct_with_derived_sibling() { "SELECT count(DISTINCT a, b), sum(b + 1) FROM t", "SELECT count(DISTINCT a, b), corr(a,b) FROM t", ] { - let result = lower_sql(sql, &catalog, AccuracyTarget::exact()).await; + let result = lower_sql(sql, &catalog).await; assert!(result.is_ok(), "{sql}: {result:?}"); } } @@ -2886,7 +2764,7 @@ async fn conditional_count_lowers_to_a_filtered_measure() { assert!( matches!( measures.as_slice(), - [AggIntent::Count { .. }, AggIntent::Sum { col: Some(3) }] + [AggIntent::Count, AggIntent::Sum { col: Some(3) }] ), "{measures:?}" ); @@ -2938,7 +2816,7 @@ async fn count_of_a_nullable_expression_filters_nulls() { assert!( matches!( find_aggregate(&qe).unwrap().1.as_slice(), - [AggIntent::Count { .. }] + [AggIntent::Count] ), "still a row count" ); @@ -2977,7 +2855,6 @@ async fn measure_filter_inside_a_rollup_is_rejected() { let err = lower_sql( "SELECT service, count(*) FILTER (WHERE latency > 1.0) FROM metrics GROUP BY ROLLUP(service)", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap_err(); @@ -2985,7 +2862,7 @@ async fn measure_filter_inside_a_rollup_is_rejected() { } async fn lower_err(sql: &str) -> String { - lower_sql(sql, &catalog(), AccuracyTarget::exact()) + lower_sql(sql, &catalog()) .await .err() .unwrap_or_else(|| panic!("expected {sql:?} to be rejected")) @@ -3137,7 +3014,6 @@ async fn duration_in_a_case_branch_is_rejected_as_a_duration() { let err = asap_frontend_sql::lower_sql( "SELECT CASE WHEN true THEN INTERVAL '1 day' ELSE d - d END FROM t", &catalog, - AccuracyTarget::exact(), ) .await .unwrap_err() @@ -3285,7 +3161,7 @@ async fn other_time_conditions_stay_scan_predicates() { WHERE ts >= now() - INTERVAL '5 minutes' AND ts < now() - INTERVAL '1 minute'", "SELECT COUNT(*) FROM events WHERE ts >= now() - INTERVAL '5 minutes'", ] { - let qe = lower_sql(sql, &catalog, AccuracyTarget::exact()) + let qe = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("{sql}: {e}")); assert!(trailing_window(&qe).is_none(), "{sql}: {qe:#?}"); diff --git a/crates/frontend-sql/tests/temporal_types.rs b/crates/frontend-sql/tests/temporal_types.rs index e0380c31b..754812680 100644 --- a/crates/frontend-sql/tests/temporal_types.rs +++ b/crates/frontend-sql/tests/temporal_types.rs @@ -1,5 +1,4 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; fn catalog() -> SqlCatalog { SqlCatalog::new().with_table( @@ -13,13 +12,9 @@ async fn temporal_subtraction_rejects_unrepresentable_duration() { for dtype in [DataType::Date, DataType::Timestamp] { let catalog = SqlCatalog::new().with_table("t", Schema::new(vec![Field::plain("d", dtype, false)])); - let error = lower_sql( - "SELECT d - d AS elapsed FROM t", - &catalog, - AccuracyTarget::exact(), - ) - .await - .unwrap_err(); + let error = lower_sql("SELECT d - d AS elapsed FROM t", &catalog) + .await + .unwrap_err(); assert!( error.to_string().contains("unsupported duration type"), "{error}" @@ -33,9 +28,7 @@ async fn date_shifts_keep_their_type() { "SELECT d + INTERVAL '30' DAY AS shifted FROM t", "SELECT d - CAST('1 day' AS INTERVAL) AS shifted FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Date); } } @@ -46,9 +39,7 @@ async fn interval_cast_lowers_like_interval_literal() { "SELECT INTERVAL '1' DAY AS duration FROM t", "SELECT CAST('1 day' AS INTERVAL) AS duration FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Interval); } } @@ -62,7 +53,7 @@ async fn nested_temporal_subtraction_is_rejected() { "SELECT CASE WHEN true THEN INTERVAL '1 day' ELSE d - d END FROM t", "SELECT d FROM t ORDER BY d - d", ] { - let result = lower_sql(query, &catalog(), AccuracyTarget::exact()).await; + let result = lower_sql(query, &catalog()).await; let error = result.expect_err(&format!("accepted unsupported duration: {query}")); assert!( error.to_string().contains("unsupported duration type"), @@ -79,9 +70,7 @@ async fn negative_intervals_keep_their_type() { "SELECT -CAST('1 day' AS INTERVAL) AS duration FROM t", "SELECT -(INTERVAL '1 day' + INTERVAL '2 days') AS duration FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Interval, "{query}"); } } @@ -93,9 +82,7 @@ async fn sql_date_literals_keep_their_type() { "SELECT DATE '2024-02-29' FROM t", "SELECT DATE '1969-12-31' + INTERVAL '1 day' FROM t", ] { - let node = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let node = lower_sql(query, &catalog()).await.unwrap(); assert_eq!(node.schema.fields[0].dtype, DataType::Date); } } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index aa8fe4680..192a32ab6 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -112,6 +112,7 @@ pub mod physical_export { /// `run_planning`) for one /// query, with the built-in models. pub mod planning { + use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot}; use asap_types::workload::{DataArrival, DataWorkload, SourceFacts}; @@ -188,12 +189,54 @@ pub mod planning { /// The logical DAG the planner pipeline selects for the single query /// `root` at `accuracy`. pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { - match run_all_stages(root, accuracy).run.selected_plan.as_deref() { - Some([QueryRoot::Operator(node)]) => Rc::clone(node), + match selected_operator_roots(&run_all_stages(root, accuracy)).as_slice() { + [node] => Rc::clone(node), other => panic!("one selected operator root: {other:?}"), } } + /// `plan`'s cost per second (`asap_cost::plan_cost`) for queries with + /// `entries` over `data`, under `settings`' cost model 1 or 2. + pub fn plan_cost_per_second( + plan: &PhysicalASAPDAG, + entries: &[asap_types::workload::QueryWorkloadEntry], + data: &DataWorkload, + settings: &asap_cost::CostInputs, + ) -> f64 { + let recurrence: Vec<_> = entries + .iter() + .map(|entry| entry.recurrence.clone()) + .collect(); + let range_evaluations: Vec<_> = + entries.iter().map(|entry| entry.range_evaluation).collect(); + let (cost, _) = asap_cost::plan_cost( + plan, + &asap_cost::node_map(plan), + &recurrence, + &range_evaluations, + data, + settings, + 1.0, + ) + .unwrap(); + cost.cost_per_second.expect("cost model 1 or 2") + } + + /// The operator roots of the plan Stage 3 built in `run`, as IR. + fn selected_operator_roots( + run: &asap_devtools::planning_stages::PlanningStagesRun, + ) -> Vec> { + let (plan, _) = run.run.selection.as_ref().expect("a plan is selected"); + plan.query_roots() + .unwrap() + .into_iter() + .filter_map(|root| match root { + QueryRoot::Operator(node) => Some(node), + QueryRoot::Scalar(_) => None, + }) + .collect() + } + /// Exact computation and each explicit single-target rule application. /// This fixture catalog is independent of the bounded MILP's visited plans. pub fn explicit_local_candidates(root: Rc) -> Vec> { @@ -201,7 +244,12 @@ pub mod planning { compose_logical_candidate, generate_per_computation_candidates, }; let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); @@ -237,7 +285,12 @@ pub mod planning { }; use asap_types::stages::HowToComputeAggregateMeasure; let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let choices = local.targets.iter().map(|target| target.ways_to_compute.iter() @@ -251,50 +304,27 @@ pub mod planning { } } - /// Logical plans visited by the bounded search for `root`, in archive order. - /// This is not the generated candidate catalog. Rule-existence tests should - /// use explicit fixtures; failed compositions in this archive still panic. + /// The selected plan's operator roots for `root`. This is not the + /// generated candidate catalog: rule-existence tests should use explicit + /// fixtures. pub fn logical_candidates( root: Rc, accuracy: AccuracyTarget, ) -> Vec> { - require_composed_operator_candidates(run_all_stages(root, accuracy).logical_candidates()) + selected_operator_roots(&run_all_stages(root, accuracy)) } - /// [`logical_candidates`] with every output accepted whatever its - /// accuracy guarantee (`ignore_accuracy_guarantees`), so Pass 1 keeps - /// the summaries no query can use for `accuracy` - /// (`PruneReason::NoErrorBoundForQueryTarget`). For tests of a - /// rewrite's structure or execution over a summary whose output has no - /// guarantee that meets `accuracy` (e.g. a value filter over an - /// estimate), not of accuracy acceptance. + /// [`logical_candidates`] with accuracy not enforced + /// (`ignore_accuracy_guarantees`): no readout gets a required accuracy. + /// For tests of a rewrite's structure or execution over a summary whose + /// output has no guarantee that meets `accuracy` (e.g. a value filter + /// over an estimate), not of accuracy acceptance. pub fn logical_candidates_ignoring_guarantees( root: Rc, accuracy: AccuracyTarget, ) -> Vec> { let models = asap_planner::PlanningModels::builtin().with_ignore_accuracy_guarantees(true); - require_composed_operator_candidates( - run_all_stages_with(root, accuracy, models).logical_candidates(), - ) - } - - /// Fail on rejected composition rather than silently skipping a candidate. - pub fn require_composed_operator_candidates( - candidates: Vec, - ) -> Vec> { - candidates - .into_iter() - .filter_map(|candidate| { - match candidate - .roots - .unwrap_or_else(|e| panic!("{} does not compose: {e}", candidate.id)) - .remove(0) - { - QueryRoot::Operator(node) => Some(node), - QueryRoot::Scalar(_) => None, - } - }) - .collect() + selected_operator_roots(&run_all_stages_with(root, accuracy, models)) } /// Whether `node` builds a summary. diff --git a/crates/integration-tests/tests/aggregate.rs b/crates/integration-tests/tests/aggregate.rs index 8aa849b8c..a781effa8 100644 --- a/crates/integration-tests/tests/aggregate.rs +++ b/crates/integration-tests/tests/aggregate.rs @@ -83,13 +83,7 @@ fn q06_sum_by_job() { fn q07_count_is_row_count() { assert_eq!( lower("count(http_requests_total)"), - agg( - vec![], - AggIntent::Count { - accuracy: AccuracyTarget::exact() - }, - scan("http_requests_total", &[]), - ), + agg(vec![], AggIntent::Count, scan("http_requests_total", &[]),), ); } @@ -184,11 +178,7 @@ fn q10_quantile_cross_series() { lower("quantile(0.99, http_requests_total)"), agg( vec![], - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::exact() - }, + AggIntent::Quantile { col: None, q: 0.99 }, scan("http_requests_total", &[]), ), ); diff --git a/crates/integration-tests/tests/filtered_aggregates.rs b/crates/integration-tests/tests/filtered_aggregates.rs index 40c52db98..440b0cd79 100644 --- a/crates/integration-tests/tests/filtered_aggregates.rs +++ b/crates/integration-tests/tests/filtered_aggregates.rs @@ -112,16 +112,13 @@ fn execute(root: &Rc) -> Vec { out } -async fn lower(sql: &str, accuracy: AccuracyTarget) -> Rc { - lower_sql(sql, &catalog(), accuracy).await.unwrap() +async fn lower(sql: &str) -> Rc { + lower_sql(sql, &catalog()).await.unwrap() } /// The selected plan for an exact `sql`, through all three stages. async fn exact_plan(sql: &str) -> Rc { - selected_dag( - lower(sql, AccuracyTarget::exact()).await, - AccuracyTarget::exact(), - ) + selected_dag(lower(sql).await, AccuracyTarget::exact()) } /// `COUNT(*) FILTER (WHERE x > 0) GROUP BY g` counts the matching rows per @@ -154,20 +151,18 @@ async fn exact_count_of_nullable_column_skips_nulls() { fn with_summary_filter(root: &Rc, filter: &Predicate) -> Rc { if let Operator::ASAP(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, - grouping, filter: None, }) = &root.operator { return Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: child.clone(), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: Some(filter.clone()), })) .unwrap(), @@ -181,7 +176,7 @@ fn with_summary_filter(root: &Rc, filter: &Predicate) -> Rc (Predicate, Schema) { - OperatorNode::reachable(&lower(sql, AccuracyTarget::exact()).await) + OperatorNode::reachable(&lower(sql).await) .into_iter() .find_map(|n| match n.non_asap() { Some(NonASAPOp::Aggregate { filters, child, .. }) => { @@ -204,7 +199,7 @@ async fn hand_filtered( ) -> Vec> { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); let mut result = vec![]; - for root in logical_candidates_ignoring_guarantees(lower(sql, target.clone()).await, target) { + for root in logical_candidates_ignoring_guarantees(lower(sql).await, target) { let builds: Vec<_> = OperatorNode::reachable(&root) .into_iter() .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) @@ -213,7 +208,7 @@ async fn hand_filtered( continue; }; let Some(ASAPOp::SummaryAgg { - family: actual, + summary_type: actual, child, .. }) = build.asap() @@ -258,7 +253,7 @@ async fn hand_filtered_kll_compiles_and_executes() { .await; let roots = hand_filtered( "SELECT g, approx_percentile_cont(x, 0.5) AS q FROM events GROUP BY g", - |f| matches!(f, FieldDataType::Sketch(kind, _) if kind == &SketchAlgorithm::Kll), + |f| matches!(f, FieldDataType::Sketch(kind) if kind == &SketchAlgorithm::Kll), &filter, ) .await; @@ -272,15 +267,16 @@ async fn hand_filtered_kll_compiles_and_executes() { /// [`hand_filtered`]). async fn kll_candidates(sql: &str) -> Vec> { let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let roots: Vec<_> = logical_candidates_ignoring_guarantees(lower(sql, target.clone()).await, target) - .into_iter() - .filter(|root| { - OperatorNode::reachable(root).iter().any(|n| { - matches!(n.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) + let roots: Vec<_> = + logical_candidates_ignoring_guarantees(lower(sql).await, target) + .into_iter() + .filter(|root| { + OperatorNode::reachable(root).iter().any(|n| { + matches!(n.asap(), Some(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) if kind == &SketchAlgorithm::Kll) - }) }) - .collect(); + }) + .collect(); assert!(!roots.is_empty(), "no KLL candidate for {sql}"); roots } diff --git a/crates/integration-tests/tests/finer_aggregate_reuse.rs b/crates/integration-tests/tests/finer_aggregate_reuse.rs index 443181401..9ce0534a7 100644 --- a/crates/integration-tests/tests/finer_aggregate_reuse.rs +++ b/crates/integration-tests/tests/finer_aggregate_reuse.rs @@ -54,8 +54,9 @@ fn cost(queries: &[String]) -> f64 { PlanSearchSettings::default(), ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - let value = selection.cost.cost_per_second.unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); println!("cost={value}"); value } diff --git a/crates/integration-tests/tests/frontend_timestamps.rs b/crates/integration-tests/tests/frontend_timestamps.rs index 4890c85fb..54703da9a 100644 --- a/crates/integration-tests/tests/frontend_timestamps.rs +++ b/crates/integration-tests/tests/frontend_timestamps.rs @@ -31,13 +31,9 @@ async fn promql_eval_time_and_sql_current_timestamp_remain_distinct() { "metrics", Schema::new(vec![Field::plain("value", DataType::Float64, false)]), ); - let sql = lower_sql( - "SELECT CURRENT_TIMESTAMP FROM metrics", - &catalog, - AccuracyTarget::exact(), - ) - .await - .expect("lower SQL CURRENT_TIMESTAMP"); + let sql = lower_sql("SELECT CURRENT_TIMESTAMP FROM metrics", &catalog) + .await + .expect("lower SQL CURRENT_TIMESTAMP"); let Some(NonASAPOp::Project { cols, child, .. }) = sql.non_asap() else { panic!("expected SQL projection, got {sql:?}"); }; diff --git a/crates/integration-tests/tests/identical_exact_queries.rs b/crates/integration-tests/tests/identical_exact_queries.rs index 9a31850a3..95b550dc3 100644 --- a/crates/integration-tests/tests/identical_exact_queries.rs +++ b/crates/integration-tests/tests/identical_exact_queries.rs @@ -37,8 +37,9 @@ fn cost(w: &FuzzWorkload) -> f64 { PlanSearchSettings::default(), ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - let value = selection.cost.cost_per_second.unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); println!("cost={value}"); value } diff --git a/crates/integration-tests/tests/independent_plan_seed.rs b/crates/integration-tests/tests/independent_plan_seed.rs index 77767994d..4faffc3a5 100644 --- a/crates/integration-tests/tests/independent_plan_seed.rs +++ b/crates/integration-tests/tests/independent_plan_seed.rs @@ -53,9 +53,8 @@ fn cost(queries: &[String]) -> f64 { PlanSearchSettings::default(), ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - println!("{queries:?}: status {:?}", selection.search_report.status); - selection.cost.cost_per_second.unwrap() + let (plan, _) = run.run.selection.as_ref().unwrap(); + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings) } /// Asserts the joint plan of `queries` costs no more than their separate diff --git a/crates/integration-tests/tests/joint_search_equivalence.rs b/crates/integration-tests/tests/joint_search_equivalence.rs index 3099cc6e7..af738cc21 100644 --- a/crates/integration-tests/tests/joint_search_equivalence.rs +++ b/crates/integration-tests/tests/joint_search_equivalence.rs @@ -7,12 +7,11 @@ use crate::planner_layering_common; -use asap_accuracy::{AccuracyModel, BuiltinAccuracyModel}; use asap_cost::{ClusterLimits, ClusterObjective, CostInputs, CostModel}; use asap_devtools::planning_stages::{lower_promql_roots, lower_sql_roots}; use asap_planner::{run_planning, PlanSearchSettings, PlanningModels, PlanningRun}; use asap_types::deployment_capabilities::DeploymentCapabilities; -use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::QueryRoot; use asap_types::workload::{ DataArrival, DataWorkload, DurationMs, PlanningWorkload, Predictability, PrometheusMetricType, @@ -87,28 +86,13 @@ fn run( } fn assert_valid(run: &PlanningRun, queries: usize) { - let selection = run + let (plan, selection) = run .selection .as_ref() .unwrap_or_else(|error| panic!("joint planning failed: {error}")); - let physical = &run - .selected_physical_plan - .as_ref() - .expect("successful selection has a physical plan") - .plan; - physical.dag.validate().unwrap(); - assert_eq!(physical.roots.len(), queries); + plan.validate().unwrap(); + assert_eq!(plan.roots.len() + plan.scalar_roots.len(), queries); assert_eq!(selection.accepted_accuracy_per_query.len(), queries); - let report = &selection.search_report; - if let (Some(bound), Some(cost)) = ( - report.cost_lower_bound_per_second, - selection.cost.cost_per_second, - ) { - assert!( - bound <= cost + 1e-8 * cost.abs().max(1e-12), - "reported bound {bound} exceeds validated cost {cost}: {report:?}" - ); - } } fn run_workload( @@ -143,12 +127,10 @@ fn repeated_queries_preserve_all_outputs_and_exact_guarantees() { AccuracyTarget::exact(), ); let run = run_workload(&workload, &settings, &deployment); - let selection = run.selection.as_ref().unwrap(); + let (_, selection) = run.selection.as_ref().unwrap(); let accepted = &selection.accepted_accuracy_per_query; assert_eq!(accepted.len(), count); for accuracy in accepted { - assert!(!accuracy.accepted_regardless_of_guarantee); - assert_eq!(accuracy.guarantee_check_rejection, None); assert_eq!(accuracy.accepted_metric, None, "every output remains exact"); } } @@ -229,35 +211,6 @@ fn composed_quantile_and_sum_outputs_are_validated_together() { ); } -// A custom accuracy policy still participates in final evaluation and prevents -// builtin-only permanent pruning assumptions from being applied to its outputs. -#[test] -fn custom_accuracy_policy_is_respected() { - struct CustomAccuracy(std::cell::Cell); - impl AccuracyModel for CustomAccuracy { - fn satisfies(&self, guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - self.0.set(self.0.get() + 1); - BuiltinAccuracyModel.satisfies(guarantee, target) - } - } - let workload = workload_of(&[("sum_over_time(m[1m])", 60_000)], AccuracyTarget::exact()); - let roots = lower_promql_roots(&workload).unwrap(); - let entries: Vec<_> = workload.query_workload.entries().collect(); - let policy = CustomAccuracy(std::cell::Cell::new(0)); - let result = run( - &roots, - &entries, - workload.data_workload.as_ref().unwrap(), - PlanningModels::builtin().with_accuracy(&policy), - false, - ); - assert_valid(&result, 1); - assert!( - policy.0.get() > 0, - "the supplied accuracy policy must evaluate outputs" - ); -} - // The SQL frontend keeps repeated and different-window COUNT DISTINCT outputs. #[test] fn repeated_sql_queries_preserve_separate_output_references() { @@ -309,7 +262,6 @@ fn rootless_request_overhead_respects_capacity_and_latency() { false, ); assert!(result.selection.is_err()); - assert!(result.selected_physical_plan.is_none()); } } @@ -331,7 +283,6 @@ fn unsupported_exact_operation_never_returns_an_unvalidated_plan() { false, ); assert!(result.selection.is_err()); - assert!(result.selected_physical_plan.is_none()); } // Deterministic small mixed workloads exercise exact temporal operators, counter diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index dffe01447..35063aa25 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -16,8 +16,7 @@ use asap_types::ir::operator::{GroupKeys, Reduction, Source}; use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - DataType, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, SketchStatistic, - SummaryUpdate, + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use futures::{executor::block_on, StreamExt}; @@ -32,7 +31,7 @@ use std::{ const TUMBLING_WINDOW_WIDTH_MS: i64 = 60_000; fn family(_k: u32) -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()) + FieldDataType::Sketch(SketchAlgorithm::Kll) } /// `t(ts, value)`, time index `ts`. fn table_schema() -> Schema { @@ -87,10 +86,9 @@ fn fixture() -> Fixture { })); new(asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { child: range, - family: family(200), + summary_type: family(200), input: SummaryUpdate::column(ColumnRef::Named("value".into())), reduction: Reduction::Reduce(GroupKeys::default()), - grouping: GroupingStrategy::PerSubpopulationInstance, filter: None, })) }) diff --git a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json index d2e26ea78..53ef51dfd 100644 --- a/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json +++ b/crates/integration-tests/tests/materialized_view_fixtures/plan_wire_version_10.json @@ -1 +1 @@ -{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "family": {"ExactAggregate": ["Sum", "Sum"]}, "filter": null, "grouping": "PerSubpopulationInstance", "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "family": {"ExactAggregate": ["Sum", "Sum"]}, "filter": null, "grouping": "PerSubpopulationInstance", "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} +{"dag": {"schema_version": 10, "dag": {"edges": [{"consumer": 1, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 0, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 2, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "producer": 1, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 3, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 4, "data_state": {"primitive": "Raw", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 3, "role": "Input", "window": "RequiresAlignedPanePhaseOrExactBoundaryResidual"}, {"consumer": 5, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "producer": 4, "role": "Input", "window": "NotApplicable"}, {"consumer": 6, "data_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "grouping": "NotApplicable", "intermediate_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "producer": 2, "role": "Input", "window": "NotApplicable"}], "nodes": [{"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 0, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"Scan": {"predicates": [], "schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "source": {"TimeSeries": {"metric": "m"}}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact over exact inputs"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 1, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "timestamp"}, "name": "ts", "nullable": false, "table": null}, {"dtype": {"Plain": "float64"}, "name": "value", "nullable": false, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "$promql_series_identity", "nullable": false, "table": null}], "time_index": 0, "unique_keys": [[4, 0]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"NonASAP": {"TimeRange": {"child": 0, "kind": "range", "range": {"nanos": 0, "secs": 60}}}}}, {"guarantee": null, "id": 2, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 1, "filter": null, "input": {"item": null, "weight": {"Column": "SampleValue"}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [2, 3]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 3, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "ingestion_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}, {"guarantee": null, "id": 4, "lifetime": {"AcrossEvaluations": "Memory"}, "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"ExactAggregate": ["Sum", "Sum"]}, "name": "state", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "SummaryState", "timing": "ingestion_time"}, "payload": {"ASAP": {"SummaryAgg": {"child": 3, "filter": null, "input": {"item": null, "weight": {"Column": {"Named": "sum"}}, "weight_domain": {"kind": "unknown_or_signed"}}, "reduction": {"Reduce": [1]}, "summary_type": {"ExactAggregate": ["Sum", "Sum"]}}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 5, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 4}}}}, {"guarantee": {"provenance": [{"kind": "exact", "reason": "exact accumulator over exact input"}], "spec": {"accuracy_metric": "absolute", "accuracy_scope": "per_value", "error": {"op": "zero"}, "failure_probability": {"op": "zero"}}}, "id": 6, "lifetime": "None", "output_schema": {"closed": true, "fields": [{"dtype": {"Plain": "utf8"}, "name": "instance", "nullable": true, "table": null}, {"dtype": {"Plain": "utf8"}, "name": "job", "nullable": true, "table": null}, {"dtype": {"Plain": "float64"}, "name": "sum", "nullable": false, "table": null}], "time_index": null, "unique_keys": [[0, 1]]}, "output_state": {"primitive": "Raw", "timing": "query_time"}, "payload": {"ASAP": {"FinalizeExactAccumulator": {"child": 2}}}}], "roots": [5, 6], "scalar_roots": []}}, "queries": [{"query": "sum by (job) (sum_over_time(m[1m]))", "root_node_id": 5}, {"query": "sum by (job, instance) (sum_over_time(m[1m]))", "root_node_id": 6}], "stored_outputs": [{"built_at": "ingestion_run", "coverage": {"definition_id": "db0c9241a79202b6925e84aef010e89a", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [{"end_offset_ms": 0, "scan_node_id": 0, "time_range_node_id": 1, "width_ms": 60000}], "stored_outputs": []}, "node_id": 2, "retention_ms": 60000}, {"built_at": "ingestion_run", "coverage": {"definition_id": "3c90a6b5ff3ecaa445ea9bf285e1dc0b", "selection": [{"columns": [], "relative_time": [{"excluded": -60000}, {"included": 0}]}]}, "inputs": {"raw_sample_windows": [], "stored_outputs": [2]}, "node_id": 4, "retention_ms": 60000}], "query_time_raw_windows": [], "selection": null} diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index d11672f3d..0aea4e6b4 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -2,12 +2,9 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, GroupingStrategy, SummaryUpdate, -}; +use asap_types::ir::schema::{ExactKind, ExactParams, FieldDataType, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; use std::rc::Rc; @@ -32,7 +29,6 @@ async fn sql_filter_projection_example() { let root = lower_sql( "SELECT l_quantity * 2 AS q2 FROM lineitem WHERE l_quantity > 10", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap(); @@ -55,7 +51,6 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { let root = lower_sql( "SELECT SUM(bytes) + 1 AS total_bytes FROM requests WHERE status = 200", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap(); @@ -78,12 +73,11 @@ async fn sql_sum_projection_before_and_after_summary_rewrite() { let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: Rc::clone(child), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named( child.schema.fields[*column].name.clone(), )), reduction: reduction.clone(), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(), @@ -123,9 +117,7 @@ async fn sql_scalar_subquery_retains_its_cardinality_contract() { "SELECT (SELECT bytes FROM requests) AS v FROM lineitem", "SELECT l_quantity NOT IN (SELECT bytes FROM requests) AS present FROM lineitem", ] { - let root = lower_sql(query, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let root = lower_sql(query, &catalog()).await.unwrap(); root.validate_structure().unwrap(); assert!(root.children().len() > 1); let wire = asap_types::test_support::physical_asap_dag_at_query_time(&root).unwrap(); @@ -150,7 +142,6 @@ async fn sql_sum_example_executes_with_sql_null_semantics() { let root = lower_sql( "SELECT SUM(bytes) + 1 AS total_bytes FROM requests WHERE status = 200", &catalog(), - AccuracyTarget::exact(), ) .await .unwrap(); @@ -231,7 +222,7 @@ async fn sql_window_and_filtered_aggregate_types() { "SELECT MIN(l_quantity) OVER (ORDER BY l_quantity ROWS BETWEEN 2 PRECEDING AND 1 PRECEDING) AS s FROM lineitem", "SELECT SUM(l_quantity) FILTER (WHERE l_quantity < 0) AS s FROM lineitem GROUP BY l_quantity", ] { - let root = lower_sql(query, &catalog(), AccuracyTarget::exact()).await.unwrap(); + let root = lower_sql(query, &catalog()).await.unwrap(); root.validate_structure().unwrap(); assert_eq!(root.schema.fields[0], Field::plain("s", DataType::Int64, true), "{query}"); } diff --git a/crates/integration-tests/tests/operator_sharing.rs b/crates/integration-tests/tests/operator_sharing.rs index 0d1d07345..db9b6d557 100644 --- a/crates/integration-tests/tests/operator_sharing.rs +++ b/crates/integration-tests/tests/operator_sharing.rs @@ -47,7 +47,7 @@ fn catalog() -> SqlCatalog { /// Lower `sql` and take the logical DAG the planner pipeline selects. async fn plan(sql: &str, accuracy: AccuracyTarget) -> Rc { - let pre = lower_sql(sql, &catalog(), accuracy.clone()) + let pre = lower_sql(sql, &catalog()) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); selected_dag(pre, accuracy) diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 564381f0a..f67394f8c 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -21,12 +21,16 @@ async fn targets(sql: &str) -> Vec { Field::plain("bytes", DataType::Float64, false), ]), ); - let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_sql(sql, &catalog, accuracy) + let root = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap() .targets @@ -75,7 +79,7 @@ fn reads_quantile_sketch(node: &std::rc::Rc) -> bo matches!( node.asap(), Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), + summary_type: asap_types::ir::schema::FieldDataType::Sketch(kind), .. }) if matches!(kind, SketchAlgorithm::Kll | SketchAlgorithm::DDSketch) ) @@ -121,7 +125,7 @@ async fn a_filter_or_group_key_over_an_estimate_has_no_guarantee() { "SELECT count(p) FROM \ (SELECT src, approx_percentile_cont(bytes, 0.99) AS p FROM flows GROUP BY src) s", ] { - let root = lower_sql(sql, &catalog, accuracy.clone()) + let root = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); // No guarantee meets the target, so Pass 1 prunes the sketch @@ -150,7 +154,7 @@ fn reads_ddsketch(node: &std::rc::Rc) -> bool { matches!( node.asap(), Some(asap_types::ir::ASAPOp::SummaryAgg { - family: asap_types::ir::schema::FieldDataType::Sketch(kind, _), + summary_type: asap_types::ir::schema::FieldDataType::Sketch(kind), .. }) if matches!(kind, SketchAlgorithm::DDSketch) ) @@ -170,12 +174,11 @@ async fn a_sql_fold_over_estimates_leaves_its_row_count_unbound() { Field::plain("bytes", DataType::Float64, false), ]), ); - let accuracy = AccuracyTarget::max_error(0.01); // q=1 is an endpoint, so this isolates the fold's row-count statistic // without assuming a sign for SQL bytes or interpolating across zero. let sql = "SELECT max(p) FROM \ (SELECT src, approx_percentile_cont(bytes, 1.0) AS p FROM flows GROUP BY src) s"; - let root = lower_sql(sql, &catalog, accuracy.clone()).await.unwrap(); + let root = lower_sql(sql, &catalog).await.unwrap(); let candidates = vec![ asap_integration_tests::planning::explicit_summary_candidate( root, @@ -209,10 +212,14 @@ async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, /// [`heap_updates`] over `flows(fields)`. async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String)> { let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); - let accuracy = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_sql(sql, &catalog, accuracy).await.unwrap(); + let root = lower_sql(sql, &catalog).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let name = |expr: &SummaryInputExpr| match expr { @@ -255,7 +262,7 @@ async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String }; for node in OperatorNode::reachable(root) { if let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), input, .. }) = node.asap() diff --git a/crates/integration-tests/tests/planner_layering_common/mod.rs b/crates/integration-tests/tests/planner_layering_common/mod.rs index 9c53b8ac9..243d0e7f1 100644 --- a/crates/integration-tests/tests/planner_layering_common/mod.rs +++ b/crates/integration-tests/tests/planner_layering_common/mod.rs @@ -24,7 +24,6 @@ use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; use asap_types::physical::Lifetime; -use asap_types::physical::PhysicalWorkloadPlan; use asap_types::workload::{PlanningWorkload, QueryRecurrence, RepeatedDemand}; pub use asap_devtools::planner_layering_examples::*; @@ -127,12 +126,12 @@ pub struct Physical { pub id: String, pub from_logical: String, pub query_roots: Vec, - pub candidate: PhysicalWorkloadPlan, + pub dag: PhysicalASAPDAG, } impl Physical { pub fn dag(&self) -> &PhysicalASAPDAG { - &self.candidate.dag + &self.dag } } @@ -262,10 +261,10 @@ pub fn is_summary(payload: &Payload) -> bool { payload, Operator::ASAP( ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } | ASAPOp::SummaryTumblingWindowChain { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. } | ASAPOp::SummaryEstimate { .. } | ASAPOp::SummaryMerge { .. } diff --git a/crates/integration-tests/tests/production_rollup_workloads.rs b/crates/integration-tests/tests/production_rollup_workloads.rs index 5cad992ea..d7caaa518 100644 --- a/crates/integration-tests/tests/production_rollup_workloads.rs +++ b/crates/integration-tests/tests/production_rollup_workloads.rs @@ -57,7 +57,7 @@ fn data() -> DataWorkload { "sources": {"http": { "series_count": 10_000, "sample_interval": 5, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "max_distinct_per_evaluation": 1_000_000, "rows_held": 120_000_000, }}, @@ -84,42 +84,23 @@ fn production( .unwrap() } -fn check( - run: &PlanningRun, - queries: usize, - expected_rollups: usize, - family: &str, - separate_cost: Option, -) { - let selection = run +fn check(run: &PlanningRun, queries: usize, expected_rollups: usize, family: &str) { + let (plan, selection) = run .selection .as_ref() .expect("validated production plan within 30s"); - println!("selected cost={:?}", selection.cost.cost_per_second); - let selected_cost = selection.cost.cost_per_second.unwrap(); - if let Some(separate) = separate_cost { - // Both searches must improve on independently built KLLs at the - // same accuracy target. - assert!(selected_cost < separate); - } let accepted = &selection.accepted_accuracy_per_query; assert_eq!(accepted.len(), queries); - assert!(accepted.iter().all(|a| !a.accepted_regardless_of_guarantee)); - if separate_cost.is_some() && accepted.iter().all(|a| a.sketches.is_empty()) { - println!("exact plan costs less than both the direct incumbent and independent KLLs"); - return; - } assert!( accepted .iter() - .all(|a| !a.accepted_regardless_of_guarantee && a.sketches.iter().any(|s| s == family)), + .all(|a| a.sketches.iter().any(|s| s == family)), "{accepted:?}" ); - let selected = &run.selected_physical_plan.as_ref().unwrap().plan; let mut seen = HashSet::new(); let mut builds = 0; let mut rollups = 0; - for root in &selected.roots { + for root in &plan.query_roots().unwrap() { let QueryRoot::Operator(root) = root else { panic!("operator output") }; @@ -149,9 +130,7 @@ async fn sql(queries: &[String], accuracy: AccuracyTarget) -> PlanningRun { let mut roots = Vec::new(); for query in queries { roots.push(QueryRoot::Operator( - lower_sql(query, &catalog(), accuracy.clone()) - .await - .unwrap(), + lower_sql(query, &catalog()).await.unwrap(), )); } production( @@ -168,7 +147,7 @@ async fn production_three_sql_groupings_share_kll() { .accuracy_metric(AccuracyMetric::Rank) .accuracy_scope(AccuracyScope::PerValue); let queries: Vec<_> = ["region, service", "region", "service"].iter().map(|keys| format!("SELECT {keys}, approx_percentile_cont(latency, 0.99) AS value FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY {keys}")).collect(); - check(&sql(&queries, accuracy).await, 3, 2, "Kll", None); + check(&sql(&queries, accuracy).await, 3, 2, "Kll"); } #[tokio::test] @@ -178,7 +157,7 @@ async fn production_sql_distinct_region_service_rollup() { .accuracy_metric(AccuracyMetric::Relative) .accuracy_scope(AccuracyScope::PerValue); let queries: Vec<_> = ["region, service", "service"].iter().map(|keys| format!("SELECT {keys}, COUNT(DISTINCT user_id) AS value FROM http WHERE ts >= now() - INTERVAL '5 minutes' GROUP BY {keys}")).collect(); - check(&sql(&queries, accuracy).await, 2, 1, "Kmv", None); + check(&sql(&queries, accuracy).await, 2, 1, "Kmv"); } // Exercise multi-input operators through the production route under more than @@ -213,15 +192,6 @@ fn production_binary_queries_use_joint_prices_with_mixed_schedules() { })) .unwrap(); let run = production(roots, &entries, &data); - let selection = run.selection.as_ref().expect("validated binary workload"); - println!("mixed binary workload report={:?}", selection.search_report); - let accepted = &selection.accepted_accuracy_per_query; - assert_eq!(accepted.len(), queries.len()); - assert!(accepted - .iter() - .all(|item| !item.accepted_regardless_of_guarantee)); - assert!(selection - .cost - .cost_per_second - .is_some_and(|cost| cost.is_finite() && cost >= 0.0)); + let (_, selection) = run.selection.as_ref().expect("validated binary workload"); + assert_eq!(selection.accepted_accuracy_per_query.len(), queries.len()); } diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs index 5626606a4..f941653b0 100644 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ b/crates/integration-tests/tests/promql_numeric_regressions.rs @@ -24,7 +24,7 @@ fn plan(query: &str, accuracy: AccuracyTarget) -> Rc { fn aggregate(node: &OperatorNode) -> (&FieldDataType, &SummaryUpdate, &Reduction) { match &node.operator { Operator::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input, reduction, .. @@ -120,7 +120,7 @@ fn checked_ratio_must_not_certify_cross_zero_interpolation() { ) .unwrap(); let node = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch); - assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), .. }) if *kind == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, + assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) if *kind == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, "the signed interpolation counterexample requires two explicit DDSketch builds"); assert!( matches!(node.operator, Operator::NonASAP(NonASAPOp::BinaryOp { .. })) @@ -228,7 +228,7 @@ fn quantile_over_temporal_average_keeps_a_legal_candidate() { matches!( n.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. }) ) @@ -257,7 +257,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { matches!( n.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. }) ) @@ -273,7 +273,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { .find_map(|node| { let state = heap(node)?; let (family, _, _) = aggregate(&state); - matches!(family, FieldDataType::Sketch(kind, _) if kind == &wanted) + matches!(family, FieldDataType::Sketch(kind) if kind == &wanted) .then_some((node, state)) }) .expect("weighted sketch candidate"); @@ -295,7 +295,7 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { // so plan selection rejects it. for state in candidates.iter().filter_map(heap) { let (family, update, _) = aggregate(&state); - if matches!(family, FieldDataType::Sketch(kind, _) if kind == &SketchAlgorithm::CmsWithHeap) + if matches!(family, FieldDataType::Sketch(kind) if kind == &SketchAlgorithm::CmsWithHeap) { assert!(!matches!( update.weight_domain, @@ -330,15 +330,3 @@ fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { physical_asap_dag(node); } } - -// A rejected composition must fail the assertion helper without relying on a planner bug. -#[test] -#[should_panic(expected = "does not compose")] -fn logical_candidates_fails_on_a_candidate_that_does_not_compose() { - asap_integration_tests::planning::require_composed_operator_candidates(vec![ - asap_devtools::planning_stages::ShownLogicalCandidate { - id: "invalid-composition".into(), - roots: Err("invalid choice".into()), - }, - ]); -} diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 398ee91de..583e631ca 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -19,8 +19,8 @@ use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::DataType; use asap_types::ir::schema::{ - EntityIdentity, ExactKind, ExactParams, FieldDataType, GroupingStrategy, Schema, - SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, + EntityIdentity, ExactKind, ExactParams, FieldDataType, Schema, SketchAlgorithm, + SketchStatistic, SummaryInputExpr, SummaryUpdate, }; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, ScalarExpr}; @@ -44,7 +44,7 @@ fn realize(root: &Rc, accuracy: AccuracyTarget) -> Rc bool { match &node.operator { Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(kind), .. }) if kind == algorithm => true, _ => node @@ -381,7 +381,7 @@ fn planner_heap_topk_reference_execution_matches_ground_truth() { /// /// ```text /// SummaryEstimate { query: Quantile{0.99} } → {quantile: Float64} -/// └─ SummaryAgg { Kll{k:269}, input: SampleValue } → {state: Sketch(Kll, {k:269})} +/// └─ SummaryAgg { Kll{k:269}, input: SampleValue } → {state: Sketch(Kll)} /// └─ FinalizeExactAccumulator /// └─ SummaryAgg { Rate, input: SampleValue } → {ts, state: ExactAggregate(Rate), …} /// └─ TimeRange{5m} → Scan → {ts, value} @@ -433,7 +433,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // same empty `by: []` (issue #163). let Some(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, .. @@ -441,10 +441,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { else { panic!("expected SummaryAgg, got {:?}", summary_input.operator); }; - assert_eq!( - family, - &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) - ); + assert_eq!(family, &FieldDataType::Sketch(SketchAlgorithm::Kll)); assert_eq!(input, &SummaryUpdate::column(ColumnRef::SampleValue)); assert_eq!( reduction, @@ -453,7 +450,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { ); assert_eq!( dtype(&summary_input.schema, "state"), - &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) + &FieldDataType::Sketch(SketchAlgorithm::Kll) ); let Some(ASAPOp::FinalizeExactAccumulator { child }) = child.asap() else { @@ -465,7 +462,7 @@ fn promql_quantile_of_rate_binds_kll_over_rate_accumulator() { // grouping concept at all — every entity stays its own summary. let Some(ASAPOp::SummaryAgg { child: leaf, - family, + summary_type: family, reduction, .. }) = child.asap() diff --git a/crates/integration-tests/tests/rollup_controlled.rs b/crates/integration-tests/tests/rollup_controlled.rs index 1178513c4..7f05e3911 100644 --- a/crates/integration-tests/tests/rollup_controlled.rs +++ b/crates/integration-tests/tests/rollup_controlled.rs @@ -59,29 +59,27 @@ fn production_distinct_rollup_finds_shared_kmv() { asap_planner::PlanSearchSettings::default(), ) .unwrap(); - let selection = run + let (plan, selection) = run .selection .as_ref() .expect("validated plan within budget"); - println!( - "production cost={:?} report={:?}", - selection.cost.cost_per_second, selection.search_report - ); let accepted = &selection.accepted_accuracy_per_query; assert_eq!(accepted.len(), 2); assert!(accepted .iter() - .all(|accuracy| !accuracy.accepted_regardless_of_guarantee - && accuracy.sketches.iter().any(|sketch| sketch == "Kmv"))); - assert!(selection.cost.cost_per_second.unwrap() < 2.0e-6); - let plan = &run.selected_physical_plan.as_ref().unwrap().plan; - let builds: HashSet<_> = nodes(&plan.roots) + .all(|accuracy| accuracy.sketches.iter().any(|sketch| sketch == "Kmv"))); + assert!( + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, data, &settings) + < 2.0e-6 + ); + let roots = plan.query_roots().unwrap(); + let builds: HashSet<_> = nodes(&roots) .iter() .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) .map(Rc::as_ptr) .collect(); assert_eq!(builds.len(), 1, "both queries reuse a single state build"); - let rollup = nodes(&plan.roots) + let rollup = nodes(&roots) .iter() .any(|n| matches!(n.asap(), Some(ASAPOp::SummaryMerge { .. }))); assert!( diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/922.json b/crates/integration-tests/tests/sharing_issue_fixtures/922.json index 997650b02..43d8c61e9 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/922.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/922.json @@ -55,7 +55,7 @@ "metrics": { "series_count": 10000, "metric_type": null, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 10000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/923.json b/crates/integration-tests/tests/sharing_issue_fixtures/923.json index 89cfe117d..753f1fb99 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/923.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/923.json @@ -53,7 +53,7 @@ "m_counter": { "series_count": 1000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 10000, "stream_norms": null, "max_distinct_per_evaluation": null, @@ -66,7 +66,7 @@ "m_gauge": { "series_count": 1000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 10000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/924.json b/crates/integration-tests/tests/sharing_issue_fixtures/924.json index 69ed9e5e6..9f955da64 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/924.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/924.json @@ -53,7 +53,7 @@ "m_counter": { "series_count": 10000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, @@ -66,7 +66,7 @@ "m_gauge": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_fixtures/925.json b/crates/integration-tests/tests/sharing_issue_fixtures/925.json index d8dd7b0d6..af3001cbe 100644 --- a/crates/integration-tests/tests/sharing_issue_fixtures/925.json +++ b/crates/integration-tests/tests/sharing_issue_fixtures/925.json @@ -53,7 +53,7 @@ "m_counter": { "series_count": 10000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, @@ -66,7 +66,7 @@ "m_gauge": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": null, "max_distinct_per_evaluation": null, diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs index 02221f75a..8d896009f 100644 --- a/crates/integration-tests/tests/sharing_issue_regressions.rs +++ b/crates/integration-tests/tests/sharing_issue_regressions.rs @@ -50,8 +50,9 @@ fn cost(w: &FuzzWorkload) -> f64 { PlanSearchSettings::default(), ) .unwrap(); - let selection = run.run.selection.as_ref().unwrap(); - let value = selection.cost.cost_per_second.unwrap(); + let (plan, _) = run.run.selection.as_ref().unwrap(); + let value = + asap_integration_tests::planning::plan_cost_per_second(plan, &entries, &w.data, &settings); println!("cost={value}"); value } diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs index f2c173214..2422d8ae3 100644 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ b/crates/integration-tests/tests/sketch_rollup.rs @@ -20,7 +20,6 @@ use asap_integration_tests::planning::explicit_local_candidates; use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::summary_coverage::CoverageError; -use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::{ DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SketchAlgorithm, }; @@ -191,10 +190,9 @@ fn replace( fn rolled_up(coarse: &OperatorNode) -> Rc { let Some(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction: Reduction::Reduce(by), - grouping, filter, }) = coarse.asap() else { @@ -204,10 +202,9 @@ fn rolled_up(coarse: &OperatorNode) -> Rc { assert_eq!(by.keys().len(), 1); let finer = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: Rc::clone(child), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: Reduction::by(vec![region, by.keys()[0]]), - grouping: grouping.clone(), filter: filter.clone(), })) .unwrap(); @@ -224,16 +221,15 @@ fn rolled_up(coarse: &OperatorNode) -> Rc { /// of the search archive and its tighter, potentially much larger capacities. async fn direct_and_rolled_up( sql: &str, - accuracy: AccuracyTarget, ) -> Vec<(SketchAlgorithm, Rc, Rc)> { - let root = lower_sql(sql, &catalog(), accuracy.clone()).await.unwrap(); + let root = lower_sql(sql, &catalog()).await.unwrap(); let mut out = Vec::new(); for candidate in explicit_local_candidates(root) { let Some(build) = build(&candidate) else { continue; }; let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(kind), .. }) = build.asap() else { @@ -253,14 +249,9 @@ async fn direct_and_rolled_up( /// counters add), so every answer is the direct build's. #[tokio::test] async fn distinct_counts_rolled_up_equal_the_direct_build() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative); - let pairs = direct_and_rolled_up( - "SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job", - accuracy, - ) - .await; + let pairs = + direct_and_rolled_up("SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job") + .await; let families: Vec<_> = pairs.iter().map(|(family, ..)| *family).collect(); assert!(families.contains(&SketchAlgorithm::Kmv), "{families:?}"); for (family, direct, rolled) in pairs { @@ -277,13 +268,9 @@ async fn distinct_counts_rolled_up_equal_the_direct_build() { #[tokio::test] async fn quantiles_rolled_up_keep_the_sketch_bound() { const EPSILON: f64 = 0.05; - for metric in [AccuracyMetric::Rank, AccuracyMetric::Relative] { - let accuracy = AccuracyTarget::max_error(EPSILON) - .failure_probability(0.01) - .accuracy_metric(metric); + { let pairs = direct_and_rolled_up( "SELECT job, approx_percentile_cont(latency, 0.99) AS value FROM http GROUP BY job", - accuracy, ) .await; for (family, direct, rolled) in pairs { @@ -318,13 +305,9 @@ async fn quantiles_rolled_up_keep_the_sketch_bound() { /// groups. #[tokio::test] async fn invalid_rollups_are_rejected() { - let accuracy = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_metric(AccuracyMetric::Relative); let root = lower_sql( "SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job", &catalog(), - accuracy.clone(), ) .await .unwrap(); @@ -335,7 +318,7 @@ async fn invalid_rollups_are_rejected() { matches!( build.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), + summary_type: FieldDataType::Sketch(..), .. }) ) @@ -364,7 +347,6 @@ async fn invalid_rollups_are_rejected() { child, input, reduction, - grouping, .. }) = finer.asap() else { @@ -372,10 +354,9 @@ async fn invalid_rollups_are_rejected() { }; let rates = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: Rc::clone(child), - family: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), + summary_type: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: None, })) .unwrap(); @@ -386,5 +367,5 @@ async fn invalid_rollups_are_rejected() { assert!(merged_rates .unwrap_err() .to_string() - .contains(&CoverageError::FamilyCannotRegroup.to_string())); + .contains(&CoverageError::SummaryTypeCannotRegroup.to_string())); } diff --git a/crates/integration-tests/tests/sql_cardinality.rs b/crates/integration-tests/tests/sql_cardinality.rs index 0a418d782..c0a03c4b8 100644 --- a/crates/integration-tests/tests/sql_cardinality.rs +++ b/crates/integration-tests/tests/sql_cardinality.rs @@ -2,7 +2,6 @@ use crate::physical_common; use asap_executor::values::Value; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; // COUNT(DISTINCT src_ip) skips NULL and keeps the WHERE, Utf8 identities and @@ -19,7 +18,6 @@ async fn sql_distinct_executes_through_raw_scan_and_native_binding() { let root = lower_sql( "SELECT COUNT(DISTINCT src_ip) AS sources FROM flows WHERE keep", &catalog, - AccuracyTarget::exact(), ) .await .unwrap(); diff --git a/crates/integration-tests/tests/sql_frequency.rs b/crates/integration-tests/tests/sql_frequency.rs index 7bf842055..a560eb274 100644 --- a/crates/integration-tests/tests/sql_frequency.rs +++ b/crates/integration-tests/tests/sql_frequency.rs @@ -4,7 +4,6 @@ use crate::physical_common; use asap_executor::values::Value; use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; const ENTROPY: &str = "SELECT -SUM(p * LN(p)) FROM (\ @@ -27,9 +26,7 @@ fn rows(sources: &[&str]) -> Vec> { } async fn run(sql: &str, sources: &[&str]) -> Option { - let root = lower_sql(sql, &catalog(), AccuracyTarget::exact()) - .await - .unwrap(); + let root = lower_sql(sql, &catalog()).await.unwrap(); let result = physical_common::execute_raw_rows(&root, rows(sources)); match result.as_slice() { [row] => match row.as_slice() { diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index 411f5b612..f8484222f 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -30,8 +30,7 @@ use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, GroupingStrategy, SketchAlgorithm, SketchStatistic, - SummaryUpdate, + ExactKind, ExactParams, FieldDataType, SketchAlgorithm, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; use asap_types::workload::SqlDialect; @@ -140,8 +139,8 @@ fn catalog() -> SqlCatalog { ) } -async fn lower(sql: &str, accuracy: AccuracyTarget) -> Rc { - lower_sql(sql, &catalog(), accuracy) +async fn lower(sql: &str) -> Rc { + lower_sql(sql, &catalog()) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } @@ -162,18 +161,13 @@ async fn clickhouse_temporal_sql_reuses_rate_and_increase_physical_summaries() { "SELECT service, {function}(latency, ts, 300000) AS v \ FROM metrics WHERE bytes > 0 GROUP BY service" ); - let pre_asap = lower_sql_dialect( - &sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect("explicit temporal SQL must lower"); + let pre_asap = lower_sql_dialect(&sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .expect("explicit temporal SQL must lower"); let planned = realize(&pre_asap, AccuracyTarget::exact()); let physical = first_summary_agg(&planned); let Operator::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, reduction, child, .. @@ -208,20 +202,16 @@ async fn clickhouse_outer_sum_recursively_binds_inner_temporal_aggregate() { SELECT service, {function}(latency, ts, {window_ms}) AS v \ FROM metrics GROUP BY service)" ); - let pre_asap = lower_sql_dialect( - &sql, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect("nested temporal SQL must lower"); + let pre_asap = lower_sql_dialect(&sql, &catalog(), SqlDialect::ClickhouseSQL) + .await + .expect("nested temporal SQL must lower"); let root = realize(&pre_asap, AccuracyTarget::exact()); fn has_temporal_summary(node: &OperatorNode) -> bool { match &node.operator { Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), + summary_type: + FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), .. }) => true, Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { @@ -237,7 +227,10 @@ async fn clickhouse_outer_sum_recursively_binds_inner_temporal_aggregate() { ); // The outer sum is an exact Sum state over the finalized per-series // values, so the export keeps both summaries. - let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &first_summary_agg(&root).operator + let Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + }) = &first_summary_agg(&root).operator else { unreachable!() }; @@ -261,11 +254,7 @@ async fn clickhouse_outer_sum_recursively_binds_inner_temporal_aggregate() { #[tokio::test] async fn sql_full_query_retains_project_and_binds_inner_aggregate() { - let pre_asap = lower( - "SELECT approx_percentile_cont(latency, 0.99) AS p99 FROM metrics", - AccuracyTarget::max_error(0.01), - ) - .await; + let pre_asap = lower("SELECT approx_percentile_cont(latency, 0.99) AS p99 FROM metrics").await; let Some(NonASAPOp::Project { cols: expected_cols, qualifier: expected_qualifier, @@ -311,7 +300,6 @@ async fn sql_join_recursively_binds_both_temporal_aggregate_children() { ON b.service=a.service", &catalog(), SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .expect("two-subquery rate ratio must lower"); @@ -384,7 +372,7 @@ async fn sql_join_recursively_binds_both_temporal_aggregate_children() { assert!(matches!( aggregate.operator, Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), + summary_type: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), .. }) )); @@ -421,7 +409,6 @@ async fn sql_relational_parents_retain_summary_bound_aggregate() { (SELECT service, approx_percentile_cont(latency, 0.9) AS p \ FROM metrics GROUP BY service) t \ WHERE t.p > 100 ORDER BY t.p DESC LIMIT 5", - AccuracyTarget::max_error(0.01), ) .await; let root = realize_ignoring_guarantees(&pre_asap, AccuracyTarget::max_error(0.01)); @@ -469,7 +456,6 @@ async fn sql_filter_keeps_read_predicate_and_summary_population_selection() { (SELECT service, approx_percentile_cont(latency, 0.9) AS p \ FROM metrics WHERE service = 'api' GROUP BY service) t \ WHERE t.p > 100", - AccuracyTarget::max_error(0.01), ) .await; let expected_read_predicate = { @@ -551,11 +537,7 @@ async fn sql_filter_keeps_read_predicate_and_summary_population_selection() { #[tokio::test] async fn sql_quantile_binds_kll_sketch_over_named_column() { - let pre_asap = lower( - "SELECT approx_percentile_cont(latency, 0.99) FROM metrics", - AccuracyTarget::max_error(0.01), - ) - .await; + let pre_asap = lower("SELECT approx_percentile_cont(latency, 0.99) FROM metrics").await; let planned = asap_integration_tests::planning::explicit_summary_candidate( pre_asap, SketchAlgorithm::Kll, @@ -583,7 +565,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { let Operator::ASAP(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, .. @@ -591,10 +573,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { else { panic!("expected SummaryAgg, got {:?}", summary_input.operator); }; - assert_eq!( - family, - &FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) - ); + assert_eq!(family, &FieldDataType::Sketch(SketchAlgorithm::Kll)); assert_eq!( input, &SummaryUpdate::column(ColumnRef::Qualified { @@ -610,7 +589,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { ); assert_eq!( summary_input.schema.fields[0].dtype, - FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) + FieldDataType::Sketch(SketchAlgorithm::Kll) ); assert!( @@ -636,11 +615,7 @@ async fn sql_quantile_binds_kll_sketch_over_named_column() { /// 1/(2ε²δ)` reaches precision 18, the largest. #[tokio::test] async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { - let pre_asap = lower( - "SELECT COUNT(DISTINCT service) FROM metrics", - AccuracyTarget::max_error(0.01), - ) - .await; + let pre_asap = lower("SELECT COUNT(DISTINCT service) FROM metrics").await; let planned = asap_integration_tests::planning::explicit_summary_candidate( pre_asap, SketchAlgorithm::Hll, @@ -662,7 +637,7 @@ async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { ); let Operator::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input, reduction, .. @@ -670,10 +645,7 @@ async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { else { panic!("expected SummaryAgg, got {:?}", summary_input.operator); }; - assert_eq!( - family, - &FieldDataType::Sketch(SketchAlgorithm::Hll, GroupingStrategy::default()) - ); + assert_eq!(family, &FieldDataType::Sketch(SketchAlgorithm::Hll)); assert_eq!( input, &SummaryUpdate::column(ColumnRef::Qualified { @@ -688,7 +660,7 @@ async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { async fn map_projection_export_keeps_the_map_over_an_avg_of_exact_states() { let pre = lower_sql_dialect( "SELECT map('job', t.service) AS labels, t.avg_bytes FROM (SELECT service, AVG(bytes) AS avg_bytes FROM metrics GROUP BY service) t WHERE t.avg_bytes > 100", - &catalog(), SqlDialect::ClickhouseSQL, AccuracyTarget::exact(), + &catalog(), SqlDialect::ClickhouseSQL, ).await.unwrap(); let root = realize(&pre, AccuracyTarget::exact()); let dag = physical_asap_dag(&root); @@ -702,7 +674,10 @@ async fn map_projection_export_keeps_the_map_over_an_avg_of_exact_states() { .nodes .iter() .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { family, .. }) => Some(family), + PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + }) => Some(family), _ => None, }) .collect(); diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs index fab612462..e624feb75 100644 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -43,7 +43,7 @@ async fn sql(query: &str) -> QueryRoot { Field::plain("bytes", DataType::Float64, false), ]), ); - let root = lower_sql(query, &catalog, accuracy()) + let root = lower_sql(query, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")); QueryRoot::Operator(root) @@ -98,7 +98,8 @@ fn output_columns(root: &QueryRoot) -> Vec<(String, String, bool)> { fn compose_all(label: &str, root: QueryRoot) -> usize { let expected = output_columns(&root); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[root]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries(&[root], &[None], &Default::default()) + .unwrap(), ) .unwrap(); let radix: Vec<_> = inventory @@ -177,7 +178,8 @@ fn promql_candidates_compose() { /// choice, every other target taking its first. fn composed_estimates(root: QueryRoot) -> Vec> { let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[root]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries(&[root], &[None], &Default::default()) + .unwrap(), ) .unwrap(); let targets = inventory.targets.len(); @@ -211,7 +213,7 @@ fn update_of(estimate: &OperatorNode) -> (SketchAlgorithm, SummaryUpdate) { unreachable!() }; let Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(kind), input, .. }) = summary_input.asap() @@ -267,7 +269,12 @@ async fn subquery_in_a_measure_filter_is_substituted() { let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ FROM flows GROUP BY src"; let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[sql(query).await]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[sql(query).await], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); assert_eq!(inventory.targets.len(), 2); @@ -399,10 +406,14 @@ async fn avg_as_sum_and_count( catalog: &SqlCatalog, query: &str, ) -> (Rc, Rc) { - let exact = lower_sql(query, catalog, accuracy()).await.unwrap(); + let exact = lower_sql(query, catalog).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(exact.clone())]) - .unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(exact.clone())], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let choice: Vec<_> = inventory @@ -536,7 +547,12 @@ async fn filtered_sql_avg_is_null_without_matching_rows() { #[test] fn promql_avg_divides_sum_and_count_vectors() { let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[promql("avg by (job) (m)")]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[promql("avg by (job) (m)")], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); assert_eq!( @@ -576,9 +592,14 @@ async fn every_candidate( HowToComputeAggregateMeasure, Result, asap_logical_asap_optimization::PerComputationCandidateError>, )> { - let root = lower_sql(query, catalog, accuracy()).await.unwrap(); + let root = lower_sql(query, catalog).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &asap_logical_asap_optimization::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); assert_eq!(inventory.targets.len(), 1, "{query}"); @@ -603,11 +624,14 @@ fn count_updates(root: &QueryRoot) -> Vec { OperatorNode::reachable(root) .iter() .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { family, input, .. }) - if !matches!( - family, - FieldDataType::ExactAggregate(asap_types::ir::schema::ExactKind::Sum, _) - ) => + Some(ASAPOp::SummaryAgg { + summary_type: family, + input, + .. + }) if !matches!( + family, + FieldDataType::ExactAggregate(asap_types::ir::schema::ExactKind::Sum, _) + ) => { Some(input.clone()) } diff --git a/crates/integration-tests/tests/time_range.rs b/crates/integration-tests/tests/time_range.rs index 6d7f8b8dc..aacf838e6 100644 --- a/crates/integration-tests/tests/time_range.rs +++ b/crates/integration-tests/tests/time_range.rs @@ -74,13 +74,7 @@ fn q14_increase() { fn q15_count_over_time() { assert_eq!( lower("count_over_time(http_requests_total[5m])"), - range_agg( - 300, - AggIntent::Count { - accuracy: AccuracyTarget::exact() - }, - "http_requests_total" - ), + range_agg(300, AggIntent::Count, "http_requests_total"), ); } @@ -159,11 +153,7 @@ fn q17_quantile_over_time() { lower("quantile_over_time(0.99, http_requests_total[5m])"), range_agg( 300, - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::exact() - }, + AggIntent::Quantile { col: None, q: 0.99 }, "http_requests_total", ), ); diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs index c4b950e79..38baf4895 100644 --- a/crates/logical-asap-optimization/src/lib.rs +++ b/crates/logical-asap-optimization/src/lib.rs @@ -50,11 +50,10 @@ //! approximate-over-approximate compositions under conservative rules (no //! independence assumptions, unknown statistics stay unknown). A //! composition with no rule leaves the guarantee `None` and keeps the -//! candidate. Pass 1 takes the deployment's `AccuracyModel` and evidence -//! only to prune summaries no query reading them can use (when they cannot -//! accept an output without its guarantee) and sizes every summary for its -//! consumers; plan selection (Stage 3) checks each selected candidate's -//! guarantee against its query's `AccuracyTarget` and never resizes. +//! candidate. Pass 1 prunes nothing for accuracy; Pass 2 gives each readout +//! its required accuracy and eliminates the candidates no query can use, +//! with the deployment's measured accuracy (`AccuracyEvidence`) read before +//! each summary's theoretical guarantee. mod pass1; mod pass2; diff --git a/crates/logical-asap-optimization/src/pass1.rs b/crates/logical-asap-optimization/src/pass1.rs index 6c2f87b43..6f56465ae 100644 --- a/crates/logical-asap-optimization/src/pass1.rs +++ b/crates/logical-asap-optimization/src/pass1.rs @@ -3,6 +3,8 @@ //! [`pass1_memo`], whose output //! memo Pass 2 extends. +#[cfg(any(test, feature = "test-support"))] +pub(crate) mod composition; /// Stage 1, Pass 1's entry: the initial candidates of the memo groups. pub(crate) mod logical_memo_candidates; /// Stage 1, Pass 1's computation targets, their candidates and their diff --git a/crates/logical-asap-optimization/src/pass1/composition.rs b/crates/logical-asap-optimization/src/pass1/composition.rs new file mode 100644 index 000000000..4e4bdfd2a --- /dev/null +++ b/crates/logical-asap-optimization/src/pass1/composition.rs @@ -0,0 +1,1054 @@ +//! Composing one whole-workload candidate DAG from one per-computation +//! candidate per target (Algorithm 1's `SUBSTITUTE`): test support only. +//! Planning adds each way to compute to the memo instead (#1207 PR 3 +//! deletes this). +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::column_resolution::ResolveError; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::topk::ranked_column; +use asap_types::ir::schema::{ColumnId, Field}; +use asap_types::ir::schema::{DataType, Schema}; +use asap_types::ir::schema::{ + EntityIdentity, ExactKind, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, + SummaryInputExpr, SummaryUpdate, WeightDomain, +}; +use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, + Predicate, ProjectItem, QueryRoot, ScalarExpr, +}; + +use asap_accuracy::propagation::{raw_data_samples_of_metric, with_guarantees}; +use asap_accuracy::sizing::sketch_statistic; + +use asap_types::stages::{ComputationTarget, HavingPlacement, HowToComputeAggregateMeasure}; + +use super::per_computation_candidates::*; +/// `update` with the open requirement that its weights are samples of +/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap +/// update whose weights are not otherwise proven. +fn with_raw_data_samples_requirement( + mut update: SummaryUpdate, + metric: Option<&str>, +) -> SummaryUpdate { + if let Some(metric) = metric { + if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { + update.weight_domain = WeightDomain::RawDataSamplesOfMetric { + metric: metric.to_string(), + }; + } + } + update +} + +/// Build one whole-workload candidate (end of Pass 1): `choice[i]` picks +/// `inventory.targets[i]`'s candidate, or past its own candidates its fused +/// computation's ([`ComputationTarget::computation_for`]). An aggregate with +/// a chosen candidate other than `Exact` is replaced measure by measure: an +/// exact accumulator by `SummaryAgg` → `FinalizeExactAccumulator`, a summary +/// by `SummaryAgg` → `SummaryEstimate`, `avg` by its `sum` and `count` +/// accumulators divided, a fused computation by one state over the absorbed +/// target's input, and any other measure by its own single-measure +/// aggregate; several measures are joined back on the group keys. A summary +/// is sized here for its measure's accuracy requirement. A measure filter +/// becomes `SummaryAgg.filter`; +/// `HAVING` is applied as its target records. A sub-DAG shared between roots +/// stays shared. Every node carries the guarantee `asap_accuracy::propagation::composed_guarantee` +/// gives it. +pub fn compose_logical_candidate( + inventory: &PerComputationCandidates, + choice: &[usize], +) -> Result, PerComputationCandidateError> { + compose(inventory, &inventory.roots, choice) +} + +/// [`compose_logical_candidate`] of `roots` (some of `inventory.roots`), +/// each summary sized for its measure's whole budget. +fn compose( + inventory: &PerComputationCandidates, + roots: &[QueryRoot], + choice: &[usize], +) -> Result, PerComputationCandidateError> { + if choice.len() != inventory.targets.len() { + return Err(PerComputationCandidateError::InvalidChoice); + } + let mut chosen: Chosen = HashMap::new(); + for (target, &index) in inventory.targets.iter().zip(choice) { + let (candidate, absorbs) = target + .computation_for(index) + .ok_or(PerComputationCandidateError::InvalidChoice)?; + chosen + .entry(Rc::as_ptr(inventory.aggregate(target))) + .or_default() + .push(ChosenComputation { + target, + candidate, + fused: absorbs.is_some(), + }); + } + // Consumer `Filter`s whose key-only `HAVING` moves below a substituted + // aggregate: each is replaced by its input. + let pushed_below: HashSet<_> = chosen + .values() + .filter(|targets| substituted(targets)) + .flat_map(|targets| &targets[0].target.having_filters) + .filter(|entry| entry.placement == HavingPlacement::BelowAggregate) + .filter_map(|entry| entry.filter.as_ref().map(Rc::as_ptr)) + .collect(); + compose_roots(roots, &chosen, &pushed_below, &Sizing::default()) +} + +/// One measure of an aggregate: the aggregate and the measure's index. +type MeasureKey = (*const OperatorNode, usize); + +/// How [`compose_logical_candidate`] sizes summaries in one composition. +#[derive(Default)] +struct Sizing { + /// Every sketch summary state built, with the measure it sizes. + built: std::cell::RefCell, MeasureKey)>>, +} + +/// Every root of `roots` with the chosen candidates substituted, each +/// summary sized as `sizing` says, and every node with its guarantee. +fn compose_roots( + roots: &[QueryRoot], + chosen: &Chosen<'_>, + pushed_below: &HashSet<*const OperatorNode>, + sizing: &Sizing, +) -> Result, PerComputationCandidateError> { + let (mut cache, mut guaranteed) = (HashMap::new(), HashMap::new()); + let mut rewrite = |node: &Rc| { + let node = rewrite(node, chosen, pushed_below, sizing, &mut cache)?; + with_guarantees(&node, &mut guaranteed).map_err(PerComputationCandidateError::from) + }; + let roots = roots + .iter() + .map(|root| { + let root = match root { + QueryRoot::Operator(node) => QueryRoot::Operator(rewrite(node)?), + QueryRoot::Scalar(expr) => { + let mut rewritten = HashMap::new(); + for node in expr.operator_refs() { + rewritten.insert(Rc::as_ptr(node), rewrite(node)?); + } + QueryRoot::Scalar( + expr.map_operator_refs(&mut |node| rewritten[&Rc::as_ptr(node)].clone()), + ) + } + }; + // A substitution this module cannot express is an error, never an + // invalid DAG. + root.validate_structure()?; + Ok(root) + }) + .collect::, PerComputationCandidateError>>()?; + // The built states, as the roots hold them: with their guarantees. + for (state, _) in sizing.built.borrow_mut().iter_mut() { + if let Some(with_guarantee) = guaranteed.get(&Rc::as_ptr(state)) { + *state = with_guarantee.clone(); + } + } + Ok(roots) +} + +/// One target's chosen computation in a whole-workload choice. +struct ChosenComputation<'a> { + /// The target whose measure it computes. + target: &'a ComputationTarget, + /// How it is computed. + candidate: &'a HowToComputeAggregateMeasure, + /// Whether the candidate is the target's fused computation's, which + /// absorbs the targets beneath; `false` for one of its own candidates. + fused: bool, +} + +/// Each aggregate's targets with their chosen computations. +type Chosen<'a> = HashMap<*const OperatorNode, Vec>>; +type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; + +/// Whether any measure of the aggregate is substituted by something other than +/// its plain exact computation. +fn substituted(targets: &[ChosenComputation<'_>]) -> bool { + targets + .iter() + .any(|chosen| *chosen.candidate != HowToComputeAggregateMeasure::Exact) +} + +fn rewrite( + node: &Rc, + chosen: &Chosen<'_>, + pushed_below: &HashSet<*const OperatorNode>, + sizing: &Sizing, + cache: &mut NodeToRewrittenNodeCache, +) -> Result, PerComputationCandidateError> { + if let Some(done) = cache.get(&Rc::as_ptr(node)) { + return Ok(done.clone()); + } + for child in node.children() { + rewrite(child, chosen, pushed_below, sizing, cache)?; + } + let changed = node + .children() + .iter() + .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); + let rebuilt = match (chosen.get(&Rc::as_ptr(node)), node.non_asap()) { + (Some(targets), _) if substituted(targets) => substitute(node, targets, sizing, cache)?, + (_, Some(NonASAPOp::Filter { child, .. })) if pushed_below.contains(&Rc::as_ptr(node)) => { + cache[&Rc::as_ptr(child)].clone() + } + _ if changed => Rc::new(node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?), + _ => node.clone(), + }; + cache.insert(Rc::as_ptr(node), rebuilt.clone()); + Ok(rebuilt) +} + +/// Replace an aggregate whose measures are `targets`' (or exact, for a +/// measure that is no target) by its substitution, with the same output +/// columns. +fn substitute( + target: &Rc, + targets: &[ChosenComputation<'_>], + sizing: &Sizing, + cache: &NodeToRewrittenNodeCache, +) -> Result, PerComputationCandidateError> { + let Some(NonASAPOp::Aggregate { + child, + reduction, + measures, + filters, + .. + }) = target.non_asap() + else { + return Err(PerComputationCandidateError::Unsupported( + "target is not an aggregate", + )); + }; + // A key-only `HAVING` is a row predicate over the input: rows of a + // dropped group never reach the state. + let having = &targets[0].target.having_filters; + // Operator refs in a predicate (a subquery) are this aggregate's + // children, already rewritten. + let rewritten = |pred: &Predicate| { + Predicate( + pred.0 + .map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), + ) + }; + // The metric whose raw data samples, or plain sums of them, are every + // value the aggregate reads: a heap-sketch weight read from them carries + // the open requirement `WeightDomain::RawDataSamplesOfMetric`, which + // plan selection confirms. + let raw_data_samples = raw_data_samples_of_metric(child); + let mut input = cache[&Rc::as_ptr(child)].clone(); + for entry in having { + if entry.placement == HavingPlacement::BelowAggregate { + let keys = reduction.group_keys().map_or(&[][..], |by| by.keys()); + let pred = Predicate(entry.predicate.0.map_columns(&mut |column| keys[column])); + input = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { + pred, + child: input, + }))?; + } + } + let parts = measures + .iter() + .enumerate() + .map(|(measure, intent)| { + // `fused`: the candidate is the target's fused computation's, + // which absorbs the target beneath. + let (candidate, raw_data_samples_of, fused) = targets + .iter() + .find(|chosen| chosen.target.measure_index == measure) + .map_or( + (&HowToComputeAggregateMeasure::Exact, None, false), + |chosen| (chosen.candidate, raw_data_samples.as_deref(), chosen.fused), + ); + let filter = filters.get(measure).and_then(Option::as_ref).map(rewritten); + let key = (Rc::as_ptr(target), measure); + if fused { + return match candidate { + HowToComputeAggregateMeasure::ExactAccumulator(_) => nested_sum(target, cache), + HowToComputeAggregateMeasure::Summary(algorithm) => whole_expression( + target, + intent, + algorithm, + raw_data_samples_of, + (sizing, key), + cache, + ), + HowToComputeAggregateMeasure::Exact => { + Err(PerComputationCandidateError::Unsupported( + "plain exact execution of a fused computation", + )) + } + }; + } + substitute_measure( + target, + intent, + candidate, + filter, + &input, + raw_data_samples_of, + (sizing, key), + ) + }) + .collect::, _>>()?; + let mut output = match <[_; 1]>::try_from(parts) { + Ok([part]) => with_names_of(target, part)?, + Err(parts) => join_measures(target, parts)?, + }; + for entry in having { + if entry.placement == HavingPlacement::AboveOutput && entry.filter.is_none() { + output = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { + pred: rewritten(&entry.predicate), + child: output, + }))?; + } + } + Ok(output) +} + +/// `output` with `target`'s column names: an estimate or accumulator names +/// its value after the operation (`quantile`, `sum`), not the aggregate's +/// output column. +pub(crate) fn with_names_of( + target: &OperatorNode, + output: Rc, +) -> Result, PerComputationCandidateError> { + let names = |node: &OperatorNode| -> Vec { + node.schema + .fields + .iter() + .map(|field| field.name.clone()) + .collect() + }; + // A top-k's ranked rows may carry their item columns after the + // aggregate's ([`ranked_as_topk`]). + if names(&output).starts_with(&names(target)) { + return Ok(with_aggregate_qualifiers(target, output)); + } + if output.schema.fields.len() != target.schema.fields.len() { + return Err(PerComputationCandidateError::Unsupported( + "substituted output with other columns than the aggregate", + )); + } + let cols = names(target) + .into_iter() + .enumerate() + .map(|(column, name)| ProjectItem { + alias: Some(name), + expr: ScalarExpr::Column(column), + }) + .collect(); + Ok(with_aggregate_qualifiers( + target, + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { + cols, + qualifier: None, + child: output, + }))?, + )) +} + +/// An equivalent aggregate replacement must retain SQL field identity. These +/// explicit qualifiers survive child rebinding; types and keys remain derived +/// from executable operators rather than copied from the requested output. +fn with_aggregate_qualifiers(target: &OperatorNode, output: Rc) -> Rc { + if output + .schema + .fields + .iter() + .zip(&target.schema.fields) + .all(|(field, expected)| field.table == expected.table) + { + return output; + } + let mut output = output.as_ref().clone(); + for (field, expected) in output.schema.fields.iter_mut().zip(&target.schema.fields) { + field.table = expected.table.clone(); + } + Rc::new(output) +} + +/// A whole-expression top-k for `target` ([`whole_expression_input`]): one +/// heap summary of `algorithm` over the absorbed aggregate's rewritten +/// input, partitioned as `target`. +fn whole_expression( + target: &OperatorNode, + intent: &AggIntent, + algorithm: &SketchAlgorithm, + raw_data_samples_of: Option<&str>, + (sizing, key): (&Sizing, MeasureKey), + cache: &NodeToRewrittenNodeCache, +) -> Result, PerComputationCandidateError> { + let (input, update, partitions) = whole_expression_input(target).ok_or( + PerComputationCandidateError::Unsupported("whole-expression top-k input"), + )?; + let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: cache[&Rc::as_ptr(&input)].clone(), + input: with_raw_data_samples_requirement(update, raw_data_samples_of), + summary_type: FieldDataType::Sketch(*algorithm), + reduction: partitions, + filter: None, + }))?; + sizing.built.borrow_mut().push((state.clone(), key)); + let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: sketch_statistic(intent).ok_or(PerComputationCandidateError::Unsupported( + "sketch evaluation", + ))?, + }))?; + ranked_as_topk(target, estimate) +} + +/// A heap's top-k `estimate` (`[partitions.., items.., topk]`, one row per +/// ranked item) in `target`'s columns: its `[keys.., topk_k]`, with +/// `topk_k` the estimated Float64 weight, then the item columns. This is +/// the exact top-k's physical shape (its ranked rows' value, then the +/// columns naming each ranked item: PromQL labels, SQL group keys), so +/// consumers of `[keys.., topk_k]` find those columns where they were. +fn ranked_as_topk( + target: &OperatorNode, + estimate: Rc, +) -> Result, PerComputationCandidateError> { + let Some(NonASAPOp::Aggregate { reduction, .. }) = target.non_asap() else { + return Err(PerComputationCandidateError::Unsupported( + "top-k target is not an aggregate", + )); + }; + let ranked_index = reduction.group_keys().map_or(0, |keys| keys.len()); + let fields = &estimate.schema.fields; + let estimated = fields.len() - 1; + let cols = target + .schema + .fields + .iter() + .enumerate() + .map(|(index, field)| { + let source = if index == ranked_index { + estimated + } else { + fields[..estimated] + .iter() + .position(|input| input.name == field.name) + .ok_or(PerComputationCandidateError::Unsupported( + "top-k estimate without a ranked-row identity column", + ))? + }; + Ok(ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(source), + }) + }) + .collect::, PerComputationCandidateError>>()?; + Ok(OperatorNode::new_shared(Operator::NonASAP( + NonASAPOp::Project { + cols, + qualifier: None, + child: estimate, + }, + ))?) +} + +/// A nested-sum merge for `target` ([`nested_sum_input`]): one exact `Sum` +/// accumulator over the absorbed sum's rewritten input, grouped as +/// `target`, finalized. +fn nested_sum( + target: &OperatorNode, + cache: &NodeToRewrittenNodeCache, +) -> Result, PerComputationCandidateError> { + let (rows, intent, reduction) = nested_sum_input(target).ok_or( + PerComputationCandidateError::Unsupported("nested-sum merge input"), + )?; + let state = exact_accumulator( + ExactKind::Sum, + &intent, + &cache[&Rc::as_ptr(&rows)], + reduction, + )?; + Ok(OperatorNode::new_shared(Operator::ASAP( + ASAPOp::FinalizeExactAccumulator { child: state }, + ))?) +} + +/// An unfiltered exact accumulator of `kind` for `intent` over `input`, +/// grouped by `reduction`. +pub(crate) fn exact_accumulator( + kind: ExactKind, + intent: &AggIntent, + input: &Rc, + reduction: Reduction, +) -> Result, PerComputationCandidateError> { + let family = FieldDataType::exact(kind); + let read = measure_input(intent, input, None)?; + Ok(OperatorNode::new_shared(Operator::ASAP( + ASAPOp::SummaryAgg { + input: summary_update(intent, &family, &reduction, &input.schema, read)?, + child: Rc::clone(input), + summary_type: family, + reduction, + filter: None, + }, + ))?) +} + +/// One measure of `target` over `input`: its group keys, then its value. +fn substitute_measure( + target: &OperatorNode, + intent: &AggIntent, + candidate: &HowToComputeAggregateMeasure, + filter: Option, + input: &Rc, + raw_data_samples_of: Option<&str>, + (sizing, key): (&Sizing, MeasureKey), +) -> Result, PerComputationCandidateError> { + let Some(NonASAPOp::Aggregate { + reduction, + output_names, + .. + }) = target.non_asap() + else { + unreachable!("substitute checked the aggregate") + }; + let state = |family: FieldDataType, intent: &AggIntent, read: SummaryInputExpr| { + Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( + ASAPOp::SummaryAgg { + child: input.clone(), + input: with_raw_data_samples_requirement( + summary_update(intent, &family, reduction, &input.schema, read)?, + raw_data_samples_of, + ), + summary_type: family, + reduction: reduction.clone(), + filter: filter.clone(), + }, + ))?) + }; + let finalized = |kind: &ExactKind, intent: &AggIntent, read| { + let state = state(FieldDataType::exact(*kind), intent, read)?; + Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( + ASAPOp::FinalizeExactAccumulator { child: state }, + ))?) + }; + Ok(match candidate { + HowToComputeAggregateMeasure::Exact => { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { + child: input.clone(), + reduction: reduction.clone(), + measures: vec![intent.clone()], + // One name per measure, as the target names this one (an + // empty name is the intent's synthetic one). + output_names: vec![output_names.get(key.1).cloned().unwrap_or_default()], + filters: vec![filter.clone()], + having: None, + }))? + } + HowToComputeAggregateMeasure::ExactAccumulator(kinds) if kinds.len() == 1 => { + let read = measure_input(intent, input, filter.as_ref())?; + finalized(&kinds[0], intent, read)? + } + HowToComputeAggregateMeasure::Summary(algorithm) => { + let family = FieldDataType::Sketch(*algorithm); + let summary_input = state( + family, + intent, + measure_input(intent, input, filter.as_ref())?, + )?; + sizing.built.borrow_mut().push((summary_input.clone(), key)); + let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query: sketch_statistic(intent).ok_or( + PerComputationCandidateError::Unsupported("sketch evaluation"), + )?, + }))?; + match intent { + AggIntent::TopK { .. } => ranked_as_topk(target, estimate)?, + _ => estimate, + } + } + HowToComputeAggregateMeasure::ExactAccumulator(kinds) => { + let (AggIntent::Avg { col }, [ExactKind::Sum, ExactKind::Count]) = + (intent, kinds.as_slice()) + else { + return Err(PerComputationCandidateError::Unsupported( + "several exact states other than avg's sum then count", + )); + }; + let sum_intent = AggIntent::Sum { col: *col }; + // The count counts the averaged column, as `avg` does: it is + // non-null (a nullable input prunes this candidate), so it counts + // the rows the sum reads. + let read = measure_input(&sum_intent, input, filter.as_ref())?; + let sum = finalized(&ExactKind::Sum, &sum_intent, read.clone())?; + let count = finalized(&ExactKind::Count, &AggIntent::Count, read)?; + if input.result_kind == OperatorResultKind::Relation { + return sql_avg(reduction, filter.is_some(), sum, count); + } + // `avg` is Float64 whatever the summed type, as `Avg`'s output is, + // and a PromQL binary operator reads Float64 sample values. A + // grouped sum or count ends with its value, cast when not Float64 + // (a count is Int64). A per-series sum keeps the input's labels, + // so its value is the sample-value column; a per-series count is + // Float64. + let (sum, count) = match reduction { + Reduction::Reduce(by) => ( + last_as_float64(by.len(), sum)?, + last_as_float64(by.len(), count)?, + ), + Reduction::PerEntity => { + let value = resolve_column_ref(&ColumnRef::SampleValue, &sum.schema) + .ok() + .and_then(|column| sum.schema.fields.get(column)); + if value.map(|field| &field.dtype) + != Some(&FieldDataType::Plain(DataType::Float64)) + { + return Err(PerComputationCandidateError::Unsupported( + "non-Float64 per-series avg", + )); + } + (sum, count) + } + }; + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + // A per-series window may hold no finite sum to divide. + checked_finite_division: matches!(reduction, Reduction::PerEntity), + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: sum, + rhs: count, + }))? + } + }) +} + +/// `node`, a grouped aggregate's `keys` group keys and then its value, with +/// the value cast to Float64 unless it already is one. +fn last_as_float64( + keys: usize, + node: Rc, +) -> Result, PerComputationCandidateError> { + if node.schema.fields.last().map(|field| &field.dtype) + == Some(&FieldDataType::Plain(DataType::Float64)) + { + return Ok(node); + } + let mut cols = key_items(keys); + cols.push(ProjectItem { + alias: None, + expr: ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Column(keys)), + to: DataType::Float64, + try_cast: false, + }, + }); + Ok(OperatorNode::new_shared(Operator::NonASAP( + NonASAPOp::Project { + cols, + qualifier: None, + child: node, + }, + ))?) +} + +/// SQL `avg` from its finalized `sum` and `count`: joined on the group keys +/// (NULL keys match), then `sum / count` as Float64. A group with no row +/// passing the measure filter, or an empty global input, counts 0 and its +/// `avg` is NULL. +fn sql_avg( + reduction: &Reduction, + filtered: bool, + sum: Rc, + count: Rc, +) -> Result, PerComputationCandidateError> { + let Reduction::Reduce(by) = reduction else { + return Err(PerComputationCandidateError::Unsupported( + "SQL avg without `by` keys", + )); + }; + if by.is_without() { + return Err(PerComputationCandidateError::Unsupported( + "SQL avg without `by` keys", + )); + } + let keys = by.len(); + let float = |column| { + Box::new(ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Column(column)), + to: DataType::Float64, + try_cast: false, + }) + }; + let count_column = keys + 1 + keys; + let mut avg = ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Div, + left: float(keys), + right: float(count_column), + semantics: ExprSemantics::Sql, + }; + if filtered || keys == 0 { + avg = ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(count_column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: ExprSemantics::Sql, + }, + ScalarExpr::Literal(ScalarValue::Null), + )], + else_expr: Some(Box::new(avg)), + }; + } + let mut cols = key_items(keys); + cols.push(ProjectItem { + alias: None, + expr: avg, + }); + Ok(OperatorNode::new_shared(Operator::NonASAP( + NonASAPOp::Project { + cols, + qualifier: None, + child: join_on_keys(keys, sum, count)?, + }, + ))?) +} + +/// `left` joined with `right` on their leading `keys` columns; NULL keys +/// match each other, as grouping treats them. +fn join_on_keys( + keys: usize, + left: Rc, + right: Rc, +) -> Result, PerComputationCandidateError> { + let width = left.schema.fields.len(); + let column = |id| Box::new(ScalarExpr::Column(id)); + let matches = (0..keys) + .map(|key| { + let equal = ScalarExpr::Compare { + left: column(key), + op: CompareOpKind::Eq, + right: column(width + key), + semantics: ExprSemantics::Sql, + }; + if right.schema.fields[key].nullable { + ScalarExpr::BoolOr(vec![ + equal, + ScalarExpr::BoolAnd(vec![ + ScalarExpr::IsNull(column(key)), + ScalarExpr::IsNull(column(width + key)), + ]), + ]) + } else { + equal + } + }) + .collect::>(); + let (kind, pred) = match matches.len() { + 0 => ( + JoinKind::Cross, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ), + _ => (JoinKind::Inner, ScalarExpr::BoolAnd(matches)), + }; + Ok(OperatorNode::new_shared(Operator::NonASAP( + NonASAPOp::Join { + kind, + pred: Predicate(pred), + left, + right, + }, + ))?) +} + +/// Join single-measure `parts` (each: group keys, then one value) on their +/// group keys and project `target`'s output columns. Every part has the same +/// groups, so an inner join keeps them all; NULL keys match each other, as +/// grouping treats them. +fn join_measures( + target: &OperatorNode, + parts: Vec>, +) -> Result, PerComputationCandidateError> { + let Some(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(by), + .. + }) = target.non_asap() + else { + return Err(PerComputationCandidateError::Unsupported( + "multi-measure aggregate without `by` keys", + )); + }; + if by.is_without() { + return Err(PerComputationCandidateError::Unsupported( + "multi-measure aggregate without `by` keys", + )); + } + let keys = by.len(); + let width = keys + 1; + let mut parts = parts.into_iter(); + let mut joined = parts.next().expect("an aggregate has measures"); + for part in parts { + joined = join_on_keys(keys, joined, part)?; + } + let mut cols = key_items(keys); + cols.extend( + (0..target.schema.fields.len() - keys).map(|measure| ProjectItem { + alias: None, + expr: ScalarExpr::Column(measure * width + keys), + }), + ); + for (item, field) in cols.iter_mut().zip(&target.schema.fields) { + item.alias = Some(field.name.clone()); + } + Ok(OperatorNode::new_shared(Operator::NonASAP( + NonASAPOp::Project { + cols, + qualifier: None, + child: joined, + }, + ))?) +} + +fn key_items(keys: usize) -> Vec { + (0..keys) + .map(|key| ProjectItem { + alias: None, + expr: ScalarExpr::Column(key), + }) + .collect() +} + +/// What a measure's state reads from each row of `input`, resolvable there. +/// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count +/// included, reads its input column (the sample value when it names none). +fn measure_input( + intent: &AggIntent, + input: &Rc, + filter: Option<&Predicate>, +) -> Result { + let schema = &input.schema; + let read = match intent { + AggIntent::Count if input.result_kind == OperatorResultKind::Relation => { + SummaryInputExpr::Column(counted_column(schema, filter)?) + } + // A top-k ranks its child's aggregate output, not a column of its own. + AggIntent::TopK { .. } => SummaryInputExpr::Column( + ranked_value(input) + .map_err(|_| { + PerComputationCandidateError::Unsupported("Top-K has no ranked column") + })? + .1, + ), + _ => summarised_input(intent, schema).ok_or(PerComputationCandidateError::Unsupported( + "input column outside child schema", + ))?, + }; + if let SummaryInputExpr::Column(column) = &read { + resolve_column_ref(column, schema).map_err(|_| { + PerComputationCandidateError::Unsupported("summary input column does not resolve") + })?; + } + Ok(read) +} + +/// The column a SQL `count` state updates on. The runtime skips NULL updates, +/// so it is a column non-null on every counted row: one the measure filter +/// proves non-null (`count(col)` lowers to the filter `col IS NOT NULL`), else +/// a non-nullable one, Float64 first (the runtime's numeric update). +fn counted_column( + schema: &Schema, + filter: Option<&Predicate>, +) -> Result { + fn not_null(expr: &ScalarExpr) -> Vec { + match expr { + ScalarExpr::IsNotNull(inner) => match **inner { + ScalarExpr::Column(column) => vec![column], + _ => vec![], + }, + ScalarExpr::BoolAnd(conjuncts) => conjuncts.iter().flat_map(not_null).collect(), + _ => vec![], + } + } + let non_nullable = |float: bool| { + schema + .fields + .iter() + .enumerate() + .filter_map(move |(id, field)| { + (!field.nullable + && (!float || field.dtype == FieldDataType::Plain(DataType::Float64))) + .then_some(id) + }) + }; + let reference = |id: usize| { + schema + .fields + .get(id) + .map(asap_types::ir::schema::Field::column_ref) + }; + filter + .map_or_else(Vec::new, |filter| not_null(&filter.0)) + .into_iter() + .chain(non_nullable(true)) + .chain(non_nullable(false)) + .find_map(|id| { + reference(id).filter(|column| resolve_column_ref(column, schema).ok() == Some(id)) + }) + .ok_or(PerComputationCandidateError::Unsupported( + "SQL count over rows with no column known to be non-null", + )) +} + +/// What each input row contributes, given what the measure reads +/// ([`measure_input`]), following the legacy realization rules: heap sketches +/// rank series identities, frequency sketches count values, and every other +/// family reads the measure's input. +fn summary_update( + intent: &AggIntent, + family: &FieldDataType, + reduction: &Reduction, + child: &Schema, + weight: SummaryInputExpr, +) -> Result { + let algorithm = match family { + FieldDataType::Sketch(kind) => Some(kind), + _ => None, + }; + Ok(match (intent, algorithm) { + (AggIntent::TopK { .. }, Some(_)) => { + if reduction.group_keys().is_some_and(|keys| keys.is_without()) { + return Err(PerComputationCandidateError::Unsupported( + "Top-K partitions given by `without`", + )); + } + let groups: Vec = reduction + .group_keys() + .into_iter() + .flat_map(|keys| keys.iter().copied()) + .collect(); + let item = if child.has_promql_series_identity() { + asap_types::ir::schema::topk::series_identity_item(child, &groups) + } else if child.closed { + // A closed child (an aggregate's output, say) lists every + // column: the item is its columns outside the partition and + // the ranked value, as the legacy current-series and counter + // rules rank them. + let SummaryInputExpr::Column(ranked) = &weight else { + return Err(PerComputationCandidateError::Unsupported( + "Top-K ranked value is not a column", + )); + }; + let value = resolve_column_ref(ranked, child).map_err(|_| { + PerComputationCandidateError::Unsupported("Top-K ranked value does not resolve") + })?; + let items: Vec<_> = child + .fields + .iter() + .enumerate() + .filter(|(index, _)| *index != value && !groups.contains(index)) + .map(|(_, field)| SummaryInputExpr::Column(field.column_ref())) + .collect(); + if items.is_empty() { + return Err(PerComputationCandidateError::Unsupported( + "Top-K item has no label outside its partition", + )); + } + SummaryInputExpr::Tuple(items) + } else { + let excluding = groups + .iter() + .filter_map(|&index| child.fields.get(index)) + .map(asap_types::ir::schema::Field::column_ref) + .collect(); + SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { excluding }) + }; + SummaryUpdate { + item: Some(item), + // The ranked value ([`measure_input`]). + weight, + // Not proven non-negative; selection decides whether CMS is admissible. + weight_domain: WeightDomain::UnknownOrSigned, + } + } + (_, Some(SketchAlgorithm::UnivMon)) | (AggIntent::Count, Some(SketchAlgorithm::Cms)) => { + SummaryUpdate { + item: Some(weight), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + } + } + _ => SummaryUpdate { + item: None, + weight, + weight_domain: WeightDomain::UnknownOrSigned, + }, + }) +} + +/// The column fed into a *single-column* summary: the intent's leading +/// positional input resolved to a name against the child schema, or the PromQL +/// sample value when it reads none. Callers are responsible for only reaching +/// here with a one-column intent — [`summarised_input`] is the general form. +pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { + match intent + .input_cols() + .first() + .and_then(|id| child_schema.fields.get(*id)) + { + Some(c) => c.column_ref(), + None => ColumnRef::SampleValue, + } +} + +/// [`ranked_column`] as a reference: the sample value when it is that +/// column, else the column by name. +pub(crate) fn ranked_value( + child: &Rc, +) -> Result<(ColumnId, ColumnRef), ResolveError> { + let ranked = ranked_column(child)?; + let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { + Ok(value) if value == ranked => ColumnRef::SampleValue, + _ => child.schema.fields[ranked].column_ref(), + }; + Ok((ranked, column)) +} + +/// What the summary consumes per input row. An intent that reads one column (or +/// none) feeds that column; `COUNT(DISTINCT a, b)` feeds the whole tuple as one +/// item, so the distinct-count sketch hashes `(a, b)` rather than `a` — the +/// difference between tuple cardinality and single-column cardinality. +/// +/// A tuple leg outside the child schema is an error rather than +/// [`summarised_column`]'s sample-value fallback: a leg has no sample-value +/// reading, and silently dropping one would under-count: `None` then. +pub(crate) fn summarised_input( + intent: &AggIntent, + child_schema: &Schema, +) -> Option { + let cols = intent.input_cols(); + if cols.len() < 2 { + return Some(SummaryInputExpr::Column(summarised_column( + intent, + child_schema, + ))); + } + let legs = cols + .iter() + .map(|id| child_schema.fields.get(*id).map(Field::column_ref)) + .collect::>>()?; + Some(SummaryInputExpr::Tuple( + legs.into_iter().map(SummaryInputExpr::Column).collect(), + )) +} diff --git a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs index 58c8a80a4..92a18e5b8 100644 --- a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs @@ -3,11 +3,6 @@ //! configuration, fused builds), with every intermediate result its own //! group (a [`Memo`]), which Pass 2 extends. -use asap_types::ir::properties::AccuracyTarget; -use asap_types::workload::DataWorkload; - -use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; - use crate::pass1::per_computation_candidates::{ generate_per_computation_candidates, PerComputationCandidateError, }; @@ -29,25 +24,15 @@ pub enum Pass1Error { /// execution, drops the summaries the deployment does not support /// (`supported_summaries`, the deployment's /// `DeploymentCapabilities::supported_summaries`; `None` supports every -/// one) and, unless `accuracy` or `evidence` may accept an output without -/// its guarantee, the summaries no query reading them can use (a query -/// `q`'s target is `accuracy_targets[q]`). It then adds each way to compute -/// a group as candidates, through `Memo::find_or_add_group` and -/// `Memo::add_candidate`. +/// one), and adds each way to compute a group as candidates, through +/// `Memo::find_or_add_group` and `Memo::add_candidate`. Accuracy prunes +/// nothing here: Pass 2's dead-candidate elimination does. pub fn pass1_memo( memo: Memo, - accuracy_targets: &[AccuracyTarget], - accuracy: &dyn AccuracyModel, - evidence: &dyn AccuracyEvidenceProvider, - data: &DataWorkload, supported_summaries: Option<&[SupportedSummary]>, ) -> Result { - let mut per_computation = generate_per_computation_candidates(&memo)? + let per_computation = generate_per_computation_candidates(&memo)? .without_unsupported_summaries(supported_summaries); - if !accuracy.may_accept_regardless_of_guarantee() && !evidence.may_provide_evidence() { - per_computation = - per_computation.without_summaries_no_query_can_use(accuracy_targets, data, accuracy); - } let _ = (memo, per_computation); todo!("#1207 PR 3: add each way to compute as candidates, one per listed θ") } diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 34e7cc85d..0580f24d7 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -23,11 +23,11 @@ //! grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). //! - Summary choices here are unsized families. Adding them to memo groups //! creates one build candidate per listed parameter configuration -//! (`parameter_list`); Stage 3 checks each query's composed accuracy at its -//! root. -//! - Candidates are pruned only when provably unable to meet the accuracy -//! requirement or to run, or when the exact candidate provably dominates -//! them (a PromQL top-k's heap over its instant vector, one row per item). +//! (`parameter_list`); Pass 2 gives readouts their required accuracy. +//! - Candidates are pruned only when provably unable to run, or when the +//! exact candidate provably dominates them (a PromQL top-k's heap over its +//! instant vector, one row per item); accuracy prunes nothing here (Pass +//! 2's dead-candidate elimination does). //! - A measure filter or `HAVING` prunes nothing. The target records the //! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] //! per consumer `Filter` directly above the aggregate: a key-only predicate @@ -37,10 +37,8 @@ //! computes the group's own rows. //! //! The input must be a logical DAG: ASAP operators are rejected. A logical -//! DAG has no physical properties (#1200). No cost model is used. The accuracy model -//! is used only to prune summaries no query reading them can use -//! ([`PerComputationCandidates::without_summaries_no_query_can_use`]); -//! ranking and discarding valid candidates belong to Stage 3. +//! DAG has no physical properties (#1200). No cost or accuracy model is +//! used; ranking and discarding valid candidates belong to Stage 3. //! //! # Whole-workload candidates (logical ASAP-aware optimization, end of Pass 1) //! @@ -102,32 +100,18 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{DataType, Schema}; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::{ - EntityIdentity, ExactKind, FieldDataType, GroupingStrategy, NonNegativeWeightProof, - SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, + ExactKind, FieldDataType, SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, }; use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, - Predicate, ProjectItem, QueryRoot, ScalarExpr, SchemaDerivationError, + NonASAPOp, OperatorNode, OperatorResultKind, Predicate, QueryRoot, SchemaDerivationError, }; use thiserror::Error; -use asap_accuracy::composition::parse_raw_data_samples_sign_statistic; -use asap_accuracy::estimators::hll; -use asap_accuracy::propagation::{ - bind_guarantees, raw_data_samples_of_metric, root_guarantee, with_guarantees, -}; -use asap_accuracy::sizing::{accuracy_budget, sizing_target, sketch_statistic}; -use asap_accuracy::AccuracyModel; -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::workload::bounds::{declared_non_negative, series_read, values_per_result}; -use asap_types::workload::DataWorkload; +use asap_accuracy::sizing::sketch_statistic; use crate::pass1::realization::summary_candidates; use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; @@ -182,7 +166,9 @@ fn prune_own( pub struct PerComputationCandidates { /// Each query's root over the memo groups' logical nodes, in workload /// order: a group several queries read is one node. Whole-workload - /// candidates are these roots with chosen candidates substituted. + /// candidates are these roots with chosen candidates substituted + /// (test support only). + #[cfg(any(test, feature = "test-support"))] pub roots: Vec, /// Each memo group's logical node, by group index. pub(crate) logical_nodes: Rc<[Rc]>, @@ -193,6 +179,7 @@ pub struct PerComputationCandidates { impl PerComputationCandidates { /// The `Aggregate` node of `target`'s memo group. + #[cfg(any(test, feature = "test-support"))] pub fn aggregate(&self, target: &ComputationTarget) -> &Rc { &self.logical_nodes[target.aggregate_group.0 as usize] } @@ -284,254 +271,6 @@ impl PerComputationCandidates { } self } - - /// This inventory without the summary candidates no query can use - /// (#1047, accuracy A1 of #1095): a summary of a target (its own, or - /// its fused computation's) is removed when, for every query - /// whose root reaches the target's aggregate, - /// `output_can_meet_target` says no: the query's output, composed - /// with that summary and every other target exact, has no error bound - /// that can meet the query's target (`query_targets[q]`) at any size, - /// as `model` judges metrics. Every other target exact is the most - /// favorable combination: an approximate input never adds a guarantee. - /// A summary some query can use is kept, even when another query - /// reaching the group cannot; plan selection rejects it for that query. - /// - /// Pass 1 cannot see Pass 2's reuse of one group's summary by another - /// group (rollup, window rollup, finer-aggregate reuse, top-k reuse), so - /// a summary is also kept whenever another target, over another - /// aggregate reading one of the same sources, offers the same family - /// and some query reading that target can use it: Pass 2 may build that - /// target's state from this one, and the reused state composes through - /// that target's queries as its own would. This keeps more than those - /// rules' preconditions need. Exact candidates are never - /// pruned, and with every fused summary pruned the fusion is not - /// offered. A summary whose composition fails is kept. - /// - /// Pass 1's entry (`pass1_memo`) - /// calls it only when the deployment's accuracy model and evidence - /// cannot accept an output without its guarantee. - pub fn without_summaries_no_query_can_use( - mut self, - query_targets: &[AccuracyTarget], - data: &DataWorkload, - model: &dyn AccuracyModel, - ) -> Self { - // The aggregates each query's root reaches. - let reached: Vec> = self - .roots - .iter() - .map(|root| { - root.operator_refs() - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - .map(|node| Rc::as_ptr(&node)) - .collect() - }) - .collect(); - // Each target's sources, and every family it offers. - let sources: Vec> = self - .targets - .iter() - .map(|target| sources_read(self.aggregate(target))) - .collect(); - let families = |target: &ComputationTarget| -> Vec { - (0..target.choice_count()) - .filter_map(|choice| match target.computation_for(choice) { - Some((HowToComputeAggregateMeasure::Summary(family), _)) => Some(*family), - _ => None, - }) - .collect() - }; - let offered: Vec> = self.targets.iter().map(families).collect(); - // Per target, its summary choices no query reading its aggregate can - // use; a target no query reaches, or whose composition fails, keeps - // every choice. - let mut unusable_by_own_queries: Vec> = vec![Vec::new(); self.targets.len()]; - for (index, target) in self.targets.iter().enumerate() { - let aggregate = Rc::as_ptr(self.aggregate(target)); - let queries: Vec = (0..self.roots.len()) - .filter(|&q| reached[q].contains(&aggregate) && q < query_targets.len()) - .collect(); - if queries.is_empty() { - continue; - } - let roots: Vec = queries.iter().map(|&q| self.roots[q].clone()).collect(); - for choice in 0..target.choice_count() { - let Some((HowToComputeAggregateMeasure::Summary(_), _)) = - target.computation_for(choice) - else { - continue; - }; - let mut choices = vec![0; self.targets.len()]; - choices[index] = choice; - let Ok(composed) = compose(&self, &roots, &choices) else { - continue; - }; - if !queries - .iter() - .zip(&composed) - .any(|(&q, root)| output_can_meet_target(root, &query_targets[q], data, model)) - { - unusable_by_own_queries[index].push(choice); - } - } - } - // The families each target's own queries can use. - let usable_families: Vec> = self - .targets - .iter() - .enumerate() - .map(|(index, target)| { - (0..target.choice_count()) - .filter(|choice| !unusable_by_own_queries[index].contains(choice)) - .filter_map(|choice| match target.computation_for(choice) { - Some((HowToComputeAggregateMeasure::Summary(family), _)) => Some(*family), - _ => None, - }) - .collect() - }) - .collect(); - // Whether Pass 2 may reuse target `index`'s `family` for another - // target's aggregate whose own queries can use it: a reused state - // composes through that target's queries as its own would. - let reusable = |index: usize, family: &SketchAlgorithm| { - let aggregate = self.targets[index].aggregate_group; - self.targets.iter().enumerate().any(|(other, target)| { - target.aggregate_group != aggregate - && offered[other].contains(family) - && usable_families[other].contains(family) - && sources[other] - .iter() - .any(|source| sources[index].contains(source)) - }) - }; - let mut unusable: Vec<(usize, usize)> = Vec::new(); - for (index, choices) in unusable_by_own_queries.iter().enumerate() { - for &choice in choices { - let Some((HowToComputeAggregateMeasure::Summary(family), _)) = - self.targets[index].computation_for(choice) - else { - continue; - }; - if !reusable(index, family) { - unusable.push((index, choice)); - } - } - } - for (index, target) in self.targets.iter_mut().enumerate() { - let own = target.ways_to_compute.len(); - let pruned_choices: Vec = unusable - .iter() - .filter(|(i, _)| *i == index) - .map(|(_, choice)| *choice) - .collect(); - if pruned_choices.is_empty() { - continue; - } - let own_pruned: Vec = pruned_choices - .iter() - .filter(|&&choice| choice < own) - .map(|&choice| target.ways_to_compute[choice].clone()) - .collect(); - let fused_pruned: Vec = pruned_choices - .iter() - .filter(|&&choice| choice >= own) - .filter_map(|&choice| { - let fused = target.fused_computation.as_ref()?; - fused.ways_to_compute.get(choice - own).cloned() - }) - .collect(); - prune_own(&mut target.ways_to_compute, |way| own_pruned.contains(way)); - if let Some(fused) = &mut target.fused_computation { - fused - .ways_to_compute - .retain(|way| !fused_pruned.contains(way)); - } - } - self - } -} - -/// The sources every scan below `node` reads, each once. -fn sources_read(node: &Rc) -> Vec { - let mut sources = Vec::new(); - for node in OperatorNode::reachable(node) { - if let Some(NonASAPOp::Scan { source, .. }) = node.non_asap() { - if !sources.contains(source) { - sources.push(source.clone()); - } - } - } - sources -} - -/// Whether a query's composed output `root` can meet `target` at some size -/// of its summaries: its guarantee is exact, or it has one in a metric -/// `model` says can meet `target` ([`AccuracyModel::metric_can_meet`]; an -/// exact target accepts only exact) whose error, and failure probability -/// when `target` sets one, are known once the statistics Pass 1 left -/// symbolic are bound and the guarantee is stated over the values the -/// target's scope covers, as plan selection states it. The sign of a -/// metric's samples is bound as plan selection binds it, proven only when -/// `data` declares the metric a counter or non-negative -/// ([`declared_non_negative`]). Classic HLL's failure probability is bound -/// from `data`'s declared distinct bound for its source at the largest -/// precision the contract covers: a bound outside the contract there (no -/// certificate, failure probability 1) is outside it at every size, so it -/// stays unbound. Every other statistic (row and series counts, stream -/// norms, an undeclared distinct bound, operand domains) is assumed -/// bindable, so no candidate is pruned for one plan selection may bind. -/// The number of values a whole-result target covers is `data`'s -/// ([`values_per_result`]): when it is not bound, a per-value guarantee -/// that may fail never meets a whole-result failure probability, whatever -/// the size (sizing splits δ over those values). -pub(crate) fn output_can_meet_target( - root: &QueryRoot, - target: &AccuracyTarget, - data: &DataWorkload, - model: &dyn AccuracyModel, -) -> bool { - let bind = |guarantee: &AccuracyGuarantee| { - guarantee.bind(&|statistic| { - if let Some(metric) = parse_raw_data_samples_sign_statistic(statistic) { - return declared_non_negative(data, metric).then_some(1.0); - } - if let Some((_, relative_error, source)) = - hll::parse_classic_failure_statistic(statistic) - { - // At the largest precision the contract covers: no - // certificate there, none at any size. - let declared = data - .sources - .get(source) - .and_then(|facts| facts.max_distinct_per_evaluation); - if let Some(bound) = declared { - let best = - hll::classic_failure_probability(hll::MAX_PRECISION, relative_error, bound); - return (best < 1.0).then_some(best); - } - } - Some(1.0) - }) - }; - let guarantee = match root_guarantee(root) { - Some(guarantee) => Some(bind(&guarantee)), - None => bind_guarantees(root, &bind), - }; - let Some(guarantee) = guarantee else { - return false; - }; - if guarantee.is_exact() { - return true; - } - let values = values_per_result(root, series_read(root, data), data); - let guarantee = guarantee.over_scope(target.accuracy_scope, values); - model.metric_can_meet(guarantee.spec.accuracy_metric, target) - && guarantee.spec.error.evaluate().is_some() - && (target.failure_probability.is_none() - || guarantee.spec.failure_probability.evaluate().is_some()) } /// The indices of `targets` some target's fused computation absorbs. @@ -549,17 +288,6 @@ pub enum PerComputationCandidateError { Structure(#[from] SchemaDerivationError), #[error("logical candidate input already contains ASAP operators")] ContainsAsap, - /// `root` (the query's index in the workload) and `measure` locate the target; - /// `computation_candidates_for_intent` alone leaves them `None`. - #[error( - "approximate accuracy of root {}, measure {} requires epsilon and delta in (0, 1)", - root.as_deref().unwrap_or("?"), - measure.map_or_else(|| "?".to_string(), |measure| measure.to_string()) - )] - InvalidAccuracy { - root: Option, - measure: Option, - }, #[error("choice must name one listed candidate per target")] InvalidChoice, #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] @@ -570,11 +298,9 @@ pub enum PerComputationCandidateError { /// order, without ranking or empirical assessment. A summary is not a claim that /// a deployment meets the measure's accuracy requirement. The one exact option /// comes first: the exact accumulator form if the intent has one, else `Exact`. -pub(super) fn computation_candidates_for_intent( - intent: &AggIntent, -) -> Result { +pub(super) fn computation_candidates_for_intent(intent: &AggIntent) -> ComputationChoices { let states = match intent { - AggIntent::Count { .. } => vec![ExactKind::Count], + AggIntent::Count => vec![ExactKind::Count], AggIntent::Sum { .. } => vec![ExactKind::Sum], AggIntent::Min { .. } => vec![ExactKind::Min], AggIntent::Max { .. } => vec![ExactKind::Max], @@ -593,21 +319,9 @@ pub(super) fn computation_candidates_for_intent( candidates: vec![exact], pruned: vec![], }; - if let Some(target) = sizing_target(intent) { - if !target.is_exact() { - let (epsilon, delta) = accuracy_budget(&target); - let unit = |value: f64| value > 0.0 && value < 1.0; - if !unit(epsilon) || !unit(delta) { - return Err(PerComputationCandidateError::InvalidAccuracy { - root: None, - measure: None, - }); - } - } + if sketch_statistic(intent).is_some() { let summaries = summary_candidates(intent).into_iter(); - if matches!(intent, AggIntent::Quantile { q, .. } if !(0.0..=1.0).contains(q)) - || target.is_exact() - { + if matches!(intent, AggIntent::Quantile { q, .. } if !(0.0..=1.0).contains(q)) { choices .pruned .extend(summaries.map(HowToComputeAggregateMeasure::Summary)); @@ -617,7 +331,7 @@ pub(super) fn computation_candidates_for_intent( .extend(summaries.map(HowToComputeAggregateMeasure::Summary)); } } - Ok(choices) + choices } /// Whether an `avg` measure's input may be null, so `count` rows differ from @@ -699,12 +413,12 @@ pub fn generate_per_computation_candidates( }) .collect(); let mut seen = HashSet::new(); - // Each aggregate group with the first query that reaches it. + // Each aggregate group. let mut aggregates = Vec::new(); // Every consumer of each group's node: `Some(parent)`, or `None` for a // query root or scalar reference. let mut consumers: HashMap<*const OperatorNode, Vec>>> = HashMap::new(); - for (query, root) in roots.iter().enumerate() { + for root in &roots { root.validate_structure()?; let operators = match root { QueryRoot::Operator(node) => vec![node], @@ -726,13 +440,13 @@ pub fn generate_per_computation_candidates( .push(Some(Rc::clone(&node))); } if matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })) { - aggregates.push((query, node)); + aggregates.push(node); } } } } let mut targets = Vec::new(); - for (query, node) in aggregates { + for node in aggregates { let Some(NonASAPOp::Aggregate { measures, filters, @@ -777,16 +491,7 @@ pub fn generate_per_computation_candidates( }); } for (measure, intent) in measures.iter().enumerate() { - let mut choices = - computation_candidates_for_intent(intent).map_err(|error| match error { - PerComputationCandidateError::InvalidAccuracy { .. } => { - PerComputationCandidateError::InvalidAccuracy { - root: Some(query.to_string()), - measure: Some(measure), - } - } - error => error, - })?; + let mut choices = computation_candidates_for_intent(intent); if choices.candidates == [HowToComputeAggregateMeasure::Exact] && choices.pruned.is_empty() { @@ -797,7 +502,7 @@ pub fn generate_per_computation_candidates( matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_)) }); } - if matches!(intent, AggIntent::Count { .. }) + if matches!(intent, AggIntent::Count) && node.children()[0].result_kind == OperatorResultKind::Relation { choices.prune(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_))); @@ -817,6 +522,7 @@ pub fn generate_per_computation_candidates( add_nested_sum_targets(&mut targets, &logical_nodes); close_absorptions(&mut targets); Ok(PerComputationCandidates { + #[cfg(any(test, feature = "test-support"))] roots, logical_nodes, targets, @@ -919,7 +625,9 @@ fn add_nested_sum_targets(targets: &mut [ComputationTarget], logical_nodes: &[Rc /// (not by `without`) of the value column of a per-series or grouped `sum` /// directly beneath it, neither filtered nor with `HAVING`, and not grouped /// by the metric name a per-series inner sum drops. -fn nested_sum_input(target: &OperatorNode) -> Option<(Rc, AggIntent, Reduction)> { +pub(super) fn nested_sum_input( + target: &OperatorNode, +) -> Option<(Rc, AggIntent, Reduction)> { use asap_types::ir::operator::non_asap::any_measure_filtered; let Some(NonASAPOp::Aggregate { child: inner, @@ -1108,7 +816,7 @@ fn reads_only_time_series(node: &OperatorNode) -> bool { /// does not apply. Rows that carry the full series identity rank it as a /// column, as [`summary_update`] does. The partitions are `target`'s, moved /// onto the input the heap reads ([`Reduction::partitions_over`]). -fn whole_expression_input( +pub(super) fn whole_expression_input( target: &OperatorNode, ) -> Option<(Rc, SummaryUpdate, Reduction)> { use crate::pass1::realization::realize_keyed_additive_summary_input; @@ -1133,10 +841,7 @@ fn whole_expression_input( // weights it cannot prove non-negative. Ask with CountSketch so one // update serves both heap families; plan selection decides whether // Count-Min is admissible, as for every other Count-Min candidate. - let family = FieldDataType::Sketch( - SketchAlgorithm::CountSketchWithHeap, - GroupingStrategy::default(), - ); + let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; let mut input = realized.input; let per_series = matches!( @@ -1164,984 +869,6 @@ fn whole_expression_input( Some((realized.child, input, partitions)) } -/// `update` with the open requirement that its weights are samples of -/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap -/// update whose weights are not otherwise proven. -fn with_raw_data_samples_requirement( - mut update: SummaryUpdate, - metric: Option<&str>, -) -> SummaryUpdate { - if let Some(metric) = metric { - if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { - update.weight_domain = WeightDomain::RawDataSamplesOfMetric { - metric: metric.to_string(), - }; - } - } - update -} - -/// Build one whole-workload candidate (end of Pass 1): `choice[i]` picks -/// `inventory.targets[i]`'s candidate, or past its own candidates its fused -/// computation's ([`ComputationTarget::computation_for`]). An aggregate with -/// a chosen candidate other than `Exact` is replaced measure by measure: an -/// exact accumulator by `SummaryAgg` → `FinalizeExactAccumulator`, a summary -/// by `SummaryAgg` → `SummaryEstimate`, `avg` by its `sum` and `count` -/// accumulators divided, a fused computation by one state over the absorbed -/// target's input, and any other measure by its own single-measure -/// aggregate; several measures are joined back on the group keys. A summary -/// is sized here for its measure's accuracy requirement. A measure filter -/// becomes `SummaryAgg.filter`; -/// `HAVING` is applied as its target records. A sub-DAG shared between roots -/// stays shared. Every node carries the guarantee `asap_accuracy::propagation::composed_guarantee` -/// gives it. -#[cfg(any(test, feature = "test-support"))] -pub fn compose_logical_candidate( - inventory: &PerComputationCandidates, - choice: &[usize], -) -> Result, PerComputationCandidateError> { - compose(inventory, &inventory.roots, choice) -} - -/// [`compose_logical_candidate`] of `roots` (some of `inventory.roots`), -/// each summary sized for its measure's whole budget. -fn compose( - inventory: &PerComputationCandidates, - roots: &[QueryRoot], - choice: &[usize], -) -> Result, PerComputationCandidateError> { - if choice.len() != inventory.targets.len() { - return Err(PerComputationCandidateError::InvalidChoice); - } - let mut chosen: Chosen = HashMap::new(); - for (target, &index) in inventory.targets.iter().zip(choice) { - let (candidate, absorbs) = target - .computation_for(index) - .ok_or(PerComputationCandidateError::InvalidChoice)?; - chosen - .entry(Rc::as_ptr(inventory.aggregate(target))) - .or_default() - .push(ChosenComputation { - target, - candidate, - fused: absorbs.is_some(), - }); - } - // Consumer `Filter`s whose key-only `HAVING` moves below a substituted - // aggregate: each is replaced by its input. - let pushed_below: HashSet<_> = chosen - .values() - .filter(|targets| substituted(targets)) - .flat_map(|targets| &targets[0].target.having_filters) - .filter(|entry| entry.placement == HavingPlacement::BelowAggregate) - .filter_map(|entry| entry.filter.as_ref().map(Rc::as_ptr)) - .collect(); - compose_roots(roots, &chosen, &pushed_below, &Sizing::default()) -} - -/// One measure of an aggregate: the aggregate and the measure's index. -type MeasureKey = (*const OperatorNode, usize); - -/// How [`compose_logical_candidate`] sizes summaries in one composition. -#[derive(Default)] -struct Sizing { - /// Every sketch summary state built, with the measure it sizes. - built: std::cell::RefCell, MeasureKey)>>, -} - -/// Every root of `roots` with the chosen candidates substituted, each -/// summary sized as `sizing` says, and every node with its guarantee. -fn compose_roots( - roots: &[QueryRoot], - chosen: &Chosen<'_>, - pushed_below: &HashSet<*const OperatorNode>, - sizing: &Sizing, -) -> Result, PerComputationCandidateError> { - let (mut cache, mut guaranteed) = (HashMap::new(), HashMap::new()); - let mut rewrite = |node: &Rc| { - let node = rewrite(node, chosen, pushed_below, sizing, &mut cache)?; - with_guarantees(&node, &mut guaranteed).map_err(PerComputationCandidateError::from) - }; - let roots = roots - .iter() - .map(|root| { - let root = match root { - QueryRoot::Operator(node) => QueryRoot::Operator(rewrite(node)?), - QueryRoot::Scalar(expr) => { - let mut rewritten = HashMap::new(); - for node in expr.operator_refs() { - rewritten.insert(Rc::as_ptr(node), rewrite(node)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| rewritten[&Rc::as_ptr(node)].clone()), - ) - } - }; - // A substitution this module cannot express is an error, never an - // invalid DAG. - root.validate_structure()?; - Ok(root) - }) - .collect::, PerComputationCandidateError>>()?; - // The built states, as the roots hold them: with their guarantees. - for (state, _) in sizing.built.borrow_mut().iter_mut() { - if let Some(with_guarantee) = guaranteed.get(&Rc::as_ptr(state)) { - *state = with_guarantee.clone(); - } - } - Ok(roots) -} - -/// One target's chosen computation in a whole-workload choice. -struct ChosenComputation<'a> { - /// The target whose measure it computes. - target: &'a ComputationTarget, - /// How it is computed. - candidate: &'a HowToComputeAggregateMeasure, - /// Whether the candidate is the target's fused computation's, which - /// absorbs the targets beneath; `false` for one of its own candidates. - fused: bool, -} - -/// Each aggregate's targets with their chosen computations. -type Chosen<'a> = HashMap<*const OperatorNode, Vec>>; -type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; - -/// Whether any measure of the aggregate is substituted by something other than -/// its plain exact computation. -fn substituted(targets: &[ChosenComputation<'_>]) -> bool { - targets - .iter() - .any(|chosen| *chosen.candidate != HowToComputeAggregateMeasure::Exact) -} - -fn rewrite( - node: &Rc, - chosen: &Chosen<'_>, - pushed_below: &HashSet<*const OperatorNode>, - sizing: &Sizing, - cache: &mut NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - rewrite(child, chosen, pushed_below, sizing, cache)?; - } - let changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); - let rebuilt = match (chosen.get(&Rc::as_ptr(node)), node.non_asap()) { - (Some(targets), _) if substituted(targets) => substitute(node, targets, sizing, cache)?, - (_, Some(NonASAPOp::Filter { child, .. })) if pushed_below.contains(&Rc::as_ptr(node)) => { - cache[&Rc::as_ptr(child)].clone() - } - _ if changed => Rc::new(node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?), - _ => node.clone(), - }; - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) -} - -/// Replace an aggregate whose measures are `targets`' (or exact, for a -/// measure that is no target) by its substitution, with the same output -/// columns. -fn substitute( - target: &Rc, - targets: &[ChosenComputation<'_>], - sizing: &Sizing, - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - child, - reduction, - measures, - filters, - .. - }) = target.non_asap() - else { - return Err(PerComputationCandidateError::Unsupported( - "target is not an aggregate", - )); - }; - // A key-only `HAVING` is a row predicate over the input: rows of a - // dropped group never reach the state. - let having = &targets[0].target.having_filters; - // Operator refs in a predicate (a subquery) are this aggregate's - // children, already rewritten. - let rewritten = |pred: &Predicate| { - Predicate( - pred.0 - .map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), - ) - }; - // The metric whose raw data samples, or plain sums of them, are every - // value the aggregate reads: a heap-sketch weight read from them carries - // the open requirement `WeightDomain::RawDataSamplesOfMetric`, which - // plan selection confirms. - let raw_data_samples = raw_data_samples_of_metric(child); - let mut input = cache[&Rc::as_ptr(child)].clone(); - for entry in having { - if entry.placement == HavingPlacement::BelowAggregate { - let keys = reduction.group_keys().map_or(&[][..], |by| by.keys()); - let pred = Predicate(entry.predicate.0.map_columns(&mut |column| keys[column])); - input = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: input, - }))?; - } - } - let parts = measures - .iter() - .enumerate() - .map(|(measure, intent)| { - // `fused`: the candidate is the target's fused computation's, - // which absorbs the target beneath. - let (candidate, raw_data_samples_of, fused) = targets - .iter() - .find(|chosen| chosen.target.measure_index == measure) - .map_or( - (&HowToComputeAggregateMeasure::Exact, None, false), - |chosen| (chosen.candidate, raw_data_samples.as_deref(), chosen.fused), - ); - let filter = filters.get(measure).and_then(Option::as_ref).map(rewritten); - let key = (Rc::as_ptr(target), measure); - if fused { - return match candidate { - HowToComputeAggregateMeasure::ExactAccumulator(_) => nested_sum(target, cache), - HowToComputeAggregateMeasure::Summary(algorithm) => whole_expression( - target, - intent, - algorithm, - raw_data_samples_of, - (sizing, key), - cache, - ), - HowToComputeAggregateMeasure::Exact => { - Err(PerComputationCandidateError::Unsupported( - "plain exact execution of a fused computation", - )) - } - }; - } - substitute_measure( - target, - intent, - candidate, - filter, - &input, - raw_data_samples_of, - (sizing, key), - ) - }) - .collect::, _>>()?; - let mut output = match <[_; 1]>::try_from(parts) { - Ok([part]) => with_names_of(target, part)?, - Err(parts) => join_measures(target, parts)?, - }; - for entry in having { - if entry.placement == HavingPlacement::AboveOutput && entry.filter.is_none() { - output = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: rewritten(&entry.predicate), - child: output, - }))?; - } - } - Ok(output) -} - -/// `output` with `target`'s column names: an estimate or accumulator names -/// its value after the operation (`quantile`, `sum`), not the aggregate's -/// output column. -fn with_names_of( - target: &OperatorNode, - output: Rc, -) -> Result, PerComputationCandidateError> { - let names = |node: &OperatorNode| -> Vec { - node.schema - .fields - .iter() - .map(|field| field.name.clone()) - .collect() - }; - // A top-k's ranked rows may carry their item columns after the - // aggregate's ([`ranked_as_topk`]). - if names(&output).starts_with(&names(target)) { - return Ok(with_aggregate_qualifiers(target, output)); - } - if output.schema.fields.len() != target.schema.fields.len() { - return Err(PerComputationCandidateError::Unsupported( - "substituted output with other columns than the aggregate", - )); - } - let cols = names(target) - .into_iter() - .enumerate() - .map(|(column, name)| ProjectItem { - alias: Some(name), - expr: ScalarExpr::Column(column), - }) - .collect(); - Ok(with_aggregate_qualifiers( - target, - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: output, - }))?, - )) -} - -/// An equivalent aggregate replacement must retain SQL field identity. These -/// explicit qualifiers survive child rebinding; types and keys remain derived -/// from executable operators rather than copied from the requested output. -fn with_aggregate_qualifiers(target: &OperatorNode, output: Rc) -> Rc { - if output - .schema - .fields - .iter() - .zip(&target.schema.fields) - .all(|(field, expected)| field.table == expected.table) - { - return output; - } - let mut output = output.as_ref().clone(); - for (field, expected) in output.schema.fields.iter_mut().zip(&target.schema.fields) { - field.table = expected.table.clone(); - } - Rc::new(output) -} - -/// A whole-expression top-k for `target` ([`whole_expression_input`]): one -/// heap summary of `algorithm` over the absorbed aggregate's rewritten -/// input, partitioned as `target`. -fn whole_expression( - target: &OperatorNode, - intent: &AggIntent, - algorithm: &SketchAlgorithm, - raw_data_samples_of: Option<&str>, - (sizing, key): (&Sizing, MeasureKey), - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let (input, update, partitions) = whole_expression_input(target).ok_or( - PerComputationCandidateError::Unsupported("whole-expression top-k input"), - )?; - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: cache[&Rc::as_ptr(&input)].clone(), - input: with_raw_data_samples_requirement(update, raw_data_samples_of), - family: FieldDataType::Sketch(*algorithm, GroupingStrategy::default()), - reduction: partitions, - grouping: GroupingStrategy::default(), - filter: None, - }))?; - sizing.built.borrow_mut().push((state.clone(), key)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: state, - query: sketch_statistic(intent).ok_or(PerComputationCandidateError::Unsupported( - "sketch evaluation", - ))?, - }))?; - ranked_as_topk(target, estimate) -} - -/// A heap's top-k `estimate` (`[partitions.., items.., topk]`, one row per -/// ranked item) in `target`'s columns: its `[keys.., topk_k]`, with -/// `topk_k` the estimated Float64 weight, then the item columns. This is -/// the exact top-k's physical shape (its ranked rows' value, then the -/// columns naming each ranked item: PromQL labels, SQL group keys), so -/// consumers of `[keys.., topk_k]` find those columns where they were. -fn ranked_as_topk( - target: &OperatorNode, - estimate: Rc, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { reduction, .. }) = target.non_asap() else { - return Err(PerComputationCandidateError::Unsupported( - "top-k target is not an aggregate", - )); - }; - let ranked_index = reduction.group_keys().map_or(0, |keys| keys.len()); - let fields = &estimate.schema.fields; - let estimated = fields.len() - 1; - let cols = target - .schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - let source = if index == ranked_index { - estimated - } else { - fields[..estimated] - .iter() - .position(|input| input.name == field.name) - .ok_or(PerComputationCandidateError::Unsupported( - "top-k estimate without a ranked-row identity column", - ))? - }; - Ok(ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(source), - }) - }) - .collect::, PerComputationCandidateError>>()?; - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: estimate, - }, - ))?) -} - -/// A nested-sum merge for `target` ([`nested_sum_input`]): one exact `Sum` -/// accumulator over the absorbed sum's rewritten input, grouped as -/// `target`, finalized. -fn nested_sum( - target: &OperatorNode, - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let (rows, intent, reduction) = nested_sum_input(target).ok_or( - PerComputationCandidateError::Unsupported("nested-sum merge input"), - )?; - let state = exact_accumulator( - ExactKind::Sum, - &intent, - &cache[&Rc::as_ptr(&rows)], - reduction, - )?; - Ok(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child: state }, - ))?) -} - -/// An unfiltered exact accumulator of `kind` for `intent` over `input`, -/// grouped by `reduction`. -pub(crate) fn exact_accumulator( - kind: ExactKind, - intent: &AggIntent, - input: &Rc, - reduction: Reduction, -) -> Result, PerComputationCandidateError> { - let family = FieldDataType::exact(kind); - let read = measure_input(intent, input, None)?; - Ok(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::SummaryAgg { - input: summary_update(intent, &family, &reduction, &input.schema, read)?, - child: Rc::clone(input), - family, - reduction, - grouping: GroupingStrategy::default(), - filter: None, - }, - ))?) -} - -/// One measure of `target` over `input`: its group keys, then its value. -fn substitute_measure( - target: &OperatorNode, - intent: &AggIntent, - candidate: &HowToComputeAggregateMeasure, - filter: Option, - input: &Rc, - raw_data_samples_of: Option<&str>, - (sizing, key): (&Sizing, MeasureKey), -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - reduction, - output_names, - .. - }) = target.non_asap() - else { - unreachable!("substitute checked the aggregate") - }; - let state = |family: FieldDataType, intent: &AggIntent, read: SummaryInputExpr| { - Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::SummaryAgg { - child: input.clone(), - input: with_raw_data_samples_requirement( - summary_update(intent, &family, reduction, &input.schema, read)?, - raw_data_samples_of, - ), - family, - reduction: reduction.clone(), - grouping: GroupingStrategy::default(), - filter: filter.clone(), - }, - ))?) - }; - let finalized = |kind: &ExactKind, intent: &AggIntent, read| { - let state = state(FieldDataType::exact(*kind), intent, read)?; - Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child: state }, - ))?) - }; - Ok(match candidate { - HowToComputeAggregateMeasure::Exact => { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: input.clone(), - reduction: reduction.clone(), - measures: vec![intent.clone()], - // One name per measure, as the target names this one (an - // empty name is the intent's synthetic one). - output_names: vec![output_names.get(key.1).cloned().unwrap_or_default()], - filters: vec![filter.clone()], - having: None, - }))? - } - HowToComputeAggregateMeasure::ExactAccumulator(kinds) if kinds.len() == 1 => { - let read = measure_input(intent, input, filter.as_ref())?; - finalized(&kinds[0], intent, read)? - } - HowToComputeAggregateMeasure::Summary(algorithm) => { - let family = FieldDataType::Sketch(*algorithm, GroupingStrategy::default()); - let summary_input = state( - family, - intent, - measure_input(intent, input, filter.as_ref())?, - )?; - sizing.built.borrow_mut().push((summary_input.clone(), key)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: sketch_statistic(intent).ok_or( - PerComputationCandidateError::Unsupported("sketch evaluation"), - )?, - }))?; - match intent { - AggIntent::TopK { .. } => ranked_as_topk(target, estimate)?, - _ => estimate, - } - } - HowToComputeAggregateMeasure::ExactAccumulator(kinds) => { - let (AggIntent::Avg { col }, [ExactKind::Sum, ExactKind::Count]) = - (intent, kinds.as_slice()) - else { - return Err(PerComputationCandidateError::Unsupported( - "several exact states other than avg's sum then count", - )); - }; - let sum_intent = AggIntent::Sum { col: *col }; - // The count counts the averaged column, as `avg` does: it is - // non-null (a nullable input prunes this candidate), so it counts - // the rows the sum reads. - let read = measure_input(&sum_intent, input, filter.as_ref())?; - let sum = finalized(&ExactKind::Sum, &sum_intent, read.clone())?; - let count = finalized( - &ExactKind::Count, - &AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }, - read, - )?; - if input.result_kind == OperatorResultKind::Relation { - return sql_avg(reduction, filter.is_some(), sum, count); - } - // `avg` is Float64 whatever the summed type, as `Avg`'s output is, - // and a PromQL binary operator reads Float64 sample values. A - // grouped sum or count ends with its value, cast when not Float64 - // (a count is Int64). A per-series sum keeps the input's labels, - // so its value is the sample-value column; a per-series count is - // Float64. - let (sum, count) = match reduction { - Reduction::Reduce(by) => ( - last_as_float64(by.len(), sum)?, - last_as_float64(by.len(), count)?, - ), - Reduction::PerEntity => { - let value = resolve_column_ref(&ColumnRef::SampleValue, &sum.schema) - .ok() - .and_then(|column| sum.schema.fields.get(column)); - if value.map(|field| &field.dtype) - != Some(&FieldDataType::Plain(DataType::Float64)) - { - return Err(PerComputationCandidateError::Unsupported( - "non-Float64 per-series avg", - )); - } - (sum, count) - } - }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - // A per-series window may hold no finite sum to divide. - checked_finite_division: matches!(reduction, Reduction::PerEntity), - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - vector_match: None, - }, - return_bool: false, - lhs: sum, - rhs: count, - }))? - } - }) -} - -/// `node`, a grouped aggregate's `keys` group keys and then its value, with -/// the value cast to Float64 unless it already is one. -fn last_as_float64( - keys: usize, - node: Rc, -) -> Result, PerComputationCandidateError> { - if node.schema.fields.last().map(|field| &field.dtype) - == Some(&FieldDataType::Plain(DataType::Float64)) - { - return Ok(node); - } - let mut cols = key_items(keys); - cols.push(ProjectItem { - alias: None, - expr: ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(keys)), - to: DataType::Float64, - try_cast: false, - }, - }); - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: node, - }, - ))?) -} - -/// SQL `avg` from its finalized `sum` and `count`: joined on the group keys -/// (NULL keys match), then `sum / count` as Float64. A group with no row -/// passing the measure filter, or an empty global input, counts 0 and its -/// `avg` is NULL. -fn sql_avg( - reduction: &Reduction, - filtered: bool, - sum: Rc, - count: Rc, -) -> Result, PerComputationCandidateError> { - let Reduction::Reduce(by) = reduction else { - return Err(PerComputationCandidateError::Unsupported( - "SQL avg without `by` keys", - )); - }; - if by.is_without() { - return Err(PerComputationCandidateError::Unsupported( - "SQL avg without `by` keys", - )); - } - let keys = by.len(); - let float = |column| { - Box::new(ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(column)), - to: DataType::Float64, - try_cast: false, - }) - }; - let count_column = keys + 1 + keys; - let mut avg = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Div, - left: float(keys), - right: float(count_column), - semantics: ExprSemantics::Sql, - }; - if filtered || keys == 0 { - avg = ScalarExpr::Case { - operand: None, - branches: vec![( - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(count_column)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), - semantics: ExprSemantics::Sql, - }, - ScalarExpr::Literal(ScalarValue::Null), - )], - else_expr: Some(Box::new(avg)), - }; - } - let mut cols = key_items(keys); - cols.push(ProjectItem { - alias: None, - expr: avg, - }); - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: join_on_keys(keys, sum, count)?, - }, - ))?) -} - -/// `left` joined with `right` on their leading `keys` columns; NULL keys -/// match each other, as grouping treats them. -fn join_on_keys( - keys: usize, - left: Rc, - right: Rc, -) -> Result, PerComputationCandidateError> { - let width = left.schema.fields.len(); - let column = |id| Box::new(ScalarExpr::Column(id)); - let matches = (0..keys) - .map(|key| { - let equal = ScalarExpr::Compare { - left: column(key), - op: CompareOpKind::Eq, - right: column(width + key), - semantics: ExprSemantics::Sql, - }; - if right.schema.fields[key].nullable { - ScalarExpr::BoolOr(vec![ - equal, - ScalarExpr::BoolAnd(vec![ - ScalarExpr::IsNull(column(key)), - ScalarExpr::IsNull(column(width + key)), - ]), - ]) - } else { - equal - } - }) - .collect::>(); - let (kind, pred) = match matches.len() { - 0 => ( - JoinKind::Cross, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ), - _ => (JoinKind::Inner, ScalarExpr::BoolAnd(matches)), - }; - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Join { - kind, - pred: Predicate(pred), - left, - right, - }, - ))?) -} - -/// Join single-measure `parts` (each: group keys, then one value) on their -/// group keys and project `target`'s output columns. Every part has the same -/// groups, so an inner join keeps them all; NULL keys match each other, as -/// grouping treats them. -fn join_measures( - target: &OperatorNode, - parts: Vec>, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) = target.non_asap() - else { - return Err(PerComputationCandidateError::Unsupported( - "multi-measure aggregate without `by` keys", - )); - }; - if by.is_without() { - return Err(PerComputationCandidateError::Unsupported( - "multi-measure aggregate without `by` keys", - )); - } - let keys = by.len(); - let width = keys + 1; - let mut parts = parts.into_iter(); - let mut joined = parts.next().expect("an aggregate has measures"); - for part in parts { - joined = join_on_keys(keys, joined, part)?; - } - let mut cols = key_items(keys); - cols.extend( - (0..target.schema.fields.len() - keys).map(|measure| ProjectItem { - alias: None, - expr: ScalarExpr::Column(measure * width + keys), - }), - ); - for (item, field) in cols.iter_mut().zip(&target.schema.fields) { - item.alias = Some(field.name.clone()); - } - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: joined, - }, - ))?) -} - -fn key_items(keys: usize) -> Vec { - (0..keys) - .map(|key| ProjectItem { - alias: None, - expr: ScalarExpr::Column(key), - }) - .collect() -} - -/// What a measure's state reads from each row of `input`, resolvable there. -/// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count -/// included, reads its input column (the sample value when it names none). -fn measure_input( - intent: &AggIntent, - input: &Rc, - filter: Option<&Predicate>, -) -> Result { - let schema = &input.schema; - let read = match intent { - AggIntent::Count { .. } if input.result_kind == OperatorResultKind::Relation => { - SummaryInputExpr::Column(counted_column(schema, filter)?) - } - // A top-k ranks its child's aggregate output, not a column of its own. - AggIntent::TopK { .. } => SummaryInputExpr::Column( - crate::pass1::realization::ranked_value(input) - .map_err(|_| { - PerComputationCandidateError::Unsupported("Top-K has no ranked column") - })? - .1, - ), - _ => crate::pass1::realization::summarised_input(intent, schema).ok_or( - PerComputationCandidateError::Unsupported("input column outside child schema"), - )?, - }; - if let SummaryInputExpr::Column(column) = &read { - resolve_column_ref(column, schema).map_err(|_| { - PerComputationCandidateError::Unsupported("summary input column does not resolve") - })?; - } - Ok(read) -} - -/// The column a SQL `count` state updates on. The runtime skips NULL updates, -/// so it is a column non-null on every counted row: one the measure filter -/// proves non-null (`count(col)` lowers to the filter `col IS NOT NULL`), else -/// a non-nullable one, Float64 first (the runtime's numeric update). -fn counted_column( - schema: &Schema, - filter: Option<&Predicate>, -) -> Result { - fn not_null(expr: &ScalarExpr) -> Vec { - match expr { - ScalarExpr::IsNotNull(inner) => match **inner { - ScalarExpr::Column(column) => vec![column], - _ => vec![], - }, - ScalarExpr::BoolAnd(conjuncts) => conjuncts.iter().flat_map(not_null).collect(), - _ => vec![], - } - } - let non_nullable = |float: bool| { - schema - .fields - .iter() - .enumerate() - .filter_map(move |(id, field)| { - (!field.nullable - && (!float || field.dtype == FieldDataType::Plain(DataType::Float64))) - .then_some(id) - }) - }; - let reference = |id: usize| { - schema - .fields - .get(id) - .map(asap_types::ir::schema::Field::column_ref) - }; - filter - .map_or_else(Vec::new, |filter| not_null(&filter.0)) - .into_iter() - .chain(non_nullable(true)) - .chain(non_nullable(false)) - .find_map(|id| { - reference(id).filter(|column| resolve_column_ref(column, schema).ok() == Some(id)) - }) - .ok_or(PerComputationCandidateError::Unsupported( - "SQL count over rows with no column known to be non-null", - )) -} - -/// What each input row contributes, given what the measure reads -/// ([`measure_input`]), following the legacy realization rules: heap sketches -/// rank series identities, frequency sketches count values, and every other -/// family reads the measure's input. -fn summary_update( - intent: &AggIntent, - family: &FieldDataType, - reduction: &Reduction, - child: &Schema, - weight: SummaryInputExpr, -) -> Result { - let algorithm = match family { - FieldDataType::Sketch(kind, _) => Some(kind), - _ => None, - }; - Ok(match (intent, algorithm) { - (AggIntent::TopK { .. }, Some(_)) => { - if reduction.group_keys().is_some_and(|keys| keys.is_without()) { - return Err(PerComputationCandidateError::Unsupported( - "Top-K partitions given by `without`", - )); - } - let groups: Vec = reduction - .group_keys() - .into_iter() - .flat_map(|keys| keys.iter().copied()) - .collect(); - let item = if child.has_promql_series_identity() { - asap_types::ir::schema::topk::series_identity_item(child, &groups) - } else if child.closed { - // A closed child (an aggregate's output, say) lists every - // column: the item is its columns outside the partition and - // the ranked value, as the legacy current-series and counter - // rules rank them. - let SummaryInputExpr::Column(ranked) = &weight else { - return Err(PerComputationCandidateError::Unsupported( - "Top-K ranked value is not a column", - )); - }; - let value = resolve_column_ref(ranked, child).map_err(|_| { - PerComputationCandidateError::Unsupported("Top-K ranked value does not resolve") - })?; - let items: Vec<_> = child - .fields - .iter() - .enumerate() - .filter(|(index, _)| *index != value && !groups.contains(index)) - .map(|(_, field)| SummaryInputExpr::Column(field.column_ref())) - .collect(); - if items.is_empty() { - return Err(PerComputationCandidateError::Unsupported( - "Top-K item has no label outside its partition", - )); - } - SummaryInputExpr::Tuple(items) - } else { - let excluding = groups - .iter() - .filter_map(|&index| child.fields.get(index)) - .map(asap_types::ir::schema::Field::column_ref) - .collect(); - SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { excluding }) - }; - SummaryUpdate { - item: Some(item), - // The ranked value ([`measure_input`]). - weight, - // Not proven non-negative; selection decides whether CMS is admissible. - weight_domain: WeightDomain::UnknownOrSigned, - } - } - (_, Some(SketchAlgorithm::UnivMon)) - | (AggIntent::Count { .. }, Some(SketchAlgorithm::Cms)) => SummaryUpdate { - item: Some(weight), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - _ => SummaryUpdate { - item: None, - weight, - weight_domain: WeightDomain::UnknownOrSigned, - }, - }) -} - #[cfg(test)] mod tests { /// Aggregate replacement qualifiers survive both renaming and later input rebinding. @@ -2174,28 +901,33 @@ mod tests { } use super::*; + use crate::pass1::composition::*; + use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind}; + use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; + use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; + use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; + use asap_types::ir::schema::{EntityIdentity, NonNegativeWeightProof, WeightDomain}; + use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, Operator, ProjectItem, ScalarExpr, + }; + + use asap_accuracy::propagation::raw_data_samples_of_metric; + use asap_accuracy::composition::{StreamNorm, PRODUCT_OPERAND_DOMAINS}; use asap_accuracy::propagation::operand_guarantee; use asap_accuracy::test_support::{ - composed_guarantee, estimate_bound_over_raw_data_samples, estimate_has_bound, - estimates_raw_data_samples_of, fold_stats, local_guarantee, + composed_guarantee, estimate_has_bound, estimates_raw_data_samples_of, fold_stats, raw_data_samples_sign_statistic, scalar_guarantee, sign_statistic, source_distinct_guarantee, stream_norm, QUOTIENT_OPERAND_DOMAINS, }; - use asap_types::ir::properties::guarantee::{ - AccuracyGuarantee, AccuracyMetric, BoundExpr, ProbabilityExpr, - }; - use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::properties::guarantee::{AccuracyMetric, BoundExpr, ProbabilityExpr}; + use asap_types::ir::schema::SketchStatistic; /// Approximate count keeps one exact option, its accumulator, beside the /// universal summary; plain exact execution is not offered. #[test] fn approximate_count_keeps_one_exact_accumulator_and_universal_choices() { - let choices = computation_candidates_for_intent(&AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.05).failure_probability(0.01), - }) - .unwrap() - .candidates; + let choices = computation_candidates_for_intent(&AggIntent::Count).candidates; assert!(!choices.contains(&HowToComputeAggregateMeasure::Exact)); assert_eq!( choices[0], @@ -2211,23 +943,18 @@ mod tests { /// answers the total. #[test] fn a_count_offers_no_count_sketch() { - let choices = computation_candidates_for_intent(&AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.05).failure_probability(0.01), - }) - .unwrap() - .candidates; + let choices = computation_candidates_for_intent(&AggIntent::Count).candidates; assert!(!choices.contains(&HowToComputeAggregateMeasure::Summary( SketchAlgorithm::CountSketch ))); - let total = sketch_statistic(&AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }) - .unwrap(); + let total = sketch_statistic(&AggIntent::Count).unwrap(); for choice in &choices { if let HowToComputeAggregateMeasure::Summary(algorithm) = choice { assert!( - asap_types::ir::schema::FamilyCapability::of_per_instance_sketch(algorithm) - .answers(&total), + asap_types::ir::schema::SummaryTypeCapability::of(&FieldDataType::Sketch( + *algorithm + )) + .answers(&total), "{algorithm:?}" ); } @@ -2898,7 +1625,12 @@ mod tests { let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) .unwrap_or(root); generate_per_computation_candidates( - &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &crate::stage0::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap() } @@ -2928,7 +1660,7 @@ mod tests { .into_iter() .filter(|node| { matches!(node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), .. + summary_type: FieldDataType::Sketch(kind), .. }) if kind.capability().answers(StatisticKind::TopK)) }) .collect() @@ -3014,7 +1746,12 @@ mod tests { AccuracyTarget::max_error(0.1), ); let inventory = generate_per_computation_candidates( - &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &crate::stage0::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let topk = topk_target(&inventory); @@ -3321,7 +2058,7 @@ mod tests { }; let Some(ASAPOp::SummaryAgg { child, - family, + summary_type: family, reduction, .. }) = state.asap() @@ -3552,10 +2289,9 @@ mod tests { }; let Some(ASAPOp::SummaryAgg { child: range, - family, + summary_type: family, input, reduction, - grouping, filter, }) = summary_input.asap() else { @@ -3564,10 +2300,9 @@ mod tests { let chain = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: Rc::clone(range.children()[0]), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: filter.clone(), tumbling_window_width_ms: 10_000, })) @@ -3662,7 +2397,7 @@ mod tests { /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. #[test] fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let family = FieldDataType::Sketch(SketchAlgorithm::Cms, GroupingStrategy::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::Cms); let query = SketchStatistic::PointCount { key: ColumnRef::SampleValue, value: None, @@ -3691,38 +2426,6 @@ mod tests { )); } - /// A Count-Min estimate over samples of `m` gets its bound gated on - /// `m`'s sign: unknown until plan selection binds it, the local bound - /// once bound to 1. - #[test] - fn count_min_estimate_over_raw_data_samples_is_gated_on_the_metric() { - let family = FieldDataType::Sketch(SketchAlgorithm::Cms, GroupingStrategy::default()); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - let update = SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }, - }; - let gated = estimate_bound_over_raw_data_samples(&family, &update, &query).expect("gated"); - assert_eq!(gated.spec.error.evaluate(), None); - let local = local_guarantee(&family, &query).unwrap(); - let statistic = raw_data_samples_sign_statistic("m"); - // Every other statistic (a stream norm) bound alike. - let others = |s: &str| Some(if s == statistic { 1.0 } else { 1_000.0 }); - let bound = gated.bind(&others).spec.error.evaluate(); - assert!(bound.is_some()); - assert_eq!(bound, local.bind(&others).spec.error.evaluate()); - let unconfirmed = gated - .bind(&|s| (s != statistic).then_some(1_000.0)) - .spec - .error - .evaluate(); - assert_eq!(unconfirmed, None); - } - /// Classic HLL's failure probability is left symbolic in a source's /// distinct bound only for one instance per group over that source's /// own rows (#693's scope condition): not over a join of two sources, @@ -3730,13 +2433,12 @@ mod tests { /// every group, whose distinct items the source's bound does not bound. #[test] fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { - use asap_types::ir::schema::HydraKind; - let hll = |grouping| FieldDataType::Sketch(SketchAlgorithm::Hll, grouping); + let hll = FieldDataType::Sketch; let input = |query| crate::test_support::lower_promql(query, AccuracyTarget::exact()); let guarantee = |family: &FieldDataType, query| { source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) }; - let own = hll(GroupingStrategy::PerSubpopulationInstance); + let own = hll(SketchAlgorithm::Hll); let symbolic = guarantee(&own, "a").expect("a source's own rows"); assert_eq!( symbolic.spec.failure_probability, @@ -3747,9 +2449,7 @@ mod tests { for (query, over) in [("a * on() b", "a join"), ("a * 2", "computed rows")] { assert!(guarantee(&own, query).is_none(), "{over}: {query}"); } - let hydra = hll(GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - }); + let hydra = hll(SketchAlgorithm::HydraHll); assert!(guarantee(&hydra, "a").is_none()); } @@ -3784,7 +2484,12 @@ mod tests { fn inventory_at(query: &str, accuracy: AccuracyTarget) -> PerComputationCandidates { let root = crate::test_support::lower_promql(query, accuracy); generate_per_computation_candidates( - &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &crate::stage0::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap() } diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs index bcc2a625d..3ab7403e6 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs @@ -9,8 +9,7 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{ - DataType, ExactKind, Field, FieldDataType, GroupingStrategy, Schema, SketchAlgorithm, - SummaryUpdate, + DataType, ExactKind, Field, FieldDataType, Schema, SketchAlgorithm, SummaryUpdate, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, @@ -19,13 +18,8 @@ use asap_types::ir::{ use asap_types::stages::{HavingEntry, HavingPlacement, HowToComputeAggregateMeasure}; use std::rc::Rc; -fn approximate() -> AccuracyTarget { - AccuracyTarget::max_error(0.05).failure_probability(0.01) -} fn candidates_for(intent: AggIntent) -> Vec { - computation_candidates_for_intent(&intent) - .unwrap() - .candidates + computation_candidates_for_intent(&intent).candidates } fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec { choices @@ -78,10 +72,7 @@ fn always() -> Predicate { /// Example 2 preserves specialized distinct summaries and the universal alternative. #[test] fn cardinality_keeps_exact_specialized_and_universal_candidates() { - let choices = candidates_for(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }); + let choices = candidates_for(AggIntent::Cardinality { cols: vec![0] }); assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); assert_eq!( algorithms(&choices), @@ -91,10 +82,7 @@ fn cardinality_keeps_exact_specialized_and_universal_candidates() { SketchAlgorithm::UnivMon ] ); - let tuple = candidates_for(AggIntent::Cardinality { - cols: vec![0, 1], - accuracy: approximate(), - }); + let tuple = candidates_for(AggIntent::Cardinality { cols: vec![0, 1] }); assert!(!algorithms(&tuple).contains(&SketchAlgorithm::UnivMon)); } @@ -102,14 +90,8 @@ fn cardinality_keeps_exact_specialized_and_universal_candidates() { #[test] fn frequency_statistics_keep_universal_choices() { for intent in [ - AggIntent::FrequencyL2 { - col: Some(0), - accuracy: approximate(), - }, - AggIntent::FrequencyEntropy { - col: Some(0), - accuracy: approximate(), - }, + AggIntent::FrequencyL2 { col: Some(0) }, + AggIntent::FrequencyEntropy { col: Some(0) }, ] { let choices = candidates_for(intent); assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); @@ -120,16 +102,15 @@ fn frequency_statistics_keep_universal_choices() { /// Scalar roots expose their producer targets; repeated references retain one target identity. #[test] fn scalar_root_producers_are_discovered_once() { - let producer = aggregate(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }); + let producer = aggregate(AggIntent::Cardinality { cols: vec![0] }); let roots = vec![ QueryRoot::Scalar(ScalarExpr::ScalarSubquery(producer.clone())), QueryRoot::Operator(producer.clone()), ]; - let candidates = - generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); + let candidates = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap(); assert_eq!(candidates.roots.len(), 2); for root in &candidates.roots { root.validate_structure().unwrap(); @@ -176,8 +157,10 @@ fn exact_promql(query: &str) -> Vec { #[test] fn example_1_exact_q1_keeps_exact_accumulators_only() { let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); - let candidates = - generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); + let candidates = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap(); let found: Vec<_> = candidates .targets .iter() @@ -196,86 +179,35 @@ fn example_1_exact_q1_keeps_exact_accumulators_only() { ); } -/// ASAP operators and invalid probability requirements fail at the stage +/// ASAP operators fail at the stage /// boundary. #[test] -fn asap_input_and_invalid_accuracy_are_rejected() { +fn asap_input_is_rejected() { let summary = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(false), - family: FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(); assert!(matches!( generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(summary)]).unwrap() + &memo_of_queries( + &[QueryRoot::Operator(summary)], + &[None], + &Default::default() + ) + .unwrap() ), Err(PerComputationCandidateError::ContainsAsap) )); - let epsilon_delta = - |epsilon, delta| AccuracyTarget::max_error(epsilon).failure_probability(delta); - for target in [ - AccuracyTarget::max_error(f64::NAN), - epsilon_delta(0.0, 0.01), - epsilon_delta(-0.1, 0.01), - epsilon_delta(1.0, 0.01), - epsilon_delta(f64::INFINITY, 0.01), - epsilon_delta(0.1, 0.0), - epsilon_delta(0.1, 1.0), - ] { - assert!( - matches!( - computation_candidates_for_intent(&AggIntent::Count { - accuracy: target.clone() - }), - Err(PerComputationCandidateError::InvalidAccuracy { - root: None, - measure: None - }) - ), - "{target:?}" - ); - } - let invalid = aggregate_over( - scan(false), - vec![ - AggIntent::Count { - accuracy: approximate(), - }, - AggIntent::Count { - accuracy: epsilon_delta(1.5, 0.01), - }, - ], - vec![], - None, - ); - let error = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(invalid)]).unwrap(), - ) - .unwrap_err(); - assert!( - matches!( - &error, - PerComputationCandidateError::InvalidAccuracy { - root: Some(root), - measure: Some(1) - } if root == "0" - ), - "{error:?}" - ); - assert!(error.to_string().contains("root 0, measure 1 "), "{error}"); } /// TopK keeps both declared heap substrates without choosing an implementation. #[test] fn topk_keeps_both_specialized_heap_choices() { - let choices = candidates_for(AggIntent::TopK { - k: 10, - accuracy: approximate(), - }); + let choices = candidates_for(AggIntent::TopK { k: 10 }); assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); assert_eq!( algorithms(&choices), @@ -289,12 +221,8 @@ fn topk_keeps_both_specialized_heap_choices() { /// Summary candidates are unsized, so they do not depend on the requested error. #[test] fn summary_candidates_are_unsized() { - let loose = candidates_for(AggIntent::Count { - accuracy: approximate(), - }); - let tight = candidates_for(AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.0001).failure_probability(0.0001), - }); + let loose = candidates_for(AggIntent::Count); + let tight = candidates_for(AggIntent::Count); assert_eq!(loose, tight); } @@ -304,17 +232,12 @@ fn summary_candidates_are_unsized() { fn multi_measure_aggregate_has_one_target_per_measure() { let node = aggregate_over( scan(false), - vec![ - AggIntent::Sum { col: Some(1) }, - AggIntent::Count { - accuracy: approximate(), - }, - ], + vec![AggIntent::Sum { col: Some(1) }, AggIntent::Count], vec![], None, ); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(node)]).unwrap(), + &memo_of_queries(&[QueryRoot::Operator(node)], &[None], &Default::default()).unwrap(), ) .unwrap(); let candidates = &inventory.targets; @@ -342,7 +265,7 @@ fn aggregate_without_realization_is_not_a_target() { population: true, }); assert!(generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(node)]).unwrap() + &memo_of_queries(&[QueryRoot::Operator(node)], &[None], &Default::default()).unwrap() ) .unwrap() .targets @@ -353,9 +276,7 @@ fn aggregate_without_realization_is_not_a_target() { /// filter, which composition applies to the state. #[test] fn filtered_measure_keeps_candidates_and_records_its_filter() { - let count = || AggIntent::Count { - accuracy: approximate(), - }; + let count = || AggIntent::Count; let filtered = aggregate_over( scan(false), vec![count(), count()], @@ -363,7 +284,12 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { None, ); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(filtered)]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(filtered)], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let targets = &inventory.targets; @@ -397,7 +323,6 @@ fn quantile() -> AggIntent { AggIntent::Quantile { col: Some(1), q: 0.5, - accuracy: approximate(), } } @@ -418,9 +343,7 @@ fn grouped_count(having: Option) -> Rc { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { child: scan(false), reduction: Reduction::by(vec![0]), - measures: vec![AggIntent::Count { - accuracy: approximate(), - }], + measures: vec![AggIntent::Count], output_names: vec![], filters: vec![], having, @@ -448,7 +371,12 @@ fn key_only_having_is_pushed_below_and_keeps_candidates() { let node = grouped_count(None); let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(root.clone())]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(root.clone())], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let targets = &inventory.targets; @@ -471,10 +399,14 @@ fn key_only_having_is_pushed_below_and_keeps_candidates() { fn value_having_stays_above_output_and_keeps_candidates() { let value = || equals(1, ScalarValue::Int64(10)); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[ - QueryRoot::Operator(having_filter(grouped_count(None), value())), - QueryRoot::Operator(grouped_count(Some(value()))), - ]) + &memo_of_queries( + &[ + QueryRoot::Operator(having_filter(grouped_count(None), value())), + QueryRoot::Operator(grouped_count(Some(value()))), + ], + &vec![None; 2], + &Default::default(), + ) .unwrap(), ) .unwrap(); @@ -499,10 +431,14 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { let key = equals(0, ScalarValue::Utf8("a".into())); let filtered = having_filter(node.clone(), key.clone()); let targets = generate_per_computation_candidates( - &memo_of_queries(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(node.clone()), - ]) + &memo_of_queries( + &[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(node.clone()), + ], + &vec![None; 2], + &Default::default(), + ) .unwrap(), ) .unwrap() @@ -521,7 +457,12 @@ fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { let first = having_filter(node.clone(), key.clone()); let second = having_filter(node, key); let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(first), QueryRoot::Operator(second)]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(first), QueryRoot::Operator(second)], + &[None, None], + &Default::default(), + ) + .unwrap(), ) .unwrap() .targets; @@ -549,10 +490,14 @@ fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { }), ); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(exists), - ]) + &memo_of_queries( + &[ + QueryRoot::Operator(filtered.clone()), + QueryRoot::Operator(exists), + ], + &vec![None; 2], + &Default::default(), + ) .unwrap(), ) .unwrap(); @@ -580,7 +525,7 @@ fn only_the_adjacent_of_stacked_filters_is_having() { let inner = having_filter(node.clone(), inner_key.clone()); let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(outer)]).unwrap(), + &memo_of_queries(&[QueryRoot::Operator(outer)], &[None], &Default::default()).unwrap(), ) .unwrap() .targets; @@ -599,20 +544,20 @@ fn only_the_adjacent_of_stacked_filters_is_having() { /// constant one: the one output row exists with no input rows. #[test] fn global_aggregate_having_stays_above() { - let count = || { - vec![AggIntent::Count { - accuracy: approximate(), - }] - }; + let count = || vec![AggIntent::Count]; let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); let targets = generate_per_computation_candidates( - &memo_of_queries(&[ - QueryRoot::Operator(having_filter( - aggregate_over(scan(false), count(), vec![], None), - never.clone(), - )), - QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), - ]) + &memo_of_queries( + &[ + QueryRoot::Operator(having_filter( + aggregate_over(scan(false), count(), vec![], None), + never.clone(), + )), + QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), + ], + &vec![None; 2], + &Default::default(), + ) .unwrap(), ) .unwrap() @@ -658,15 +603,11 @@ fn above_scalar(column: usize, subquery: Rc) -> Predicate { /// aggregate's `HAVING`; one that also filters the aggregate gives one entry. #[test] fn scalar_reference_filter_is_not_having() { - let count = || { - vec![AggIntent::Count { - accuracy: approximate(), - }] - }; + let count = || vec![AggIntent::Count]; let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(scan(false), above_scalar(1, agg.clone())); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()).unwrap(), ) .unwrap(); let targets = &inventory.targets; @@ -681,7 +622,12 @@ fn scalar_reference_filter_is_not_having() { let agg = aggregate_over(scan(false), count(), vec![], None); let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(root.clone())]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(root.clone())], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap() .targets; @@ -700,9 +646,11 @@ fn scalar_reference_filter_is_not_having() { #[test] fn promql_scalar_operand_is_not_having() { let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); - let targets = generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()) - .unwrap() - .targets; + let targets = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap() + .targets; assert!(!targets.is_empty()); for target in &targets { assert!( @@ -713,12 +661,11 @@ fn promql_scalar_operand_is_not_having() { } let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); - let inventory = generate_per_computation_candidates(&memo_of_queries(&roots).unwrap()).unwrap(); - assert!(having_of(&inventory, |intent| matches!( - intent, - AggIntent::Count { .. } - )) - .is_empty()); + let inventory = generate_per_computation_candidates( + &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), + ) + .unwrap(); + assert!(having_of(&inventory, |intent| matches!(intent, AggIntent::Count)).is_empty()); let sum = having_of(&inventory, |intent| matches!(intent, AggIntent::Sum { .. })); assert_eq!(sum.len(), 1, "{sum:?}"); assert_eq!(sum[0].placement, HavingPlacement::AboveOutput); @@ -740,7 +687,12 @@ fn having_shared_with_a_project_consumer_stays_above() { })) .unwrap(); let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(filtered), QueryRoot::Operator(project)]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(filtered), QueryRoot::Operator(project)], + &[None, None], + &Default::default(), + ) + .unwrap(), ) .unwrap() .targets; @@ -766,9 +718,7 @@ fn without_and_per_entity_having_stays_above() { let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { child: scan(false), reduction: reduction.clone(), - measures: vec![AggIntent::Count { - accuracy: approximate(), - }], + measures: vec![AggIntent::Count], output_names: vec![], filters: vec![], having: None, @@ -776,7 +726,7 @@ fn without_and_per_entity_having_stays_above() { .unwrap(); let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()).unwrap(), ) .unwrap() .targets; @@ -799,13 +749,15 @@ fn without_and_per_entity_having_stays_above() { /// with an exact guarantee. #[test] fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { - use crate::pass1::per_computation_candidates::compose_logical_candidate; - let producer = aggregate(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }); + use crate::pass1::composition::compose_logical_candidate; + let producer = aggregate(AggIntent::Cardinality { cols: vec![0] }); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(producer.clone())]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(producer.clone())], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); let exact = compose_logical_candidate(&inventory, &[0]).unwrap(); @@ -830,13 +782,16 @@ fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { /// The one root of the workload candidate composed from `roots` and `choice`. fn composed(roots: Vec>, choice: &[usize]) -> Vec> { - use crate::pass1::per_computation_candidates::compose_logical_candidate; + use crate::pass1::composition::compose_logical_candidate; let inventory = generate_per_computation_candidates( &memo_of_queries( &roots - .into_iter() + .iter() + .cloned() .map(QueryRoot::Operator) .collect::>(), + &vec![None; roots.len()], + &Default::default(), ) .unwrap(), ) @@ -959,7 +914,7 @@ fn composed_avg_divides_sum_and_count_states() { for (side, kind) in [(left, ExactKind::Sum), (right, ExactKind::Count)] { assert!(matches!( summary_agg(side), - ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(found, _), .. } if *found == kind + ASAPOp::SummaryAgg { summary_type: FieldDataType::ExactAggregate(found, _), .. } if *found == kind )); } } @@ -1011,11 +966,7 @@ fn composed_multi_measure_aggregate_joins_its_measures() { /// executor finds one name per measure. #[test] fn composed_mixed_exact_and_sketch_measures_keep_output_names() { - let quantile = |q| AggIntent::Quantile { - col: Some(1), - q, - accuracy: approximate(), - }; + let quantile = |q| AggIntent::Quantile { col: Some(1), q }; let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { child: scan(false), reduction: Reduction::by(vec![]), @@ -1054,10 +1005,7 @@ fn composed_guarantees() { let value = equals(1, ScalarValue::Int64(10)); let roots = composed( vec![ - aggregate(AggIntent::Cardinality { - cols: vec![0], - accuracy: approximate(), - }), + aggregate(AggIntent::Cardinality { cols: vec![0] }), aggregate(AggIntent::Sum { col: Some(1) }), having_filter(grouped_quantile(None), value), ], @@ -1078,11 +1026,16 @@ fn composed_guarantees() { /// a subquery in it reads the subquery's substitution. #[test] fn own_having_subquery_is_substituted() { - use crate::pass1::per_computation_candidates::compose_logical_candidate; + use crate::pass1::composition::compose_logical_candidate; let inner = aggregate(quantile()); let outer = grouped_quantile(Some(above_scalar(1, inner.clone()))); let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(outer.clone())]).unwrap(), + &memo_of_queries( + &[QueryRoot::Operator(outer.clone())], + &[None], + &Default::default(), + ) + .unwrap(), ) .unwrap(); assert_eq!(inventory.targets.len(), 2); @@ -1136,7 +1089,9 @@ fn per_series_rate_keeps_labels_and_retypes_its_state() { let [out] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); let rate = FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate); let ASAPOp::SummaryAgg { - family, reduction, .. + summary_type: family, + reduction, + .. } = summary_agg(&out) else { unreachable!() @@ -1192,7 +1147,7 @@ fn memo_cached_rules_preserve_occurrence_specific_having() { QueryRoot::Operator(shared), QueryRoot::Operator(having_filter(other, key)), ]; - let memo = memo_of_queries(&roots).unwrap(); + let memo = memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(); let cached = generate_per_computation_candidates(&memo).unwrap(); assert_eq!(cached.targets.len(), 1); // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/logical-asap-optimization/src/pass1/realization.rs index f510bdf65..9e60623ec 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/logical-asap-optimization/src/pass1/realization.rs @@ -5,13 +5,11 @@ use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::Reduction; -use asap_types::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; +use asap_types::ir::scalar::column_resolution::resolve_column_ref; use asap_types::ir::scalar::expr_ir::ColumnRef; -use asap_types::ir::schema::topk::ranked_column; -use asap_types::ir::schema::ColumnId; use asap_types::ir::schema::{ - EntityIdentity, FamilyCapability, Field, FieldDataType, NonNegativeWeightProof, Schema, - SketchAlgorithm, SummaryInputExpr, SummaryUpdate, WeightDomain, + EntityIdentity, Field, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, + SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, }; use asap_types::ir::{NonASAPOp, OperatorNode}; #[cfg(test)] @@ -35,19 +33,21 @@ const FAMILY_PREFERENCE: &[SketchAlgorithm] = &[ SketchAlgorithm::UnivMon, ]; -/// The sketch kinds that can serve an intent, most-preferred first: the -/// families whose family-capability table row answers the intent's -/// statistic, in preference order, less UnivMon for a distinct-tuple +/// The sketch algorithms that can serve an intent, most-preferred first: +/// those whose capability table row answers the intent's statistic, in +/// preference order, less UnivMon for a distinct-tuple /// count. Empty for an intent no sketch /// statistic serves. pub(crate) fn summary_candidates(intent: &AggIntent) -> Vec { - candidates_by(intent, FamilyCapability::of_per_instance_sketch) + candidates_by(intent, |algorithm| { + SummaryTypeCapability::of(&FieldDataType::Sketch(*algorithm)) + }) } -/// [`summary_candidates`] with each family's table row read from `capability`. +/// [`summary_candidates`] with each algorithm's table row read from `capability`. fn candidates_by( intent: &AggIntent, - capability: impl Fn(&SketchAlgorithm) -> FamilyCapability, + capability: impl Fn(&SketchAlgorithm) -> SummaryTypeCapability, ) -> Vec { let Some(statistic) = sketch_statistic(intent) else { return Vec::new(); @@ -66,62 +66,6 @@ fn candidates_by( .collect() } -/// The column fed into a *single-column* summary: the intent's leading -/// positional input resolved to a name against the child schema, or the PromQL -/// sample value when it reads none. Callers are responsible for only reaching -/// here with a one-column intent — [`summarised_input`] is the general form. -pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { - match intent - .input_cols() - .first() - .and_then(|id| child_schema.fields.get(*id)) - { - Some(c) => c.column_ref(), - None => ColumnRef::SampleValue, - } -} - -/// [`ranked_column`] as a reference: the sample value when it is that -/// column, else the column by name. -pub(crate) fn ranked_value( - child: &Rc, -) -> Result<(ColumnId, ColumnRef), ResolveError> { - let ranked = ranked_column(child)?; - let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { - Ok(value) if value == ranked => ColumnRef::SampleValue, - _ => child.schema.fields[ranked].column_ref(), - }; - Ok((ranked, column)) -} - -/// What the summary consumes per input row. An intent that reads one column (or -/// none) feeds that column; `COUNT(DISTINCT a, b)` feeds the whole tuple as one -/// item, so the distinct-count sketch hashes `(a, b)` rather than `a` — the -/// difference between tuple cardinality and single-column cardinality. -/// -/// A tuple leg outside the child schema is an error rather than -/// [`summarised_column`]'s sample-value fallback: a leg has no sample-value -/// reading, and silently dropping one would under-count: `None` then. -pub(crate) fn summarised_input( - intent: &AggIntent, - child_schema: &Schema, -) -> Option { - let cols = intent.input_cols(); - if cols.len() < 2 { - return Some(SummaryInputExpr::Column(summarised_column( - intent, - child_schema, - ))); - } - let legs = cols - .iter() - .map(|id| child_schema.fields.get(*id).map(Field::column_ref)) - .collect::>>()?; - Some(SummaryInputExpr::Tuple( - legs.into_iter().map(SummaryInputExpr::Column).collect(), - )) -} - /// The physical input consumed by one summary realization. Most summaries /// consume the logical aggregate's immediate child and summarize its declared /// input value. Composite realizations can instead consume a larger @@ -145,7 +89,7 @@ pub(crate) fn realize_keyed_additive_summary_input( if !matches!(intent, AggIntent::TopK { .. }) { return None; } - let FieldDataType::Sketch(kind, _) = family else { + let FieldDataType::Sketch(kind) = family else { return None; }; let heap_algorithm = kind; @@ -199,8 +143,8 @@ pub(crate) fn realize_keyed_additive_summary_input( let samples = summed_per_series_samples(child); // A raw data sample's count or a count's row weighs one. let counted = match &samples { - Some((_, inner)) => matches!(inner, AggIntent::Count { .. }), - None => matches!(measures.as_slice(), [AggIntent::Count { .. }]), + Some((_, inner)) => matches!(inner, AggIntent::Count), + None => matches!(measures.as_slice(), [AggIntent::Count]), }; let weight = match measures.as_slice() { _ if counted => SummaryInputExpr::Constant(1.0), @@ -321,8 +265,7 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc else { return None; }; - let [inner @ (AggIntent::Sum { col: None } | AggIntent::Count { .. })] = inner.as_slice() - else { + let [inner @ (AggIntent::Sum { col: None } | AggIntent::Count)] = inner.as_slice() else { return None; }; if by.is_without() @@ -352,10 +295,9 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc #[cfg(test)] mod tests { use super::*; + use crate::pass1::composition::summarised_input; use crate::test_support::metric_scan; - use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::schema::GroupingStrategy; use asap_types::ir::{ExprSemantics, Predicate}; /// A heap sketch consumes every raw row, so the keyed input rule refuses @@ -365,9 +307,7 @@ mod tests { let count_by_job = |filter: Option| { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::exact(), - }], + measures: vec![AggIntent::Count], output_names: vec![], filters: vec![filter], having: None, @@ -375,12 +315,8 @@ mod tests { })) .unwrap() }; - let intent = AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::max_error(0.01), - }; - let family = - FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap, GroupingStrategy::default()); + let intent = AggIntent::TopK { k: 5 }; + let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); let output = Reduction::Reduce(asap_types::ir::operator::operator_properties::GroupKeys::none()); assert!(realize_keyed_additive_summary_input( @@ -422,10 +358,7 @@ mod tests { ), ]; for (cols, want) in cases { - let intent = AggIntent::Cardinality { - cols: cols.clone(), - accuracy: AccuracyTarget::max_error(0.01), - }; + let intent = AggIntent::Cardinality { cols: cols.clone() }; assert_eq!( summarised_input(&intent, &scan.schema), Some(want), @@ -440,50 +373,22 @@ mod tests { #[test] fn summary_candidates_keep_the_hand_written_lists() { use SketchAlgorithm as S; - let accuracy = AccuracyTarget::max_error(0.01); - let cardinality = |cols: Vec| AggIntent::Cardinality { - cols, - accuracy: accuracy.clone(), - }; + let cardinality = |cols: Vec| AggIntent::Cardinality { cols }; let cases = [ ( - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: accuracy.clone(), - }, + AggIntent::Quantile { col: None, q: 0.99 }, vec![S::Kll, S::DDSketch], ), (cardinality(vec![]), vec![S::Hll, S::Kmv, S::UnivMon]), (cardinality(vec![1]), vec![S::Hll, S::Kmv, S::UnivMon]), (cardinality(vec![1, 2]), vec![S::Hll, S::Kmv]), + (AggIntent::FrequencyL2 { col: None }, vec![S::UnivMon]), + (AggIntent::FrequencyEntropy { col: None }, vec![S::UnivMon]), ( - AggIntent::FrequencyL2 { - col: None, - accuracy: accuracy.clone(), - }, - vec![S::UnivMon], - ), - ( - AggIntent::FrequencyEntropy { - col: None, - accuracy: accuracy.clone(), - }, - vec![S::UnivMon], - ), - ( - AggIntent::TopK { - k: 10, - accuracy: accuracy.clone(), - }, + AggIntent::TopK { k: 10 }, vec![S::CmsWithHeap, S::CountSketchWithHeap], ), - ( - AggIntent::Count { - accuracy: accuracy.clone(), - }, - vec![S::Cms, S::UnivMon], - ), + (AggIntent::Count, vec![S::Cms, S::UnivMon]), (AggIntent::Sum { col: None }, vec![]), ]; for (intent, want) in cases { @@ -497,15 +402,12 @@ mod tests { fn a_dropped_readout_drops_the_family() { // Count-Min-with-heap answered from Count-Min's row: no top-k readout. let heap_without_top_k = |family: &SketchAlgorithm| { - FamilyCapability::of_per_instance_sketch(match family { - SketchAlgorithm::CmsWithHeap => &SketchAlgorithm::Cms, - other => other, - }) - }; - let top_k = AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::max_error(0.01), + SummaryTypeCapability::of(&FieldDataType::Sketch(match family { + SketchAlgorithm::CmsWithHeap => SketchAlgorithm::Cms, + other => *other, + })) }; + let top_k = AggIntent::TopK { k: 10 }; assert_eq!( candidates_by(&top_k, heap_without_top_k), vec![SketchAlgorithm::CountSketchWithHeap] diff --git a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs index 02fa8f6c5..3c90e7809 100644 --- a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs @@ -6,17 +6,21 @@ use crate::pass2::memo::rules::{ finer_aggregate, rollup, scan_labels, summary_inputs, topk, Pass2RuleError, }; use crate::pass2::memo::window_candidates::add_local_window_composition; +use asap_accuracy::AccuracyEvidence; use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; use asap_types::workload::QueryWorkloadEntry; /// Stage 1, Pass 2: rule 1 ([`pass2_local_window_composition`]), then rules /// 2–7 and the logical edges ([`pass2_after_local_window_composition`]). -/// `entries[i]` is query `i`'s workload entry. +/// `entries[i]` is query `i`'s workload entry; `evidence` is the +/// deployment's measured accuracy, which each edge's $\alpha^{\log}$ reads +/// before the theoretical guarantee. pub fn generate_logical_asap_memo_candidates( pass1: Memo, entries: &[QueryWorkloadEntry], + evidence: &AccuracyEvidence, ) -> Result { - pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?) + pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?, evidence) } /// Pass 2's rule 1 alone: Pass 1's memo with the tumbling-window layouts @@ -32,9 +36,12 @@ pub fn pass2_local_window_composition( } /// The rest of Pass 2 on [`pass2_local_window_composition`]'s memo: rules -/// 2–7, then the logical edges with their pruning caps. +/// 2–7, then the readouts' required accuracy and the logical edges, with +/// $\alpha^{\log}$ from `evidence` or the theoretical guarantee +/// (`asap_accuracy::producer_satisfies_logical_requirements`). pub fn pass2_after_local_window_composition( mut memo: Memo, + evidence: &AccuracyEvidence, ) -> Result { // 2. Scan widening. scan_labels::add_scan_label_candidates(&mut memo)?; @@ -48,6 +55,6 @@ pub fn pass2_after_local_window_composition( rollup::add_rollup_candidates(&mut memo)?; // 7. Window rollup. rollup::add_window_rollup_candidates(&mut memo)?; - let _ = memo; - todo!("#1207 PR 4: the logical edges, each input's floor tightened by its pruning cap") + let _ = (memo, evidence); + todo!("#1207 PR 4: readout requirements, dead-candidate elimination and the logical edges") } diff --git a/crates/logical-asap-optimization/src/stage0.rs b/crates/logical-asap-optimization/src/stage0.rs index 5764cfeaf..4ab97f678 100644 --- a/crates/logical-asap-optimization/src/stage0.rs +++ b/crates/logical-asap-optimization/src/stage0.rs @@ -3,14 +3,23 @@ //! its expression, found or added by [`Memo::find_or_add_group`], so //! identical sub-expressions are one group. Stage 1 starts from these groups. +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{QueryRoot, SchemaDerivationError}; use asap_types::stages::Memo; +use asap_types::workload::DataWorkload; /// Stage 0: the memo groups of the workload's query roots, query `i` -/// being `roots[i]`, every group without candidates. Fails with the first +/// being `roots[i]` with accuracy target `accuracy_targets[i]` (`None`: +/// none), every group without candidates. Each query is added with its +/// $n_q$, the number of values one evaluation returns, which `data` bounds; +/// a `Scan`'s column signs are `data`'s declared ones. Fails with the first /// node that breaks the node rules over its children's groups /// ([`Memo::find_or_add_group`]). -pub fn memo_of_queries(roots: &[QueryRoot]) -> Result { - let _ = roots; - todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group") +pub fn memo_of_queries( + roots: &[QueryRoot], + accuracy_targets: &[Option], + data: &DataWorkload, +) -> Result { + let _ = (roots, accuracy_targets, data); + todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group, each query with its target and n_q") } diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index 7eceb36ee..d1699ab00 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -6,8 +6,9 @@ use std::rc::Rc; // Stage 1 internals other crates' tests call: Pass 1's working form, // and single-candidate composition. +pub use crate::pass1::composition::compose_logical_candidate; pub use crate::pass1::per_computation_candidates::{ - compose_logical_candidate, generate_per_computation_candidates, PerComputationCandidates, + generate_per_computation_candidates, PerComputationCandidates, }; use asap_types::ir::properties::AccuracyTarget; @@ -175,7 +176,7 @@ pub fn time_range(range: Duration, child: Rc) -> Rc /// The Count-Min summary of a Stage 1 candidate of `query`. pub fn cms_summary(query: &str) -> OperatorNode { - use crate::pass1::per_computation_candidates::compose_logical_candidate; + use crate::pass1::composition::compose_logical_candidate; use crate::pass1::per_computation_candidates::generate_per_computation_candidates; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::{ASAPOp, QueryRoot}; @@ -183,7 +184,12 @@ pub fn cms_summary(query: &str) -> OperatorNode { let root = lower_promql(query, target); // Inspect each local computation choice under the built-in deployment. let per_computation = generate_per_computation_candidates( - &crate::stage0::memo_of_queries(&[QueryRoot::Operator(root)]).unwrap(), + &crate::stage0::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &asap_types::workload::DataWorkload::default(), + ) + .unwrap(), ) .unwrap() .without_unsupported_summaries(None); @@ -207,7 +213,7 @@ pub fn cms_summary(query: &str) -> OperatorNode { matches!( node.asap(), Some(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), + summary_type: FieldDataType::Sketch(kind), .. }) if *kind == SketchAlgorithm::Cms ) diff --git a/crates/physical-asap-optimization/src/catalog/mod.rs b/crates/physical-asap-optimization/src/catalog/mod.rs index e8c35a9be..391775557 100644 --- a/crates/physical-asap-optimization/src/catalog/mod.rs +++ b/crates/physical-asap-optimization/src/catalog/mod.rs @@ -1,13 +1,9 @@ //! Stage 2's physical candidates: expand each logical candidate into one //! physical candidate per valid $(\tau, \ell)$, and each logical edge //! into the physical edges between the two candidates' alternatives -//! ([`generate_physical_asap_memo_candidates`]); and realize one selected -//! DAG as a physical plan (#1047). The output type, +//! ([`generate_physical_asap_memo_candidates`]). The output type, //! `PhysicalASAPMemoCandidates`, lives in `asap_types::stages`; Stage 3 //! selects from it. -mod realize; - -pub use realize::Stage2AssignmentRealizer; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; @@ -19,8 +15,8 @@ use asap_types::workload::DataWorkload; /// logical edge into its physical edges with $\alpha^{\log}$ and /// $\alpha^{\mathrm{phys}}$. Every alternative rejected is recorded in the /// output's `pruned_candidates` with its `PruneReason`; a value outside a -/// dimension's declared value set (`AcrossEvaluations` on a candidate that -/// is not a chain) is not offered and not recorded. Costs nothing, and +/// dimension's declared value set (`AcrossEvaluations` on a candidate of a +/// group with no known grid) is not offered and not recorded. Costs nothing, and /// cannot fail. pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs deleted file mode 100644 index 9abbc6b8b..000000000 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Realize a selected DAG through Stage 2: each node's physical properties -//! come from its `PhysicalCandidate` (concepts doc, section 3, -//! "Realizer/export"). This validates execution legality, not cost estimates -//! or accuracy targets. - -use asap_types::ir::QueryRoot; -use asap_types::stages::{ - AssignmentRealizer, DagOfSelectedCandidates, PhysicalASAPMemoCandidates, - PhysicalCandidateError, RealizedPlan, -}; -use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; - -/// Stage 2's [`AssignmentRealizer`], over the planning context: the -/// workload's original query roots (one per entry of `entries`), its data, -/// and whether the deployment keeps query-time results across -/// evaluations. The facade builds it, passes it to Stage 3, and builds the -/// selected plan with it. -pub struct Stage2AssignmentRealizer<'a> { - pub original_roots: &'a [QueryRoot], - pub entries: &'a [QueryWorkloadEntry], - pub data: &'a DataWorkload, - pub query_time_results_retained: bool, -} - -impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { - fn realize( - &self, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - ) -> Result { - let _ = (physical, selected); - todo!("#1207 PR 6: build the selected DAG, each node at its PhysicalCandidate's properties") - } -} diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index 24b02fb14..5ef0d90d1 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -7,9 +7,9 @@ //! $(\tau, \ell)$, each logical edge into its physical edges with //! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives //! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from -//! that output, and [`Stage2AssignmentRealizer`] builds the selected DAG as a -//! physical plan. The crate depends on no other stage crate. +//! that output and builds the plan. The crate depends on no other stage +//! crate. mod catalog; -pub use catalog::{generate_physical_asap_memo_candidates, Stage2AssignmentRealizer}; +pub use catalog::generate_physical_asap_memo_candidates; diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index a4168acc7..d6c5fef6f 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -6,22 +6,18 @@ //! admissible part of the physical edges $\mathcal E$, selects with a MILP //! over $x_v$, $y_{u,c,i}$, $z_{q,v}$ and $t_v$ (constraints (A)–(I); //! (G) and (H) hold because $y$ exists only on admissible edges), and -//! decodes $\{x = 1\}$ and $\{y = 1\}$ into a -//! `DagOfSelectedCandidates`, which the injected realizer builds. The -//! extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, +//! builds the plan (`PhysicalASAPDAG`) from $\{x = 1\}$ and $\{y = 1\}$ +//! as its last step. The extensions add execution contexts $\kappa$ with $x^{\mathrm{mat}}_v$, //! $x_{\kappa,v}$, $y_{\kappa,u,c,i}$ and the retention amount $h_v$ //! (section 5), and accuracy at each query root, constraint (K), with //! per-candidate error bounds $E_c$ and $D_c$ (section 4). Costs come from //! `asap-cost`. mod milp; -mod models; mod report; mod select; -pub use models::AccuracyCheckInputs; pub use report::{ - large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, InexactObjective, - RejectionCause, SearchReport, SearchStatus, Selection, + large_peak_memory_warning, AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, }; pub use select::select_plan; diff --git a/crates/plan-selection/src/milp/model.rs b/crates/plan-selection/src/milp/model.rs index e976556c0..bdcd7701a 100644 --- a/crates/plan-selection/src/milp/model.rs +++ b/crates/plan-selection/src/milp/model.rs @@ -5,7 +5,7 @@ //! |---|---|---| //! | baseline (section 3) | $x_v$, $y_{u,c,i}$, $z_{q,v}$, $t_v$ ([`SelectionVariables`]) | (A) [`input_satisfaction_constraints`], (B) [`producer_selection_constraints`], (C)/(D) [`query_coverage_constraints`], (E) [`acyclicity_constraints`], (F) [`utilization_constraints`]; (G), (H): $y$ exists only on admissible edges | //! | execution contexts (section 5) | $x^{\mathrm{mat}}_v$, $x_{\kappa,v}$, $y_{\kappa,u,c,i}$, $h_v$ ([`ExecutionContextVariables`]) | (R) [`retention_constraints`] | -//! | accuracy (section 4) | $E_c$, $F_{c,i}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | +//! | accuracy (section 4) | $E_{c,m}$, $F_{c,i,m}$, $D_c$, $H_{c,i}$ ([`AccuracyVariables`]) | (K) [`accuracy_constraints`] | //! //! Every function body is `todo!()` until #1207 PRs 6 (baseline), 7 //! (contexts) and 8 (accuracy) fill them in; until then nothing calls @@ -16,6 +16,7 @@ use std::collections::HashMap; use good_lp::{Constraint, Variable}; +use asap_types::ir::properties::AccuracyMetric; use asap_types::stages::{ ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidateId, PhysicalEdgeId, }; @@ -32,8 +33,9 @@ struct SelectionVariables { y: HashMap, /// $z_{q,v}$, for $v \in G_q$. z: HashMap<(QueryId, PhysicalCandidateId), Variable>, - /// $t_v$, by [`PhysicalCandidateId`]. - t: Vec, + /// $t_v$, by [`PhysicalCandidateId`]: only for candidates on a cycle of + /// admissible edges (a strongly connected component); `None` elsewhere. + t: Vec>, } /// The execution-context variables (section 5), which replace $x$ and the @@ -51,10 +53,11 @@ struct ExecutionContextVariables { /// The accuracy variables of constraint (K) (section 4), all continuous. struct AccuracyVariables { - /// $E_c$, by [`PhysicalCandidateId`]. - error_bound: Vec, - /// $F_{c,i}$: the chosen producer's error bound on input $i$ of $c$. - error_bound_of_input: HashMap<(PhysicalCandidateId, usize), Variable>, + /// $E_{c,m}$: candidate $c$'s error bound in metric $m$. + error_bound: HashMap<(PhysicalCandidateId, AccuracyMetric), Variable>, + /// $F_{c,i,m}$: the chosen producer's error bound in metric $m$ on input + /// $i$ of $c$. + error_bound_of_input: HashMap<(PhysicalCandidateId, usize, AccuracyMetric), Variable>, /// $D_c$, by [`PhysicalCandidateId`]. failure_probability: Vec, /// $H_{c,i}$: the chosen producer's failure probability on input $i$ of $c$. @@ -88,7 +91,8 @@ fn query_coverage_constraints( todo!("#1207 PR 6: constraints (C) and (D)") } -/// (E) $t_u + 1 \le t_c + |\mathcal C|(1 - y_{u,c,i})$. +/// (E) $t_u + 1 \le t_c + n(1 - y_{u,c,i})$, only on edges inside a strongly +/// connected component of the admissible edges, $n$ its size. fn acyclicity_constraints( physical: &PhysicalASAPMemoCandidates, variables: &SelectionVariables, @@ -106,13 +110,11 @@ fn utilization_constraints( todo!("#1207 PR 6: constraint (F)") } -/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$. -/// -/// Assumptions (decided 2026-11-06): an edge's need is the time horizon -/// the queries it serves read, a constant per edge (a 5m range over 10 s -/// windows needs 30 windows), so (R) is linear; query latency is ignored -/// until the latency design: a result counts as read when its evaluation -/// starts. +/// (R) $h_v \ge \operatorname{need}_{\kappa,v,c,i}\, y_{\kappa,v,c,i}$: for a +/// time-indexed producer (raw data, a chain, results stored per evaluation +/// time) the reading path's offset plus range, a constant per edge, so (R) +/// is linear. Query latency is ignored: a result counts as read when its +/// evaluation starts. fn retention_constraints( physical: &PhysicalASAPMemoCandidates, variables: &ExecutionContextVariables, @@ -121,8 +123,9 @@ fn retention_constraints( todo!("#1207 PR 7: constraint (R)") } -/// (K): $E_c$ and $D_c$ per candidate, big-M on selected edges, and the -/// root checks $E_v \le \varepsilon_q + M_q(1 - z_{q,v})$, +/// (K): $E_{c,m}$ per candidate and metric and $D_c$ per candidate, big-M on +/// selected edges, the metric rule on the selected path, and the root +/// checks $E_{v,m_q} \le \varepsilon_q + M_q(1 - z_{q,v})$, /// $D_v \le \delta_q + (1 - z_{q,v})$. fn accuracy_constraints( physical: &PhysicalASAPMemoCandidates, diff --git a/crates/plan-selection/src/models.rs b/crates/plan-selection/src/models.rs deleted file mode 100644 index 763476654..000000000 --- a/crates/plan-selection/src/models.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! The accuracy models Stage 3 checks with: [`AccuracyCheckInputs`]. - -use asap_accuracy::{AccuracyEvidenceProvider, AccuracyModel}; - -/// Inputs of the composed-accuracy check. `raw_data_retained` is a Stage 2 -/// input only; to ignore accuracy guarantees the facade passes an -/// accept-all `model` ([`asap_accuracy::IgnoreAccuracyGuarantees`]). -#[derive(Clone, Copy)] -pub struct AccuracyCheckInputs<'a> { - /// Decides whether a candidate's accuracy guarantee meets its query's - /// target, and whether the deployment accepts an output regardless of - /// its guarantee. - pub model: &'a dyn AccuracyModel, - /// Deployment facts the guarantees need (operand domains, observed - /// guarantees). - pub evidence: &'a dyn AccuracyEvidenceProvider, -} diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index 7cac6899e..0a8be7bf3 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -1,15 +1,12 @@ -//! What plan selection reports: the selection, each rejected candidate and why, -//! and the search report. - -use std::collections::BTreeMap; +//! What plan selection reports beside the plan: the accepted accuracy, the +//! candidates it pruned and the selection MILP's size and time; and each +//! rejected candidate and why. use asap_cost::analytical::AnalyticalCostError; -use asap_cost::{CostModel, PlanCost}; +use asap_cost::CostModel; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::properties::AccuracyMetric; -use asap_types::stages::{ - DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate, SelectedNodeId, -}; +use asap_types::stages::{PhysicalCandidate, PrunedCandidate}; use thiserror::Error; /// Peak memory above which a candidate's report carries a warning when the @@ -45,7 +42,7 @@ pub struct Rejection { pub reason: String, } -/// Why Stage 3 did not select a candidate. The first six mean the +/// Why Stage 3 did not select a candidate. The first five mean the /// candidate failed a check and is invalid ([`Self::failed_a_check`]); the /// rest mean it was valid and lost under the cost model. JSON: the /// snake_case variant name, e.g. `"costlier"`. @@ -55,15 +52,8 @@ pub enum RejectionCause { /// A query's accuracy target is not met: its output's guarantee misses /// it and no deployment evidence meets it. AccuracyTargetMissed, - /// A query's plan reads a Count-Min sketch whose update weights are not - /// proven non-negative (e.g. samples of a metric declared neither a - /// counter nor non-negative). The executor cannot run it, so the deployment's accuracy - /// model cannot accept it either. - CountMinWeightsNotProvenNonNegative, - /// A query's output has no accuracy guarantee (Stage 1 has no - /// analytical rule for it and the deployment's evidence supplies none), - /// and the deployment's accuracy model did not accept it directly - /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`](asap_accuracy::AccuracyModel::accepts_output_regardless_of_guarantee)). + /// A query's output has no accuracy guarantee: no analytical rule and + /// no deployment evidence supplies one. NoAccuracyGuarantee, /// A query's latency is over its `max_latency_ms` bound. LatencyBoundExceeded, @@ -74,9 +64,9 @@ pub enum RejectionCause { /// Under cost model 3, its usage is over one of the cluster's limits. OverClusterCapacity, /// Its cost overflows the cost model's arithmetic, so it cannot be - /// priced. + /// costed. CostNotRepresentable, - /// Under cost models 1 and 2, its price is higher than the selected + /// Under cost models 1 and 2, its cost is higher than the selected /// candidate's. Costlier, /// Under cost model 3's `lowest-latency`, it is slower than the @@ -102,7 +92,6 @@ impl RejectionCause { matches!( self, Self::AccuracyTargetMissed - | Self::CountMinWeightsNotProvenNonNegative | Self::NoAccuracyGuarantee | Self::LatencyBoundExceeded | Self::LatencyOverRepeatInterval @@ -113,66 +102,30 @@ impl RejectionCause { } } -/// The accuracy Stage 3 accepted for one query of a valid candidate: the -/// metric the query's target set (if any) and the metric of the guarantee -/// that met it. With no metric set, any metric meeting the target's ε, δ -/// and scope is accepted, so ε means different things across candidates -/// (e.g. 0.01 of the true value vs 0.01 of the rank); this record makes -/// the accepted metric visible. +/// The accuracy Stage 3 accepted for one query: the metric of the error +/// bound that met its target, and the sketches its output reads. #[derive(Debug, Clone, PartialEq)] pub struct AcceptedAccuracy { - /// The target's metric; `None` when the user did not set one. - pub target_metric: Option, - /// The metric the accepted guarantee is stated in; `None` for an exact - /// output, which meets every metric, and for one the deployment's model - /// accepted regardless of its guarantee (no guarantee was checked). + /// The metric of the accepted error bound; `None` for an exact output, + /// which meets every metric, and for a query without an accuracy target. pub accepted_metric: Option, - /// Whether the deployment's accuracy model accepted the output directly, - /// whatever its guarantee says - /// ([`AccuracyModel::accepts_output_regardless_of_guarantee`](asap_accuracy::AccuracyModel::accepts_output_regardless_of_guarantee)), as - /// every output is with [`IgnoreAccuracyGuarantees`](asap_accuracy::IgnoreAccuracyGuarantees); `false` when its - /// guarantee was checked and met the target. - pub accepted_regardless_of_guarantee: bool, - /// When accepted regardless of its guarantee, the cause the guarantee - /// check would have rejected it with - /// ([`RejectionCause::NoAccuracyGuarantee`] or - /// [`RejectionCause::AccuracyTargetMissed`]): the output was accepted - /// only because of the accuracy model (e.g. - /// [`IgnoreAccuracyGuarantees`](asap_accuracy::IgnoreAccuracyGuarantees)). `None` when - /// its guarantee meets the target. - pub guarantee_check_rejection: Option, /// The sketch algorithms the query's output reads, in plan order /// (e.g. `["Kll"]`); empty for an exact plan. pub sketches: Vec, } impl std::fmt::Display for AcceptedAccuracy { - /// E.g. `metric not set; accepted rank (Kll)`, or - /// `metric relative; accepted relative (DDSketch)`, or - /// `metric rank; accepted by the deployment's accuracy model regardless of guarantee (Kll)`. + /// E.g. `accepted rank (Kll)`, or `accepted exact`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let name = |metric: AccuracyMetric| { - serde_json::to_value(metric) - .ok() - .and_then(|value| value.as_str().map(str::to_owned)) - .unwrap_or_else(|| format!("{metric:?}")) - }; - match self.target_metric { - None => write!(f, "metric not set; ")?, - Some(metric) => write!(f, "metric {}; ", name(metric))?, - } match self.accepted_metric { - _ if self.accepted_regardless_of_guarantee => { - write!( - f, - "accepted by the deployment's accuracy model regardless of guarantee" - )?; - if let Some(cause) = self.guarantee_check_rejection { - write!(f, ", ignoring a failed guarantee check ({cause:?})")?; - } - } None => write!(f, "accepted exact")?, - Some(metric) => write!(f, "accepted {}", name(metric))?, + Some(metric) => { + let name = serde_json::to_value(metric) + .ok() + .and_then(|value| value.as_str().map(str::to_owned)) + .unwrap_or_else(|| format!("{metric:?}")); + write!(f, "accepted {name}")? + } } if !self.sketches.is_empty() { write!(f, " ({})", self.sketches.join(", "))?; @@ -181,83 +134,22 @@ impl std::fmt::Display for AcceptedAccuracy { } } -/// Stage 3's result ([`select_plan`](crate::select_plan)): the selected DAG -/// over the [`PhysicalASAPMemoCandidates`](asap_types::stages::PhysicalASAPMemoCandidates) -/// it was given, its cost and accepted accuracy, and the search report. -/// The facade builds the plan from `selected_dag` with the same -/// [`AssignmentRealizer`](asap_types::stages::AssignmentRealizer). +/// What Stage 3 reports about its own run, beside the plan +/// ([`select_plan`](crate::select_plan)). The plan's cost is +/// `asap_cost::plan_cost` of the plan; this does not repeat it. #[derive(Debug, Clone, PartialEq)] pub struct Selection { - /// The selected plan: one node per run of a candidate. - pub selected_dag: DagOfSelectedCandidates, - /// Nodes of `selected_dag` whose open count-min weight requirement - /// Stage 3 proved; the facade applies the proof to each node's operator - /// node when it builds the plan. - pub nodes_with_proven_non_negative_weights: Vec, - pub cost: PlanCost, /// Per query, the accuracy Stage 3 accepted for the selected plan. pub accepted_accuracy_per_query: Vec, - /// The candidates Stage 3 ruled out before the search: dominated ones and - /// those whose usage could not be costed. - pub pruned_candidates: Vec>, - pub search_report: SearchReport, -} - -/// How the search behind a selection ended, and what it cost. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct SearchReport { - pub status: SearchStatus, - /// The solver's lower bound on the selected plan's cost per second; - /// `None` when the solver's dual bound is not finite. - pub cost_lower_bound_per_second: Option, - /// The largest MILP the search built. + /// The candidates Stage 3 ruled out before the selection MILP: + /// dominated ones and those whose usage could not be costed. + pub pruned_candidates: Vec>, + /// The selection MILP's size. pub milp_variables: usize, pub milp_constraints: usize, - pub model_construction_ms: u64, - pub solver_ms: u64, - pub validation_ms: u64, - /// Set when `asap_cost::plan_cost` of the selected plan differs from - /// the selection MILP's objective and the model encoded some cost terms - /// inexactly (open questions on #1095, e.g. the working-memory term), so - /// the plan is returned with its cost equality unproven. `None` when - /// the two are equal (within a relative 10⁻⁶). A - /// difference with no inexact term is an error - /// ([`ErrorSelectingPlan::ObjectiveDiffersFromPlanCost`]). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub inexact_objective: Option, -} - -/// The selected plan's cost per second as the selection MILP's objective -/// and as `asap_cost::plan_cost` computes it (both without request fees), -/// and the cost terms the model encoded inexactly, with how many times. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct InexactObjective { - pub objective_per_second: f64, - pub plan_cost_per_second: f64, - pub inexact_terms: BTreeMap, -} - -impl SearchReport { - /// How far the lower bound is below `cost_per_second`, as a fraction - /// of it in [0, 1]; `None` without a bound. - pub fn relative_gap(&self, cost_per_second: f64) -> Option { - self.cost_lower_bound_per_second.map(|bound| { - if cost_per_second == 0.0 { - 0.0 - } else { - ((cost_per_second - bound).max(0.0) / cost_per_second.abs()).min(1.0) - } - }) - } -} - -/// How the search ended with a validated plan. JSON: snake_case. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SearchStatus { - /// The search proved the plan optimal over the catalog it was given - /// (the only outcome: the solve has no time limit). - Optimal, + /// How long building and solving the selection MILP took. + pub milp_construction_ms: u64, + pub milp_solve_ms: u64, } /// Why Stage 3 selected no plan. @@ -283,36 +175,12 @@ pub enum ErrorSelectingPlan { }, #[error("no valid candidate: {0:?}")] NoValidCandidate(Vec), - /// Stage 2 rejected a plan Stage 3 decoded from its catalog: a Stage 2 - /// catalog bug. - #[error("the realizer rejected the selected plan: {0}")] - RealizerRejected(String), /// The memo search's MILP solver failed. #[error("MILP solver: {0}")] Solver(String), /// The selection MILP implements cost model 1 only (#1095 §3). #[error("the selection MILP does not support {0}")] UnsupportedCostModel(String), - /// `asap_cost::plan_cost` of the selected plan differs from the - /// selection MILP's objective (both without request fees) and the model - /// encoded every cost term exactly: a bug in a cost term. - #[error( - "the selection MILP's objective {objective_per_second:e} differs from plan_cost \ - {plan_cost_per_second:e} (per second, without request fees)" - )] - ObjectiveDiffersFromPlanCost { - objective_per_second: f64, - plan_cost_per_second: f64, - }, - /// The selected plan fails a check the selection MILP encodes - /// (accuracy through the budget shares, latency): a bug in the - /// accuracy weights, the sizing rule or a latency term, never a reason - /// to re-solve. - #[error("the selected plan fails a check the MILP encodes ({cause:?}): {reason}")] - SelectedPlanFailedCheck { - cause: RejectionCause, - reason: String, - }, } #[cfg(test)] diff --git a/crates/plan-selection/src/select.rs b/crates/plan-selection/src/select.rs index f0468ae33..c2989b28a 100644 --- a/crates/plan-selection/src/select.rs +++ b/crates/plan-selection/src/select.rs @@ -1,36 +1,33 @@ //! Stage 3, plan selection: [`select_plan`] builds the selection MILP over //! Stage 2's physical candidates and admissible edges -//! ([`crate::milp::model`]), solves it, decodes $\{x = 1\}$ and $\{y = 1\}$ -//! into a [`DagOfSelectedCandidates`](asap_types::stages::DagOfSelectedCandidates), -//! and realizes it once through the injected realizer. +//! ([`crate::milp::model`]), solves it, and builds the plan from the +//! selected executions as its last step. use asap_cost::CostInputs; -use asap_types::stages::{AssignmentRealizer, PhysicalASAPMemoCandidates}; +use asap_types::ir::physical_export::PhysicalASAPDAG; +use asap_types::stages::PhysicalASAPMemoCandidates; use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; -use crate::models::AccuracyCheckInputs; use crate::report::{ErrorSelectingPlan, Selection}; /// Stage 3, plan selection: select one plan among `physical`'s candidates -/// (module documentation). The selection is realized through `realizer`, -/// checked against every query's accuracy target (`accuracy_checks`) and -/// priced under `cost_inputs`. `entries[q]` is query `q`'s workload entry; -/// `data` the data workload with its bounds. +/// (module documentation) under `cost_inputs`, and build it: one +/// `PhysicalASAPDAG` node per selected execution, its inputs from the +/// selected edges, each node's `selected_physical_candidate` set. +/// `entries[q]` is query `q`'s workload entry; `data` the data workload +/// with its bounds. /// -/// Err: invalid or unsupported cost inputs, an infeasible catalog, a -/// solver failure, or the realizer rejected the selected plan -/// ([`ErrorSelectingPlan::RealizerRejected`]: a Stage 2 catalog bug). +/// Err: invalid or unsupported cost inputs, an infeasible catalog, or a +/// solver failure. pub fn select_plan( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, cost_inputs: &CostInputs, - realizer: &dyn AssignmentRealizer, -) -> Result { +) -> Result<(PhysicalASAPDAG, Selection), ErrorSelectingPlan> { cost_inputs .validate() .map_err(ErrorSelectingPlan::InvalidSettings)?; - let _ = (physical, entries, data, accuracy_checks, realizer); - todo!("#1207 PR 6: the baseline MILP over x, y, z, t, its decode and realization") + let _ = (physical, entries, data); + todo!("#1207 PR 6: the baseline MILP over x, y, z, t, and the PhysicalASAPDAG of its solution") } diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 63802bcf6..6cf31bc74 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -36,8 +36,9 @@ asap-plan-selection = { path = "../plan-selection" } asap-physical-asap-optimization = { path = "../physical-asap-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -# The deployment accuracy model tests implement `AccuracyModel`. asap-accuracy = { path = "../accuracy", features = ["test-support"] } +# Tests read a plan's roots as IR (`PhysicalASAPDAG::query_roots`). +asap-types = { path = "../types", features = ["test-support"] } # The SQL frontend plans through DataFusion, which is async; `e2e_plan` is # therefore async and its tests need a runtime. tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index 96003283a..64bfc668a 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -25,8 +25,7 @@ use std::collections::{HashMap, HashSet}; -use asap_cost::{NodePriceSource, OperatorKind}; -use asap_plan_selection::{SearchReport, Selection}; +use asap_cost::{NodeCostSource, OperatorKind, PlanCost}; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, RawDataRetention, @@ -38,7 +37,6 @@ use asap_types::ir::physical_export::{ }; use asap_types::ir::properties::ExecutionTiming; -use asap_types::stages::RealizedPlan; use asap_types::workload::QueryWorkloadEntry; use serde::{Deserialize, Serialize}; @@ -46,7 +44,7 @@ use crate::{e2e_plan, PlanError, UserInput}; /// The version of the plan-file format [`InstallablePlan`] serializes to: /// the DAG, its nodes' fields and the query texts. -pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 13; +pub const INSTALLABLE_PLAN_WIRE_VERSION: u32 = 14; /// The selected plan, ready to install and execute: what /// [`plan_installable`] returns and a runtime writes as its plan file. @@ -65,7 +63,8 @@ pub struct InstallablePlan { pub query_texts: Vec, } -/// What Stage 3 recorded about the plan it selected, for reporting. +/// The selected plan's cost (`asap_cost::plan_cost` of the plan), for +/// reporting. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SelectedPlanReport { @@ -75,15 +74,6 @@ pub struct SelectedPlanReport { /// The resources Stage 3 predicts it uses, for a runtime to report /// against what it measures. pub predicted_usage: PredictedUsage, - /// How the search behind the selection ended (`Selection::search_report`). - pub search_report: SearchReport, - /// The workload queries (entry indices) whose output Stage 3 accepted - /// only because the accuracy model accepts it regardless of its - /// guarantee (e.g. `PlanningModels::ignore_accuracy_guarantees`): - /// the guarantee check would have rejected it. Empty when every - /// output's guarantee meets its target; omitted from JSON then. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub queries_accepted_without_guarantee: Vec, } /// What Stage 3 predicts the selected plan uses, per second of wall time @@ -134,7 +124,7 @@ pub struct PredictedNodeUsage { /// 2 and 3. pub cost_per_second: Option, /// Whether its CPU came from the measured cost table or the formulas. - pub price_source: NodePriceSource, + pub cost_source: NodeCostSource, } /// One query's predicted latency per evaluation. @@ -176,10 +166,6 @@ pub enum InstallablePlanError { /// [`InstallableDagError`]). #[error("{0}")] Uninstallable(InstallableDagError), - /// Internal bug: Stage 3 records a tumbling-window retention for node `node_id` - /// of the selected plan, which is not one of its tumbling-window chains. - #[error("tumbling-window retention for node {node_id}, which is not a tumbling-window chain of the selected plan")] - TumblingWindowRetentionOfUnknownChain { node_id: PhysicalASAPNodeId }, /// Unsatisfiable request: the executor cannot refresh materialized view /// `node_id` from its inputs (`reason`); for a refresh group, `node_id` /// is its first view and `reason` names every view compiled with it. @@ -206,9 +192,8 @@ impl From for InstallablePlanError { /// Plan `input` for a deployment runtime: the query-language frontends and /// Stages 1-3 as [`e2e_plan`] runs them, returning the single selected DAG -/// ready to install, and beside it Stage 3's report of the selection -/// (`None` for a pass that selects without pricing), which is not part of -/// the plan file. Supported summaries the caller leaves unrestricted +/// ready to install, and beside it the plan's cost report, which is not +/// part of the plan file. Supported summaries the caller leaves unrestricted /// (`DeploymentCapabilities::supported_summaries` `None`) default to the /// families the Planner's executor runs; raw-data and query-time-result /// retention stay the caller's. Stage 3 does not compile candidates; @@ -230,19 +215,18 @@ pub async fn plan_installable( }; let output = e2e_plan(input).await?; let _ = (output, entries); - todo!("#1207 PR 6: install the plan the realizer built from the selected DAG") + todo!("#1207 PR 6: install the plan Stage 3 built, with its plan_cost report") } -/// The installable plan of `realized`, the plan the realizer -/// (`AssignmentRealizer::realize`) built from the selected DAG of a -/// workload whose entries are `entries` (in `QueryWorkload::entries()` -/// order), with Stage 3's report of its `selection` when there is one: what -/// [`plan_installable`] returns after planning. A caller that runs the -/// planning stages itself (e.g. a developer tool) uses it to see what a +/// The installable plan of `plan`, the plan Stage 3 built for a workload +/// whose entries are `entries` (in `QueryWorkload::entries()` order), with +/// the report of its `cost` (`asap_cost::plan_cost` of `plan`) when there is +/// one: what [`plan_installable`] returns after planning. A caller that runs +/// the planning stages itself (e.g. a developer tool) uses it to see what a /// runtime would install. pub fn installable_plan_of( - realized: &RealizedPlan, - selection: Option<&Selection>, + plan: &PhysicalASAPDAG, + cost: Option<&PlanCost>, entries: &[QueryWorkloadEntry], ) -> Result<(InstallablePlan, Option), InstallablePlanError> { // Each query's longest range-evaluation span, which the views it @@ -252,8 +236,8 @@ pub fn installable_plan_of( .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) .collect(); let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - let expanded = tumbling_window_retention_of_expanded_tumbling_windows(realized)?; - let report = selection.map(|selection| report_of(selection, &expanded.installed_node_ids)); + let expanded = tumbling_window_retention_of_expanded_tumbling_windows(plan)?; + let report = cost.map(|cost| report_of(cost, &expanded.installed_node_ids)); let reader_span_ms_by_node = reader_span_ms_by_node(&expanded.dag, &range_spans_ms); let plan = InstallablePlan::of_dag( expanded.dag, @@ -612,31 +596,21 @@ struct ExpandedWithRetention { installed_node_ids: HashMap>, } -/// The realized plan's DAG with its tumbling-window chains expanded; each -/// tumbling window takes the retention Stage 2 gives its chain -/// (`RealizedPlan::tumbling_window_retention_ms`). +/// The plan's DAG with its tumbling-window chains expanded; each tumbling +/// window takes its chain's retention (`retention_ms`, Stage 3's $h_v$). fn tumbling_window_retention_of_expanded_tumbling_windows( - realized: &RealizedPlan, + plan: &PhysicalASAPDAG, ) -> Result { - let _ = realized; - todo!("#1207 PR 6: expand the realized DAG's tumbling-window chains") + let _ = plan; + todo!("#1207 PR 6: expand the plan's tumbling-window chains") } -/// Stage 3's record of its selected plan. +/// The report of the selected plan's `cost`. fn report_of( - selection: &Selection, + cost: &PlanCost, installed_node_ids: &HashMap>, ) -> SelectedPlanReport { - let cost = &selection.cost; - let queries_accepted_without_guarantee = selection - .accepted_accuracy_per_query - .iter() - .enumerate() - .filter(|(_, accepted)| accepted.guarantee_check_rejection.is_some()) - .map(|(query, _)| query) - .collect(); SelectedPlanReport { - queries_accepted_without_guarantee, cost_per_second: cost.cost_per_second, predicted_usage: PredictedUsage { cpu_core_seconds_per_second: cost @@ -659,7 +633,7 @@ fn report_of( timing: node.timing, cpu_core_seconds_per_second: node.cpu_core_seconds_per_second, cost_per_second: node.cost_per_second, - price_source: node.price_source, + cost_source: node.cost_source, }) .collect(), per_query: cost @@ -672,7 +646,6 @@ fn report_of( }) .collect(), }, - search_report: selection.search_report.clone(), } } @@ -708,7 +681,7 @@ mod tests { "predictability": {"predictable": {"known_at": null}}}]}, "data_workload": {"arrival": {"ingesting": {"rows_per_second": 10000.0}}, "sources": {"data": {"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": true}}}}"#, + "sample_interval": 100, "column_signs": {"value": "non_negative"}}}}}"#, ) .unwrap(); let capabilities = DeploymentCapabilities { diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index f114db832..da85ad88e 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -6,10 +6,10 @@ //! ``` //! //! The default pass, [`Stage1ToStage3Pass`], orchestrates the stages -//! ([`run_planning`]): Stage 0, Pass 1, Pass 2 over every tumbling-window width, -//! Stage 2 (`generate_physical_asap_memo_candidates`) and Stage 3 -//! (`asap_plan_selection::select_plan`), and builds the selected plan through -//! Stage 2's realizer. The stages never call each other. +//! ([`run_planning`]): Stage 0, Pass 1, Pass 2, Stage 2 +//! (`generate_physical_asap_memo_candidates`) and Stage 3 +//! (`asap_plan_selection::select_plan`), which builds the selected plan. +//! The stages never call each other. //! //! [`e2e_plan`] runs both steps. [`plan_installable`] runs them for a //! deployment runtime and returns the single selected DAG ready to install @@ -29,7 +29,7 @@ mod pass; mod run; pub use pass::{ optimize, OptimizationInput, OptimizationInputError, OptimizationPass, OptimizeError, - PassNameConflict, PassRegistry, PlanOutput, PlanningModels, QueryPlan, + PassNameConflict, PassRegistry, PlanOutput, PlanningModels, }; pub use run::{ run_planning, PlanSearchSettings, PlanningReport, PlanningRun, SelectionError, @@ -47,13 +47,11 @@ pub use installable::{ // `InstallablePlan`, so it depends on this crate and `asap-executor` alone. pub use asap_cost::{ ClusterLimits, ClusterObjective, CostInputs, CostModel, CostTableProvenance, - MeasuredOperationCost, MeasuredOperationCosts, NodePriceSource, OperatorCost, OperatorKind, + MeasuredOperationCost, MeasuredOperationCosts, NodeCostSource, OperatorCost, OperatorKind, ResourceSpeed, ResourceUnitCost, SummaryOperation, }; pub use asap_frontend_sql::SqlCatalog; -pub use asap_plan_selection::{ - AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, SearchReport, SearchStatus, Selection, -}; +pub use asap_plan_selection::{AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection}; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; pub use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, @@ -226,17 +224,12 @@ async fn lower(input: &UserInput<'_>) -> Result, }; let mut lowered = Vec::new(); for (index, entry) in input.workload.query_workload.entries().enumerate() { - let expr = lower_sql_dialect( - &entry.query.0, - catalog, - dialect.clone(), - entry.requirements.accuracy.clone(), - ) - .await - .map_err(|source| PlanError::Lowering { - entry_index: Some(index), - source: LoweringError::Sql(source), - })?; + let expr = lower_sql_dialect(&entry.query.0, catalog, dialect.clone()) + .await + .map_err(|source| PlanError::Lowering { + entry_index: Some(index), + source: LoweringError::Sql(source), + })?; lowered.push(expr.into()); } Ok(lowered) diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 532cf66fa..273a8b2b0 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -14,26 +14,18 @@ //! downstream consumers rely on. use std::collections::BTreeMap; -use std::rc::Rc; -use asap_types::ir::OperatorNode; use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; use crate::run::{SelectionError, Stage1ToStage3Pass}; -use asap_accuracy::{ - AccuracyEvidenceProvider, AccuracyModel, BuiltinAccuracyModel, IgnoreAccuracyGuarantees, - NoAccuracyEvidence, -}; +use asap_accuracy::AccuracyEvidence; use asap_cost::CostInputs; use asap_logical_asap_optimization::PerComputationCandidateError; -use asap_plan_selection::{AccuracyCheckInputs, Selection}; use asap_types::deployment_capabilities::DeploymentCapabilities; -use asap_types::ir::physical_export::PhysicalASAPNodeId; +use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; -static BUILTIN_ACCURACY_MODEL: BuiltinAccuracyModel = BuiltinAccuracyModel; -static IGNORE_ACCURACY_GUARANTEES: IgnoreAccuracyGuarantees = IgnoreAccuracyGuarantees; -static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; +static NO_ACCURACY_EVIDENCE: AccuracyEvidence = AccuracyEvidence {}; static DEFAULT_COST_INPUTS: std::sync::LazyLock = std::sync::LazyLock::new(CostInputs::default); static DEFAULT_DEPLOYMENT: std::sync::LazyLock = @@ -45,21 +37,15 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = /// a built-in default, evidence about a particular deployment cannot. /// /// The facade splits them by stage: Stage 1 reads the supported summaries -/// of `deployment_capabilities`, Stage 2 its retention, and Stage 3 -/// (plan selection) checks each candidate's guarantee with `accuracy` and -/// `evidence` ([`AccuracyCheckInputs`]) and compares candidates under -/// `cost_inputs` ([`CostInputs`]). +/// of `deployment_capabilities`, Pass 2 the accuracy `evidence`, Stage 2 the +/// deployment's retention, and Stage 3 (plan selection) compares candidates +/// under `cost_inputs` ([`CostInputs`]). #[derive(Clone, Copy)] #[non_exhaustive] pub struct PlanningModels<'a> { - /// Decides whether a candidate's accuracy guarantee meets its query's - /// target, and whether the deployment accepts an output regardless of - /// its guarantee; [`BuiltinAccuracyModel`] by default. - /// `ignore_accuracy_guarantees` overrides it. - pub accuracy: &'a dyn AccuracyModel, - /// Deployment facts the guarantees need (operand domains, observed - /// guarantees); [`NoAccuracyEvidence`] by default. - pub evidence: &'a dyn AccuracyEvidenceProvider, + /// The deployment's measured accuracy, which Pass 2 reads before each + /// summary's theoretical guarantee; none by default. + pub evidence: &'a AccuracyEvidence, /// The deployment's cost inputs: the cost model (default /// cost model 1 with illustrative unit costs), the resource speed and /// the non-repeating query horizon. @@ -69,25 +55,18 @@ pub struct PlanningModels<'a> { /// whether it keeps query-time results across evaluations (Stage 2); /// default [`DeploymentCapabilities::default`]. pub deployment_capabilities: &'a DeploymentCapabilities, - /// Accept every query output regardless of its accuracy guarantee: the - /// facade passes Stage 3 [`IgnoreAccuracyGuarantees`] in place of - /// `accuracy`. For comparing candidates on cost alone, e.g. a - /// benchmark of sketch plans with no analytical guarantee; - /// executability checks still apply, and each output accepted only - /// because of it is marked in the report - /// (`AcceptedAccuracy::guarantee_check_rejection`). Default `false`. + /// Do not enforce accuracy: the facade gives every query "no accuracy + /// target" before Pass 2, so no readout gets a required accuracy and + /// every θ connects (a top-k's heap still holds k). For comparing + /// candidates on cost alone, e.g. a benchmark of sketch plans with no + /// analytical guarantee. Default `false`. pub ignore_accuracy_guarantees: bool, } impl<'a> PlanningModels<'a> { - /// These accuracy and evidence models with the default plan selection - /// settings. - pub fn new( - accuracy: &'a dyn AccuracyModel, - evidence: &'a dyn AccuracyEvidenceProvider, - ) -> Self { + /// This evidence with the default plan selection settings. + pub fn new(evidence: &'a AccuracyEvidence) -> Self { Self { - accuracy, evidence, cost_inputs: &DEFAULT_COST_INPUTS, deployment_capabilities: &DEFAULT_DEPLOYMENT, @@ -95,43 +74,19 @@ impl<'a> PlanningModels<'a> { } } - /// The built-in models. + /// The built-in models: no accuracy evidence. pub fn builtin() -> PlanningModels<'static> { - PlanningModels { - accuracy: &BUILTIN_ACCURACY_MODEL, - evidence: &NO_ACCURACY_EVIDENCE, - cost_inputs: &DEFAULT_COST_INPUTS, - deployment_capabilities: &DEFAULT_DEPLOYMENT, - ignore_accuracy_guarantees: false, - } - } - - /// Stage 3's accuracy check inputs: `evidence`, and `accuracy`, or - /// [`IgnoreAccuracyGuarantees`] with `ignore_accuracy_guarantees`. - pub fn accuracy_check_inputs(&self) -> AccuracyCheckInputs<'a> { - AccuracyCheckInputs { - model: if self.ignore_accuracy_guarantees { - &IGNORE_ACCURACY_GUARANTEES - } else { - self.accuracy - }, - evidence: self.evidence, - } + PlanningModels::new(&NO_ACCURACY_EVIDENCE) } - /// These models, accepting every query output regardless of its - /// accuracy guarantee when `ignore` (`ignore_accuracy_guarantees`). + /// These models, enforcing no accuracy target when `ignore` + /// (`ignore_accuracy_guarantees`). pub fn with_ignore_accuracy_guarantees(mut self, ignore: bool) -> Self { self.ignore_accuracy_guarantees = ignore; self } - pub fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { - self.accuracy = accuracy; - self - } - - pub fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidenceProvider) -> Self { + pub fn with_evidence(mut self, evidence: &'a AccuracyEvidence) -> Self { self.evidence = evidence; self } @@ -182,123 +137,19 @@ pub enum OptimizationInputError { // ── Output ─────────────────────────────────────────────────────────────── -/// One query's selected post-ASAP DAG, or one operator sub-DAG a standalone -/// scalar query reads. The DAG is `root`. -#[derive(Debug, Clone)] -pub struct QueryPlan { - /// Index into `QueryWorkload::entries()`. - pub entry_index: usize, - pub root: Rc, -} - -/// One multi-root workload DAG with query bindings in entry order; -/// `check_contract` enforces that. -/// -/// Plans are not deduplicated across entries: a summary state that several -/// queries share appears in each of their plans as the same `Rc`, so a -/// consumer that deploys or costs the workload must dedupe by `Rc::ptr_eq`. +/// The selected plan: one `PhysicalASAPDAG`, the only plan structure, in +/// memory and, serialized, in the plan file installed on the executor. Its +/// `roots` answer the operator queries and its `scalar_roots` the standalone +/// scalar queries, in entry order; `check_contract` enforces that. #[derive(Debug, Clone)] #[non_exhaustive] pub struct PlanOutput { - pub plans: Vec, - /// Standalone scalar queries, keyed by position in the batch, which is - /// the entry index because `check_contract` requires dense entry - /// indices. Each operator reference is the root of one of - /// `scalar_plans`. - pub scalar_roots: Vec<(usize, asap_types::ir::ScalarExpr)>, - /// One plan per distinct operator sub-DAG a scalar root reads, keyed by - /// that scalar's entry, in `scalar_roots` order. - pub scalar_plans: Vec, - /// How plan selection chose the plan, when the pass selects among priced - /// candidates. - pub selection: Option, - /// For each tumbling-window chain of the selected plan materialized beyond one - /// evaluation, by node id in its physical DAG, how long its completed - /// tumbling windows are retained ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); - /// empty for a pass that does not set it. - pub tumbling_window_retention_ms: BTreeMap, + pub plan: PhysicalASAPDAG, } impl PlanOutput { - pub fn new(plans: Vec) -> Self { - Self { - plans, - scalar_roots: Vec::new(), - scalar_plans: Vec::new(), - selection: None, - tumbling_window_retention_ms: BTreeMap::new(), - } - } - - /// Entry indices in output order. - pub fn entry_indices(&self) -> Vec { - let mut indices: Vec<_> = self - .plans - .iter() - .map(|p| p.entry_index) - .chain(self.scalar_roots.iter().map(|(i, _)| *i)) - .collect(); - if !self.scalar_roots.is_empty() { - indices.sort_unstable(); - } - indices - } - - /// All query roots in workload order, including standalone scalars. - pub fn roots(&self) -> Vec { - let mut roots: Vec<_> = self - .plans - .iter() - .map(|p| { - ( - p.entry_index, - asap_types::ir::QueryRoot::Operator(Rc::clone(&p.root)), - ) - }) - .chain( - self.scalar_roots - .iter() - .map(|(i, expr)| (*i, asap_types::ir::QueryRoot::Scalar(expr.clone()))), - ) - .collect(); - roots.sort_by_key(|(i, _)| *i); - roots.into_iter().map(|(_, root)| root).collect() - } - - /// The selected operator roots. Use `roots()` to include scalar queries. - pub fn operator_roots(&self) -> Vec> { - self.plans.iter().map(|p| Rc::clone(&p.root)).collect() - } - - /// Unique operators in the entire workload DAG, including scalar-plan dependencies. - /// Several query roots can reach the same operator; it is returned once. - pub fn operators(&self) -> Vec> { - let mut seen = std::collections::HashSet::new(); - let mut nodes = Vec::new(); - for root in self.roots() { - let inputs = match root { - asap_types::ir::QueryRoot::Operator(node) => vec![node], - asap_types::ir::QueryRoot::Scalar(expr) => { - expr.operator_refs().into_iter().cloned().collect() - } - }; - for input in inputs { - for node in OperatorNode::reachable(&input) { - if seen.insert(Rc::as_ptr(&node)) { - nodes.push(node); - } - } - } - } - nodes - } - - pub fn len(&self) -> usize { - self.plans.len() + self.scalar_roots.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 + pub fn new(plan: PhysicalASAPDAG) -> Self { + Self { plan } } } @@ -315,6 +166,12 @@ pub enum OptimizeError { /// defect in the pass, not in its input. #[error("pass `{pass}` violated the output contract: {detail}")] ContractViolation { pass: &'static str, detail: String }, + /// The pass returned an invalid DAG: a defect in the pass. + #[error("pass `{pass}` returned an invalid plan: {error}")] + InvalidPlan { + pass: &'static str, + error: PhysicalASAPDAGValidationError, + }, } // ── The pass ───────────────────────────────────────────────────────────── @@ -333,9 +190,9 @@ pub trait OptimizationPass { /// Run `pass` over `input`: validate once, then check the output contract. /// /// Prefer this over calling [`OptimizationPass::optimize`] directly. A pass is -/// third-party code, but `ASAPQuery-backend` and `ASAPCollector` read every -/// pass's output against the same contract; checking here means a defective -/// pass fails at the boundary instead of corrupting a deployment. +/// third-party code, but every deployment runtime reads every pass's output +/// against the same contract; checking here means a defective pass fails at +/// the boundary instead of corrupting a deployment. pub fn optimize( pass: &dyn OptimizationPass, input: OptimizationInput<'_>, @@ -348,53 +205,26 @@ pub fn optimize( Ok(output) } -/// Structural checks only — that every query is accounted for, in order. -/// Whether the pass chose *well* is not checked. +/// Structural checks only — that every query is answered once, by an +/// operator root or the scalar root keyed with its entry index, and that +/// the DAG is valid. Whether the pass chose *well* is not checked. fn check_contract( output: &PlanOutput, pass: &'static str, expected_len: usize, ) -> Result<(), OptimizeError> { - let violation = |detail: String| OptimizeError::ContractViolation { pass, detail }; - - if output.len() != expected_len { - return Err(violation(format!( - "{} plan(s) for {expected_len} workload entry/entries", - output.len() - ))); - } - for (position, entry_index) in output.entry_indices().into_iter().enumerate() { - if entry_index != position { - return Err(violation(format!( - "plan at position {position} carries entry_index {entry_index}" - ))); - } - } - // `scalar_plans` holds one plan per distinct reference of each scalar - // root, in `scalar_roots` order, keyed by that scalar's entry. - let mut scalar_plans = output.scalar_plans.iter(); - for (entry_index, expr) in &output.scalar_roots { - let mut refs: Vec<&Rc> = Vec::new(); - for node in expr.operator_refs() { - if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { - refs.push(node); - } - } - for node in refs { - match scalar_plans.next() { - Some(plan) if plan.entry_index == *entry_index && Rc::ptr_eq(&plan.root, node) => {} - _ => { - return Err(violation(format!( - "scalar root {entry_index} reads an operator without its scalar plan" - ))) - } - } - } - } - if scalar_plans.next().is_some() { - return Err(violation( - "a scalar plan is read by no scalar root".to_string(), - )); + let plan = &output.plan; + let answered = plan.roots.len() + plan.scalar_roots.len(); + if answered != expected_len { + return Err(OptimizeError::ContractViolation { + pass, + detail: format!("{answered} plan root(s) for {expected_len} workload entry/entries"), + }); + } + // An empty workload's plan has no roots, which `validate` rejects. + if expected_len > 0 { + plan.validate() + .map_err(|error| OptimizeError::InvalidPlan { pass, error })?; } Ok(()) } @@ -465,137 +295,60 @@ mod tests { struct Stub(&'static str); + fn empty_plan() -> PlanOutput { + PlanOutput::new(PhysicalASAPDAG { + nodes: vec![], + edges: vec![], + roots: vec![], + scalar_roots: vec![], + }) + } + impl OptimizationPass for Stub { fn name(&self) -> &'static str { self.0 } fn optimize(&self, _input: OptimizationInput<'_>) -> Result { - Ok(PlanOutput::new(Vec::new())) + Ok(empty_plan()) } } - /// Dropping a query is rejected: the output is positionally aligned with - /// the workload's entries, so a short vector is not a partial result. + /// Dropping a query is rejected: the plan's roots answer the workload's + /// entries in order, so a plan with fewer roots is not a partial result. #[test] fn contract_rejects_a_plan_count_that_does_not_cover_every_entry() { - let output = PlanOutput::new(Vec::new()); - let err = check_contract(&output, "stub", 2).unwrap_err(); + let err = check_contract(&empty_plan(), "stub", 2).unwrap_err(); assert!(matches!( err, OptimizeError::ContractViolation { pass: "stub", .. } )); } - /// A scalar root's operator reference must be the root of its scalar - /// plan; a scalar root without one is rejected. + /// A scalar root keyed by an entry index the workload does not have is + /// rejected as an invalid plan. #[test] - fn contract_rejects_a_scalar_reference_without_its_plan() { - use asap_types::ir::operator::non_asap::NonASAPOp; - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{Operator, ScalarExpr}; - let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![Field::plain("ts", DataType::Timestamp, false)], - 0, - vec![], - ), - })) - .unwrap(); - let mut output = PlanOutput::new(Vec::new()); + fn contract_rejects_a_scalar_root_keyed_beyond_the_entries() { + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::ScalarExpr; + let mut output = empty_plan(); output + .plan .scalar_roots - .push((0, ScalarExpr::PromqlScalarFromVector(scan))); + .push((1, ScalarExpr::Literal(ScalarValue::Float64(1.0)))); let err = check_contract(&output, "stub", 1).unwrap_err(); assert!(matches!( err, - OptimizeError::ContractViolation { pass: "stub", .. } + OptimizeError::InvalidPlan { + pass: "stub", + error: PhysicalASAPDAGValidationError::ScalarRootPosition(1) + } )); } - /// A scalar plan read from `root`, keyed by `entry_index`. - fn scalar_plan(entry_index: usize, root: &Rc) -> QueryPlan { - QueryPlan { - entry_index, - root: Rc::clone(root), - } - } - - /// `scalar_plans` holds one plan per distinct reference of each scalar - /// root, in order and keyed by that root's entry: a plan out of order, - /// under another entry, or read by no scalar root is rejected. - #[test] - fn contract_checks_scalar_plan_order_entry_and_count() { - use asap_types::ir::operator::non_asap::NonASAPOp; - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::scalar::ArithmeticOpKind; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{ExprSemantics, Operator, ScalarExpr}; - let series = |metric: &str| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![Field::plain("ts", DataType::Timestamp, false)], - 0, - vec![], - ), - })) - .unwrap() - }; - let (a, b) = (series("a"), series("b")); - let read = |node: &Rc| ScalarExpr::PromqlScalarFromVector(Rc::clone(node)); - let combine = |left, right| ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Sub, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - }; - let with_plans = |plans: Vec| { - let mut output = PlanOutput::new(Vec::new()); - // `a - b`, then `a - a`: one plan per distinct reference. - output.scalar_roots = vec![ - (0, combine(read(&a), read(&b))), - (1, combine(read(&a), read(&a))), - ]; - output.scalar_plans = plans; - check_contract(&output, "stub", 2) - }; - with_plans(vec![ - scalar_plan(0, &a), - scalar_plan(0, &b), - scalar_plan(1, &a), - ]) - .unwrap(); - for plans in [ - // Wrong order. - vec![scalar_plan(0, &b), scalar_plan(0, &a), scalar_plan(1, &a)], - // Wrong entry key. - vec![scalar_plan(0, &a), scalar_plan(1, &b), scalar_plan(1, &a)], - // An extra plan. - vec![ - scalar_plan(0, &a), - scalar_plan(0, &b), - scalar_plan(1, &a), - scalar_plan(1, &a), - ], - ] { - assert!(matches!( - with_plans(plans), - Err(OptimizeError::ContractViolation { pass: "stub", .. }) - )); - } - } - /// The matching count passes. #[test] fn contract_accepts_an_empty_workload() { - let output = PlanOutput::new(Vec::new()); - assert!(check_contract(&output, "stub", 0).is_ok()); + assert!(check_contract(&empty_plan(), "stub", 0).is_ok()); } /// Registering a name twice fails instead of overwriting, so a comparison diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 64dba4aa0..c181bed92 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -1,33 +1,25 @@ //! The facade's orchestration of the planning stages (#1047): Stage 0 -//! (the memo of logical groups), Pass 1, Pass 2, Stage 2, Stage 3, and -//! finally the selected plan built through Stage 2's realizer. No stage -//! calls another, and none has a deadline: each runs to completion. +//! (the memo of logical groups), Pass 1, Pass 2, Stage 2 and Stage 3, which +//! builds the selected plan. No stage calls another, and none has a +//! deadline: each runs to completion. use asap_logical_asap_optimization::memo_of_queries; -use std::collections::HashSet; use std::rc::Rc; use std::time::Instant; use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; use asap_logical_asap_optimization::{pass1_memo, Pass1Error, PerComputationCandidateError}; -use asap_physical_asap_optimization::{ - generate_physical_asap_memo_candidates, Stage2AssignmentRealizer, -}; -use asap_plan_selection::{select_plan, ErrorSelectingPlan, SearchReport, Selection}; +use asap_physical_asap_optimization::generate_physical_asap_memo_candidates; +use asap_plan_selection::{select_plan, ErrorSelectingPlan, Selection}; use asap_types::deployment_capabilities::DeploymentCapabilitiesError; +use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::schema::schema_support::with_promql_series_identity; -use asap_types::ir::{OperatorNode, QueryRoot}; -use asap_types::stages::{ - AssignmentRealizer, LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates, - PhysicalCandidateError, RealizedPlan, -}; -use asap_types::workload::bounds::with_confirmed_weights_at; +use asap_types::ir::QueryRoot; +use asap_types::stages::{LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates}; use asap_types::workload::{DataWorkload, QueryLanguage, QueryWorkloadEntry}; -use crate::pass::{ - OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels, QueryPlan, -}; +use crate::pass::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput, PlanningModels}; /// How the facade searches. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -52,9 +44,6 @@ pub enum SelectionError { /// Stage 3 selected no plan. #[error("plan selection: {0}")] Selection(ErrorSelectingPlan), - /// Stage 2 could not build the plan Stage 3 selected. - #[error("building the selected plan: {0}")] - Realization(PhysicalCandidateError), /// The run stopped before plan selection (`asap_planner --run-until`). #[error("plan selection did not run")] NotRun, @@ -74,8 +63,6 @@ pub struct PlanningReport { pub pass2_ms: f64, pub stage2_ms: f64, pub stage3_ms: f64, - /// Stage 3's search report, when it selected a plan. - pub search_report: Option, } /// Every stage's output for one workload ([`run_planning`]). @@ -90,13 +77,8 @@ pub struct PlanningRun { /// Stage 2's output, when it ran: the catalog `selection`'s candidate /// ids index. pub physical_asap_candidates: Option, - /// Stage 3's selection. - pub selection: Result, - /// The selected plan as Stage 2 built it, with its tumbling-window retention. - pub selected_physical_plan: Option, - /// The plan to run: the selected plan's roots with the weight proofs - /// Stage 3 made applied. - pub selected_plan: Option>, + /// Stage 3's plan and its report of the selection. + pub selection: Result<(PhysicalASAPDAG, Selection), SelectionError>, pub report: PlanningReport, } @@ -114,21 +96,18 @@ pub fn run_planning( for root in &roots { root.validate_structure()?; } - // Stage 0: the memo of logical groups. - let initial = memo_of_queries(&roots)?; - // Pass 1: the per-computation candidates the deployment can run and - // some query can use. + // Stage 0: the memo of logical groups, each query with its accuracy + // target, or none when accuracy is not enforced. let accuracy_targets: Vec<_> = entries .iter() - .map(|entry| entry.requirements.accuracy.clone()) + .map(|entry| { + (!models.ignore_accuracy_guarantees).then(|| entry.requirements.accuracy.clone()) + }) .collect(); - let accuracy = models.accuracy_check_inputs(); + let initial = memo_of_queries(&roots, &accuracy_targets, data)?; + // Pass 1: the per-computation candidates the deployment can run. let pass1 = pass1_memo( initial, - &accuracy_targets, - accuracy.model, - accuracy.evidence, - data, models .deployment_capabilities .supported_summaries @@ -144,16 +123,8 @@ pub fn run_planning( ..PlanningReport::default() }; let pass1_memo = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); - let realizer = Stage2AssignmentRealizer { - original_roots: &roots, - entries, - data, - query_time_results_retained: models.deployment_capabilities.query_time_results_retained, - }; let mut logical_asap_candidates = None; let mut physical_asap_candidates = None; - let mut selected_physical_plan = None; - let mut selected_plan = None; let selection = (|| { models .deployment_capabilities @@ -161,10 +132,9 @@ pub fn run_planning( .map_err(SelectionError::InvalidDeploymentCapabilities)?; let pass1 = pass1.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?; - // Pass 2: local window composition over every legal - // tumbling-window width, then the sharing rules. + // Pass 2: the reuse rules, readout requirements and logical edges. let started = Instant::now(); - let logical = generate_logical_asap_memo_candidates(pass1, entries); + let logical = generate_logical_asap_memo_candidates(pass1, entries, models.evidence); report.pass2_ms = elapsed_ms(started); let logical = logical_asap_candidates.insert( logical.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?, @@ -184,44 +154,17 @@ pub fn run_planning( let physical = physical_asap_candidates.insert(physical); report.physical_candidates = physical.physical_candidates.len(); report.stage2_pruned_candidates = physical.pruned_candidates.len(); - // Stage 3. + // Stage 3: the selection and the plan it builds. let started = Instant::now(); - let selection = select_plan( - physical, - entries, - data, - &models.accuracy_check_inputs(), - models.cost_inputs, - &realizer, - ); + let selection = select_plan(physical, entries, data, models.cost_inputs); report.stage3_ms = elapsed_ms(started); - let selection = selection.map_err(SelectionError::Selection)?; - report.search_report = Some(selection.search_report.clone()); - // The selected plan, as Stage 2 builds it. - let realized = realizer - .realize(physical, &selection.selected_dag) - .map_err(SelectionError::Realization)?; - let proven: HashSet<*const OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - selected_plan = Some(with_confirmed_weights_at( - &realized.plan.roots, - data, - &|node| proven.contains(&Rc::as_ptr(node)), - )); - selected_physical_plan = Some(realized); - Ok(selection) + selection.map_err(SelectionError::Selection) })(); Ok(PlanningRun { pass1_memo, logical_asap_candidates, physical_asap_candidates, selection, - selected_physical_plan, - selected_plan, report, }) } @@ -235,8 +178,8 @@ fn elapsed_ms(started: Instant) -> f64 { /// identity as a column (the row representation per-series state needs at /// runtime; a root with no such representation is planned over its labels /// alone), through [`run_planning`] with every entry's accuracy target. The -/// output is the selected plan's roots, with the weight proofs plan -/// selection made applied, and its tumbling-window retention. +/// output is the plan Stage 3 built; its `Selection` report is +/// [`run_planning`]'s. #[derive(Debug, Default, Clone, Copy)] pub struct Stage1ToStage3Pass; @@ -277,37 +220,8 @@ impl OptimizationPass for Stage1ToStage3Pass { input.models, PlanSearchSettings::default(), )?; - let selection = run.selection?; - let roots = run.selected_plan.expect("a selection has a plan"); - let mut output = PlanOutput::new(Vec::new()); - for (entry_index, root) in roots.iter().enumerate() { - match root { - QueryRoot::Operator(node) => output.plans.push(QueryPlan { - entry_index, - root: Rc::clone(node), - }), - QueryRoot::Scalar(expr) => { - // One plan per distinct operator the scalar reads. - let mut refs: Vec<&Rc> = Vec::new(); - for node in expr.operator_refs() { - if !refs.iter().any(|r| Rc::ptr_eq(r, node)) { - refs.push(node); - } - } - output - .scalar_plans - .extend(refs.into_iter().map(|node| QueryPlan { - entry_index, - root: Rc::clone(node), - })); - output.scalar_roots.push((entry_index, expr.clone())); - } - } - } - if let Some(realized) = run.selected_physical_plan { - output.tumbling_window_retention_ms = realized.tumbling_window_retention_ms; - } - output.selection = Some(selection); + let (plan, _selection) = run.selection?; + let output = PlanOutput::new(plan); Ok(output) } } diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 2dcf278b9..44d0f4afb 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -1,8 +1,6 @@ //! End-to-end coverage for the library facade (#429) and for plugging a //! different optimization pass into it (#430). -use std::rc::Rc; - use asap_frontend_sql::{lower_sql_dialect, SqlCatalog}; use asap_planner::PlanningModels; use asap_planner::{e2e_plan, FrontendInput, PlanError, UserInput, UserInputError}; @@ -102,12 +100,12 @@ async fn plans_every_query_in_entry_order() { ); let output = e2e_plan(input).await.expect("workload plans"); - assert_eq!(output.plans.len(), 2); - assert_eq!(output.entry_indices(), vec![0, 1]); + assert_eq!(output.plan.roots.len(), 2); + assert!(output.plan.scalar_roots.is_empty()); } -/// The facade selects what the planning stages select over the frontends' -/// roots ([`run_planning`]), and reports that selection. +/// The facade returns the plan the planning stages build over the frontends' +/// roots ([`run_planning`]). #[tokio::test] async fn facade_plans_match_run_planning() { let workload = sql_workload( @@ -134,14 +132,9 @@ async fn facade_plans_match_run_planning() { let mut targets = Vec::new(); for (index, entry) in workload.query_workload.entries().enumerate() { let accuracy = entry.requirements.accuracy.clone(); - let expr = lower_sql_dialect( - &entry.query.0, - &catalog, - SqlDialect::DataFusionSQL, - accuracy.clone(), - ) - .await - .expect("lowers"); + let expr = lower_sql_dialect(&entry.query.0, &catalog, SqlDialect::DataFusionSQL) + .await + .expect("lowers"); roots.push((index, expr.into())); targets.push(Some(accuracy)); } @@ -163,16 +156,8 @@ async fn facade_plans_match_run_planning() { asap_planner::PlanSearchSettings::default(), ) .unwrap(); - let selection = run.selection.as_ref().expect("a candidate is selected"); - // Separate searches have different timings. The facade must preserve - // the chosen execution, cost, and guarantees. - let facade = output.selection.as_ref().expect("facade reports selection"); - assert_eq!(facade.cost, selection.cost); - assert_eq!( - facade.accepted_accuracy_per_query, - selection.accepted_accuracy_per_query - ); - assert_eq!(&output.roots(), run.selected_plan.as_ref().unwrap()); + let (plan, _selection) = run.selection.as_ref().expect("a candidate is selected"); + assert_eq!(&output.plan, plan); } /// A repeating SQL query reaches the optimizer, not only `query_batch` entries. @@ -199,11 +184,10 @@ async fn lowers_repeating_sql_entries_too() { let output = e2e_plan(input).await.expect("workload plans"); assert_eq!( - output.len(), + output.plan.roots.len(), 2, "batch entry and repeating entry both planned" ); - assert_eq!(output.entry_indices(), vec![0, 1]); } /// A caller-supplied pass replaces the shipped algorithm entirely: the trait is @@ -247,10 +231,10 @@ async fn runs_a_caller_supplied_pass_instead_of_the_shipped_one() { assert_eq!(detail, "saw 1 entry/entries"); } -/// The harness catches a pass that mislabels which query a plan belongs to, -/// rather than letting the mislabelled plan reach a deployment. +/// The harness catches a pass whose plan answers fewer queries than the +/// workload has, rather than letting the plan reach a deployment. #[tokio::test] -async fn harness_rejects_a_pass_that_mislabels_entry_indices() { +async fn harness_rejects_a_pass_that_drops_a_query() { struct Mangling; impl OptimizationPass for Mangling { @@ -259,9 +243,7 @@ async fn harness_rejects_a_pass_that_mislabels_entry_indices() { } fn optimize(&self, input: OptimizationInput<'_>) -> Result { let mut output = Stage1ToStage3Pass.optimize(input)?; - for plan in output.plans.iter_mut() { - plan.entry_index += 1; - } + output.plan.roots.pop(); Ok(output) } } @@ -309,7 +291,7 @@ async fn rejects_a_frontend_that_does_not_match_the_workload_language() { )); } -/// A repeating PromQL query yields one plan carrying its selected DAG root. +/// A repeating PromQL query yields a plan with one root. #[tokio::test] async fn each_plan_carries_its_selected_root() { let workload = PlanningWorkload { @@ -337,10 +319,7 @@ async fn each_plan_carries_its_selected_root() { let input = UserInput::new(&workload, FrontendInput::Promql, PlanningModels::builtin()); let output = e2e_plan(input).await.expect("workload plans"); - assert_eq!(output.plans.len(), 1); - assert_eq!(output.plans[0].entry_index, 0); - let _: &Rc<_> = &output.plans[0].root; - assert_eq!(output.operator_roots().len(), 1); + assert_eq!(output.plan.roots.len(), 1); } /// Scalar-only and mixed workloads preserve entry bindings without wrapper nodes. @@ -373,25 +352,15 @@ async fn scalar_roots_survive_planning_in_workload_order() { )) .await .unwrap(); - assert_eq!( - output.entry_indices(), - (0..queries.len()).collect::>() - ); - assert!(matches!( - output.roots()[0], - asap_types::ir::QueryRoot::Scalar(_) - )); - assert_eq!(output.roots().len(), queries.len()); + let roots = output.plan.query_roots().unwrap(); + assert!(matches!(roots[0], asap_types::ir::QueryRoot::Scalar(_))); + assert_eq!(roots.len(), queries.len()); if queries.len() == 3 { - assert_eq!(output.plans[0].entry_index, 1); - let asap_types::ir::QueryRoot::Scalar(expr) = &output.roots()[2] else { + assert!(matches!(roots[1], asap_types::ir::QueryRoot::Operator(_))); + let asap_types::ir::QueryRoot::Scalar(expr) = &roots[2] else { panic!() }; assert_eq!(expr.operator_refs().len(), 1); - assert!(Rc::ptr_eq( - expr.operator_refs()[0], - &output.scalar_plans[0].root - )); } } } diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 09ae0c05b..f345ee885 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -332,15 +332,19 @@ fn p99_avg_workload(values_non_negative: bool) -> PlanningWorkload { "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 10000.0}}}}, "sources": {{"data": {{"series_count": 1000, "metric_type": "gauge", - "sample_interval": 100, "values_non_negative": {values_non_negative}}}}}}}}}"# + "sample_interval": 100, "column_signs": {{"value": "{sign}"}}}}}}}}}}"#, + sign = if values_non_negative { + "non_negative" + } else { + "unknown" + } )) .unwrap() } /// The MVP benchmark's p99 query over an undeclared gauge, which has no /// valid plan with the built-in model, gets one with -/// `ignore_accuracy_guarantees`, and the report marks its output as -/// accepted without a guarantee. +/// `ignore_accuracy_guarantees`. #[tokio::test] async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { let capabilities = DeploymentCapabilities { @@ -358,8 +362,7 @@ async fn ignoring_accuracy_guarantees_plans_the_undeclared_p99_query() { )) .await .expect("a plan is selected"); - let report = report.as_ref().expect("Stage 3's report"); - assert_eq!(report.queries_accepted_without_guarantee, vec![0]); + report.as_ref().expect("the plan's cost report"); assert!( plan.raw_data_retention().unwrap().is_empty(), "no raw samples read" diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index b47d1997e..0bdb17910 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -1,5 +1,5 @@ //! Structurally identical summary producers chosen by different queries are -//! shared: one `Rc` across their plans, planned by Pass 2 +//! shared: one node of the plan across their roots, planned by Pass 2 //! (ASAP-aware CSE). Some older cost-selection tests stay ignored: with the built-in cost model, //! plan selection still selects an exact plan for at least one of their //! queries (#580). @@ -11,7 +11,8 @@ use asap_frontend_promql::promql_metric_names; use asap_frontend_sql::SqlCatalog; use asap_planner::{e2e_plan, FrontendInput, UserInput}; -use asap_planner::{PlanOutput, PlanningModels, QueryPlan}; +use asap_planner::{PlanOutput, PlanningModels}; +use asap_types::ir::QueryRoot; use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::{DataType, Field, Schema}; @@ -120,9 +121,24 @@ async fn plan_sql(queries: &[&str]) -> PlanOutput { e2e_plan(input).await.expect("workload plans") } -/// Every summary state (`SummaryAgg`) each plan reaches, in traversal order. -fn plan_states(plan: &QueryPlan) -> Vec> { - OperatorNode::reachable(&plan.root) +/// Each operator query's root of `output`'s plan, as IR, in entry order. +fn operator_roots(output: &PlanOutput) -> Vec> { + output + .plan + .query_roots() + .unwrap() + .into_iter() + .filter_map(|root| match root { + QueryRoot::Operator(node) => Some(node), + QueryRoot::Scalar(_) => None, + }) + .collect() +} + +/// Every summary state (`SummaryAgg`) a query's root reaches, in traversal +/// order. +fn plan_states(root: &Rc) -> Vec> { + OperatorNode::reachable(root) .into_iter() .filter(|node| { matches!( @@ -135,12 +151,11 @@ fn plan_states(plan: &QueryPlan) -> Vec> { /// Every summary state each plan reaches; each plan selects at least one. fn states(output: &PlanOutput) -> Vec>> { - output - .plans + operator_roots(output) .iter() - .map(|plan| { - let states = plan_states(plan); - assert!(!states.is_empty(), "{:?}", plan.root); + .map(|root| { + let states = plan_states(root); + assert!(!states.is_empty(), "{root:?}"); states }) .collect() @@ -158,8 +173,8 @@ fn same_states(states: &[Vec>]) -> bool { /// The summary states a consumer would run, deduplicated by pointer. fn unique_deployments(output: &PlanOutput) -> usize { let mut seen: Vec<*const OperatorNode> = Vec::new(); - for plan in &output.plans { - for state in plan_states(plan) { + for root in operator_roots(output) { + for state in plan_states(&root) { let ptr = Rc::as_ptr(&state); if !seen.contains(&ptr) { seen.push(ptr); @@ -203,12 +218,10 @@ async fn sql_p50_and_p99_share_one_producer() { let output = plan_sql(&[p50, p99]).await; assert!(same_states(&states(&output))); assert_eq!(unique_deployments(&output), 1); - let names: Vec<_> = output - .plans + let names: Vec<_> = operator_roots(&output) .iter() - .map(|plan| { - plan.root - .schema + .map(|root| { + root.schema .fields .iter() .map(|field| field.name.clone()) @@ -241,7 +254,7 @@ async fn sql_p50_and_p99_share_one_producer() { /// Front ends build structurally identical references as separate nodes. /// The selected graph may reuse a producer or choose separate physical -/// candidates. Every distinct selected reference must be exported exactly once. +/// candidates; the scalar root reads at least the distinct ones. #[tokio::test] async fn identical_scalar_references_satisfy_the_plan_contract() { for (query, distinct) in [ @@ -252,23 +265,12 @@ async fn identical_scalar_references_satisfy_the_plan_contract() { ), ] { let output = plan_promql(&[(query, 0.01)]).await; - let references: std::collections::HashSet<_> = output.scalar_roots[0] + let references: std::collections::HashSet<_> = output.plan.scalar_roots[0] .1 .operator_refs() .into_iter() - .map(Rc::as_ptr) + .copied() .collect(); - assert_eq!(output.scalar_plans.len(), references.len(), "{query}"); assert!(references.len() >= distinct, "{query}"); - let (_, expr) = &output.scalar_roots[0]; - for node in expr.operator_refs() { - assert!( - output - .scalar_plans - .iter() - .any(|read| Rc::ptr_eq(&read.root, node)), - "{query}" - ); - } } } diff --git a/crates/types/src/ir/materialized_view.rs b/crates/types/src/ir/materialized_view.rs index cb5aa9250..83edc4b99 100644 --- a/crates/types/src/ir/materialized_view.rs +++ b/crates/types/src/ir/materialized_view.rs @@ -70,8 +70,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use super::physical_export::{ - offset_below, PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, - PhysicalASAPNodeId, ScanWindow, + PhysicalASAPDAG, PhysicalASAPDAGNode, PhysicalASAPDAGValidationError, PhysicalASAPNodeId, }; use crate::ir::operator::asap::ASAPOp; use crate::ir::operator::node::{Operator, OperatorNode}; @@ -509,8 +508,8 @@ impl PhysicalASAPDAG { /// `SummaryTumblingWindowChain` (a DAG with its chains unexpanded), its newest tumbling window's, /// `(tumbling_window_width, s mod tumbling_window_width)` with `s` the sum of the offsets of /// the `TimeShift`s below it; for any other node, that of its window - /// ([`Self::view_windows`], which counts a chain below it as all its - /// tumbling windows). A chain and each of its expanded tumbling windows have the same one. + /// (the windows below it, a chain below it counting as all its tumbling + /// windows). A chain and each of its expanded tumbling windows have the same one. /// `None` when the window is unbounded or empty. O(nodes + edges). pub fn refresh_window(&self, node: PhysicalASAPNodeId) -> Option { self.refresh_windows([node])[&node] @@ -522,45 +521,8 @@ impl PhysicalASAPDAG { &self, nodes: impl IntoIterator, ) -> HashMap> { - let payloads = self.payloads_by_id(); - let (chains, others): (Vec<_>, Vec<_>) = nodes.into_iter().partition(|id| { - matches!( - payloads.get(id), - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) - ) - }); - let mut windows: HashMap> = self - .view_windows(others) - .into_iter() - .map(|(id, window)| { - let refresh_window = match window { - ScanWindow::Within { - end_offset_ms, - width_ms, - } => RefreshWindow::of_window(end_offset_ms, width_ms), - ScanWindow::Unbounded => None, - }; - (id, refresh_window) - }) - .collect(); - for chain in chains { - let Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child, - tumbling_window_width_ms, - .. - })) = payloads.get(&chain) - else { - continue; - }; - // The newest tumbling window ends at the offsets of the shifts below the chain. - let end_offset_ms = offset_below(*child, |id| payloads.get(&id).copied()); - let width_ms = i64::try_from(*tumbling_window_width_ms).unwrap_or(0); - windows.insert( - chain, - end_offset_ms.and_then(|offset| RefreshWindow::of_window(offset, width_ms)), - ); - } - windows + let _ = nodes.into_iter(); + todo!("#1207 PR 6: a node's refresh window from the TimeRange and TimeShift nodes above its inputs") } /// Sets `retention_ms` on every materialized view and every query-time @@ -580,7 +542,7 @@ impl PhysicalASAPDAG { /// nodes once keeps them for the largest requirement among those nodes. /// /// A query-time `Scan` keeps its rows for its consumers' window - /// ([`Self::scan_window`]'s `lookback_ms`) plus their longest + /// (how far back they read) plus their longest /// range-evaluation span, and at least the retention of every view /// refreshed on first read from it (which may be refreshed for its /// oldest step); `None` when its window is unbounded. In an installed @@ -594,70 +556,8 @@ impl PhysicalASAPDAG { tumbling_window_retention_ms_by_node: &HashMap, reader_span_ms_by_node: &HashMap, ) -> Result, InstallableDagError> { - let (mut views, scans_read): (Vec, Vec>) = - self.views_with_node_retention()?.into_iter().unzip(); - let mut reach_ms_by_node = HashMap::new(); - let mut position_of = HashMap::new(); - for (position, view) in views.iter_mut().enumerate() { - let reach_ms = reach_ms_of(&view.inputs, &reach_ms_by_node); - let span_ms = match view.inputs.materialized_views.is_empty() { - true => 0, - false => reader_span_ms_by_node - .get(&view.node_id) - .copied() - .unwrap_or(0), - }; - view.retention_ms = tumbling_window_retention_ms_by_node - .get(&view.node_id) - .copied() - .unwrap_or(reach_ms + span_ms); - reach_ms_by_node.insert(view.node_id, reach_ms); - position_of.insert(view.node_id, position); - } - // Views before their inputs: an input stays while every view built - // from it may still be refreshed. - for index in (0..views.len()).rev() { - let view = &views[index]; - let needed_ms = match view.refresh { - MaterializedViewRefresh::OnFirstRead => view.retention_ms, - MaterializedViewRefresh::AtIngestion => reach_ms_by_node[&view.node_id], - }; - for input in view.inputs.materialized_views.clone() { - let retained = &mut views[position_of[&input]].retention_ms; - *retained = (*retained).max(needed_ms); - } - } - let mut retention_ms_by_node: HashMap> = views - .iter() - .map(|view| (view.node_id, Some(view.retention_ms))) - .collect(); - // How long each scan's rows are needed by the views refreshed on - // first read from it. - let mut view_need_ms_by_scan: HashMap = HashMap::new(); - for (view, scans) in views.iter().zip(&scans_read) { - if view.refresh == MaterializedViewRefresh::OnFirstRead { - for scan in scans { - let need_ms = view_need_ms_by_scan.entry(*scan).or_insert(0); - *need_ms = (*need_ms).max(view.retention_ms); - } - } - } - let scan_windows = self.query_time_scan_windows(); - for node in self.nodes.iter().filter(|node| is_query_time_scan(node)) { - let window_ms = scan_windows[&node.id].lookback_ms(); - let retention_ms = window_ms.map(|window_ms| { - let span_ms = reader_span_ms_by_node.get(&node.id).copied().unwrap_or(0); - let views_ms = view_need_ms_by_scan.get(&node.id).copied().unwrap_or(0); - (window_ms + span_ms).max(views_ms) - }); - retention_ms_by_node.insert(node.id, retention_ms); - } - for node in &mut self.nodes { - if let Some(&retention_ms) = retention_ms_by_node.get(&node.id) { - node.retention_ms = retention_ms; - } - } - Ok(views) + let _ = (tumbling_window_retention_ms_by_node, reader_span_ms_by_node); + todo!("#1207 PR 6: each materialized node's retention_ms is its h_v, which Stage 3 sets") } /// For each raw-data source some query-time `Scan` reads, in the order @@ -665,52 +565,10 @@ impl PhysicalASAPDAG { /// scans, `None` when one reads it unbounded: the raw data a runtime /// keeps for the DAG's queries, the raw-data counterpart of each view's /// retention. A source read only at ingestion time is absent. Checks - /// that each scan reading a bounded window ([`Self::scan_window`]) has a + /// that each scan reading a bounded window has a /// retention covering it. pub fn raw_data_retention(&self) -> Result, InstallableDagError> { - let scan_windows = self.query_time_scan_windows(); - let mut retention: Vec = Vec::new(); - for node in self.nodes.iter().filter(|node| is_query_time_scan(node)) { - let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.payload else { - continue; - }; - let window_ms = scan_windows[&node.id].lookback_ms(); - let lookback_ms = match (window_ms, node.retention_ms) { - (None, _) => None, - (Some(window_ms), Some(kept_ms)) if kept_ms >= window_ms => Some(kept_ms), - (Some(_), Some(_)) => { - return Err(InstallableDagError::ScanRetentionTooShort { node_id: node.id }) - } - (Some(_), None) => { - return Err(InstallableDagError::ScanWithoutRetention { node_id: node.id }) - } - }; - match retention - .iter_mut() - .find(|retained| &retained.source == source) - { - Some(retained) => { - retained.lookback_ms = - retained.lookback_ms.zip(lookback_ms).map(|(a, b)| a.max(b)) - } - None => retention.push(RawDataRetention { - source: source.clone(), - lookback_ms, - }), - } - } - Ok(retention) - } - - /// [`Self::scan_window`] of each query-time `Scan`, with no chain materialized, - /// by id. - fn query_time_scan_windows(&self) -> HashMap { - let scans = self - .nodes - .iter() - .filter(|node| is_query_time_scan(node)) - .map(|node| node.id); - self.scan_windows(scans, &HashSet::new()) + todo!("#1207 PR 6: each source's retention from its Scan nodes' retention_ms") } /// Checks that every raw-data window the DAG reads (each `TimeRange`, @@ -743,41 +601,20 @@ impl PhysicalASAPDAG { }; let mut derived = Vec::new(); let mut coverage_by_node: HashMap = HashMap::new(); - // The window each view summarizes ([`PhysicalASAPDAG::view_window`]): - // its raw-data windows and those of the views it reads, unioned. - let mut window_by_view: HashMap = HashMap::new(); for &id in index.children_first.iter().filter(|id| views.contains(id)) { let node = index.nodes[&id]; let (inputs, scans) = index.inputs_of_view(id, &views)?; - let read_windows: Vec = inputs - .materialized_views - .iter() - .filter_map(|read| window_by_view.get(read).copied()) - .collect(); - if let Some(window) = inputs - .raw_data_windows - .iter() - .map(|window| ScanWindow::Within { - end_offset_ms: window.end_offset_ms, - width_ms: window.width_ms, - }) - .chain(read_windows.iter().copied()) - .reduce(ScanWindow::union) - { - window_by_view.insert(id, window); - } let refreshed_at_ingestion = node.output_state.timing == ExecutionTiming::IngestionTime; if refreshed_at_ingestion { // An ingestion run reads each view for the window ending at // its end, so a view read must end where the raw windows do. + if !inputs.materialized_views.is_empty() { + todo!("#1207 PR 6: the window a materialized view it reads ends at, from that node's time constraints") + } let mut end_offsets_ms: Vec = inputs .raw_data_windows .iter() .map(|window| window.end_offset_ms) - .chain(read_windows.iter().filter_map(|window| match window { - ScanWindow::Within { end_offset_ms, .. } => Some(*end_offset_ms), - ScanWindow::Unbounded => None, - })) .collect(); end_offsets_ms.sort_unstable(); end_offsets_ms.dedup(); @@ -844,8 +681,8 @@ fn reach_ms_of( /// The nodes of a DAG by id, each node's inputs in edge order, and two /// facts every raw-data window check reads, computed in one pass each over -/// the DAG (O(nodes + edges)). `scan_window` counts the windows of a `Scan` -/// for pricing; this walks down from each window to check its shape. +/// the DAG (O(nodes + edges)); it walks down from each window to check its +/// shape. struct ViewIndex<'a> { dag: &'a PhysicalASAPDAG, nodes: HashMap, diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 4bde9c1b2..799ce1ade 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -1,14 +1,13 @@ //! The pre-ASAP aggregation-intent vocabulary — "what to compute, not how". //! -//! This IR carries intent ("compute a quantile to ε=0.01 accuracy"); the -//! choice between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is a +//! This IR carries intent ("compute a quantile"); an accuracy target belongs +//! to the query, not to a measure. The choice between `HashAgg` / `SortAgg` / `SketchAgg(KLL{k=200})` is a //! post-ASAP cost-aware decision, not encoded here. //! //! `AggIntent::TopK` is a first-class *intent*: "the k most frequent keys by -//! value, to accuracy ε." Like `Quantile`, the exact-vs-approximate -//! realisation — an exact heap / sort+limit when `accuracy: Exact`, a -//! heavy-hitter sketch when approximate — is a post-ASAP cost-aware decision, -//! not encoded here. The semantic distinction that *is* made at lowering is +//! value." Like `Quantile`, the exact-vs-approximate realisation — an exact +//! heap / sort+limit or a heavy-hitter sketch — is a post-ASAP cost-aware +//! decision, not encoded here. The semantic distinction that *is* made at lowering is //! intent vs operator: a heavy-hitter aggregate becomes `TopK`, whereas a //! generic `ORDER BY value LIMIT k` stays as the `NonASAPOp::Sort + Limit` //! operator pair. @@ -16,14 +15,13 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use crate::ir::properties::AccuracyTarget; use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType}; /// "What to compute" — the vocabulary the planner pivots on. /// /// Grouping for `TopK` rides on the enclosing `NonASAPOp::Aggregate`'s `reduction` /// (positional `ColumnId`s), like every other aggregate; the intent itself -/// carries only `k` + the accuracy target. +/// carries only `k`. /// /// The single-column reducers (`Sum` / `Min` / `Max` / `Avg` / `StdDev` / /// `Variance` / `Quantile`) carry `col: Option` — the input @@ -45,9 +43,7 @@ use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType}; #[serde(bound(serialize = "C: Serialize", deserialize = "C: Deserialize<'de>"))] pub enum AggIntent { // ── Data-model-agnostic ────────────────────────────────────────────── - Count { - accuracy: AccuracyTarget, - }, + Count, Sum { col: Option, }, @@ -85,16 +81,14 @@ pub enum AggIntent { col: Option, #[serde(with = "crate::serde_f64")] q: f64, - accuracy: AccuracyTarget, }, - /// Heavy-hitter top-k to the given accuracy. The group-by keys live on + /// Heavy-hitter top-k. The group-by keys live on /// the enclosing `Aggregate.by`. // // Unlike `Quantile`/`Cardinality`, `TopK` ranks by the *aggregate output* // rather than a base column, so it carries no `col` — see #13 / #25. TopK { k: usize, - accuracy: AccuracyTarget, }, /// Distinct count over `cols`, in argument order. One column is SQL /// `COUNT(DISTINCT col)`; several count distinct *tuples* @@ -103,17 +97,14 @@ pub enum AggIntent { /// and `distinct_over_time` leave it so. Cardinality { cols: Vec, - accuracy: AccuracyTarget, }, /// L2 norm of the frequency vector of distinct input values. FrequencyL2 { col: Option, - accuracy: AccuracyTarget, }, /// Shannon entropy in bits of the frequency distribution of input values. FrequencyEntropy { col: Option, - accuracy: AccuracyTarget, }, // ── Time-series streaming derivatives ──────────────────────────────── @@ -491,7 +482,7 @@ impl AggIntent { /// locate it without an alias lookup. pub(crate) fn output_column(&self, input: &Field) -> Field { match self { - AggIntent::Count { .. } => col("count", DataType::Int64, false), + AggIntent::Count => col("count", DataType::Int64, false), AggIntent::Sum { .. } => col("sum", input.dtype.clone(), false), AggIntent::Min { .. } => col("min", input.dtype.clone(), input.nullable), AggIntent::Max { .. } => col("max", input.dtype.clone(), input.nullable), @@ -613,7 +604,7 @@ pub mod topk { impl Ranking { pub fn from_aggregate(aggregate: &AggIntent) -> Self { match aggregate { - AggIntent::Count { .. } => Self::Frequency, + AggIntent::Count => Self::Frequency, AggIntent::Sum { .. } => Self::WeightedSum, _ => Self::NonAdditive, } @@ -633,7 +624,7 @@ pub mod topk { pub fn agg_has_exact_per_entity_kernel(op: &AggIntent) -> bool { matches!( op, - AggIntent::Count { .. } + AggIntent::Count | AggIntent::Sum { .. } | AggIntent::Avg { .. } | AggIntent::Min { .. } @@ -648,19 +639,12 @@ pub fn agg_has_exact_per_entity_kernel(op: &AggIntent) -> bool { /// Default `Cardinality` intent over the sample value — HLL standard error at /// precision p=14. pub fn default_cardinality() -> AggIntent { - AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::max_error(1.04 / ((1u64 << 14) as f64).sqrt()), - } + AggIntent::Cardinality { cols: vec![] } } /// Default `Quantile` intent over the sample value at φ = `q`, `accuracy = ε 0.01`. pub fn default_quantile(q: f64) -> AggIntent { - AggIntent::Quantile { - col: None, - q, - accuracy: AccuracyTarget::max_error(0.01), - } + AggIntent::Quantile { col: None, q } } #[cfg(test)] @@ -689,10 +673,7 @@ mod tests { // output shape as the one-column form — the count of distinct tuples. #[test] fn distinct_tuple_cardinality_contract() { - let intent = AggIntent::Cardinality { - cols: vec![2, 5], - accuracy: AccuracyTarget::max_error(0.01), - }; + let intent = AggIntent::Cardinality { cols: vec![2, 5] }; assert_eq!(intent.input_cols(), vec![2, 5]); let output = intent.output_column(&c("x", DataType::Float64)); assert_eq!(output.name, "cardinality"); @@ -702,14 +683,7 @@ mod tests { #[test] fn output_column_names_are_intent_keyed() { let v = c("value", DataType::Float64); - assert_eq!( - AggIntent::::Count { - accuracy: AccuracyTarget::exact() - } - .output_column(&v) - .name, - "count" - ); + assert_eq!(AggIntent::::Count.output_column(&v).name, "count"); assert_eq!( AggIntent::::Sum { col: None } .output_column(&v) @@ -717,13 +691,9 @@ mod tests { "sum" ); assert_eq!( - AggIntent::::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::max_error(0.01) - } - .output_column(&v) - .name, + AggIntent::::Quantile { col: None, q: 0.99 } + .output_column(&v) + .name, "quantile_0_99" ); } @@ -752,12 +722,7 @@ mod tests { #[test] fn topk_ranking_classifies_aggregate_inputs() { use topk::Ranking::*; - assert_eq!( - topk::Ranking::from_aggregate(&AggIntent::Count { - accuracy: AccuracyTarget::exact() - }), - Frequency - ); + assert_eq!(topk::Ranking::from_aggregate(&AggIntent::Count), Frequency); assert_eq!( topk::Ranking::from_aggregate(&AggIntent::Sum { col: None }), WeightedSum @@ -785,38 +750,20 @@ mod tests { .is_empty(), "empty = PromQL sample value" ); - assert!(AggIntent::::Count { - accuracy: AccuracyTarget::exact() - } - .input_cols() - .is_empty()); + assert!(AggIntent::::Count.input_cols().is_empty()); } #[test] fn agg_intent_serde_roundtrip() { for v in [ - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::max_error(0.01), - }, + AggIntent::Quantile { col: None, q: 0.99 }, AggIntent::Quantile { col: Some(3), q: 0.99, - accuracy: AccuracyTarget::max_error(0.01), - }, - AggIntent::Cardinality { - cols: vec![2], - accuracy: AccuracyTarget::exact(), - }, - AggIntent::Cardinality { - cols: vec![2, 3], - accuracy: AccuracyTarget::exact(), - }, - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::max_error(0.01), }, + AggIntent::Cardinality { cols: vec![2] }, + AggIntent::Cardinality { cols: vec![2, 3] }, + AggIntent::TopK { k: 10 }, ] { let json = serde_json::to_string(&v).unwrap(); let back: AggIntent = serde_json::from_str(&json).unwrap(); @@ -828,9 +775,9 @@ mod tests { #[test] fn cardinality_rejects_legacy_col_payloads() { for payload in [ - r#"{"kind":"cardinality","col":2,"accuracy":{"accuracy_metric":"absolute","max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#, - r#"{"kind":"cardinality","col":null,"accuracy":{"accuracy_metric":"absolute","max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#, - r#"{"kind":"cardinality","col":2,"cols":[1],"accuracy":{"accuracy_metric":"absolute","max_error":0.0,"failure_probability":0.0,"accuracy_scope":"whole_result"}}"#, + r#"{"kind":"cardinality","col":2}"#, + r#"{"kind":"cardinality","col":null}"#, + r#"{"kind":"cardinality","col":2,"cols":[1]}"#, ] { assert!( serde_json::from_str::(payload).is_err(), diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 962c1884f..0adde2639 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -1,5 +1,6 @@ //! ASAP operators: summary-state construction, state operations and evaluations. -//! The summary family, kind/algorithm and parameters are committed here. +//! The summary type (family and algorithm) is an operator argument; θ is a +//! candidate's logical property. use std::rc::Rc; @@ -9,8 +10,8 @@ use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; use crate::ir::operator::operator_properties::Reduction; -use crate::ir::schema::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; -use crate::ir::schema::{ColumnId, DataType, FamilyCapability, Field, FieldDataType, Schema}; +use crate::ir::schema::sketch::{SketchStatistic, SummaryUpdate}; +use crate::ir::schema::{ColumnId, DataType, Field, FieldDataType, Schema, SummaryTypeCapability}; use crate::ir::SchemaDerivationError; /// Why an ASAP operator cannot be used yet. @@ -22,15 +23,14 @@ pub const UNIMPLEMENTED_ASAP_OP: &str = #[serde(bound(deserialize = "C: Deserialize<'de>"))] pub enum ASAPOp> { /// Summary aggregation. Output: grouping columns + one field carrying - /// partial summary state per group, typed `family`. + /// partial summary state per group, typed `summary_type`. SummaryAgg { child: C, - /// Which summary family realizes this aggregation. Never + /// The summary type (family and algorithm) of the state. Never /// `FieldDataType::Plain`. - family: FieldDataType, + summary_type: FieldDataType, input: SummaryUpdate, reduction: Reduction, - grouping: GroupingStrategy, #[serde(default)] filter: Option>, }, @@ -71,8 +71,8 @@ pub enum ASAPOp> { group_by: Option>, }, /// A chain of tumbling windows (concepts doc, section 3, "Chain - /// groups"): the `SummaryAgg` with these `family`, `input`, `reduction`, - /// `grouping` and `filter`, built over each window + /// groups"): the `SummaryAgg` with these `summary_type`, `input`, + /// `reduction` and `filter`, built over each window /// `(k·w, (k+1)·w]` of `child`, `w` = `tumbling_window_width_ms`, on a /// grid anchored at the epoch. Each window is built when it ends. Output: /// one row per window, (window end, keys, state), a state with a time @@ -83,10 +83,9 @@ pub enum ASAPOp> { #[serde(alias = "PaneChain")] SummaryTumblingWindowChain { child: C, - family: FieldDataType, + summary_type: FieldDataType, input: SummaryUpdate, reduction: Reduction, - grouping: GroupingStrategy, #[serde(default)] filter: Option>, #[serde(alias = "pane_width_ms")] @@ -105,7 +104,7 @@ pub enum ASAPOp> { outer: C, inner: C, key: ColumnId, - family: FieldDataType, + summary_type: FieldDataType, }, Extension { child: C, @@ -144,35 +143,31 @@ impl ASAPOp { match self { SummaryAgg { child, - family, + summary_type: family, input, reduction, - grouping, filter, } => SummaryAgg { child: f(child), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: filter .as_ref() .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), }, SummaryTumblingWindowChain { child, - family, + summary_type: family, input, reduction, - grouping, filter, tumbling_window_width_ms, } => SummaryTumblingWindowChain { child: f(child), - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction: reduction.clone(), - grouping: grouping.clone(), filter: filter .as_ref() .map(|p| crate::ir::scalar::Predicate(p.0.map_operator_refs(&mut f))), @@ -210,12 +205,12 @@ impl ASAPOp { outer, inner, key, - family, + summary_type: family, } => SummaryJoin { outer: f(outer), inner: f(inner), key: *key, - family: family.clone(), + summary_type: family.clone(), }, Extension { child, name } => Extension { child: f(child), @@ -250,9 +245,18 @@ impl ASAPOp { /// The summary state this operator produces, if it produces state. pub fn produced_state(&self) -> Option<&FieldDataType> { match self { - ASAPOp::SummaryAgg { family, .. } - | ASAPOp::SummaryTumblingWindowChain { family, .. } - | ASAPOp::SummaryJoin { family, .. } => Some(family), + ASAPOp::SummaryAgg { + summary_type: family, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: family, + .. + } + | ASAPOp::SummaryJoin { + summary_type: family, + .. + } => Some(family), ASAPOp::SummaryMerge { children, .. } => children.first().and_then(|child| { child .schema() @@ -274,13 +278,13 @@ impl ASAPOp { Ok(match self { SummaryAgg { child, - family, + summary_type: family, reduction, .. } | SummaryTumblingWindowChain { child, - family, + summary_type: family, reduction, .. } => summary_agg_schema(child.schema(), family, reduction)?, @@ -381,15 +385,12 @@ impl ASAPOp { ) } }; - let accuracy = crate::ir::properties::AccuracyTarget::exact(); let measure = match evaluation { - PopulationStatistic::Quantile { q } => AggIntent::Quantile { - q: *q, - col: column, - accuracy, - }, + PopulationStatistic::Quantile { q } => { + AggIntent::Quantile { q: *q, col: column } + } PopulationStatistic::Sum => AggIntent::Sum { col: column }, - PopulationStatistic::Count => AggIntent::Count { accuracy }, + PopulationStatistic::Count => AggIntent::Count, PopulationStatistic::Average => AggIntent::Avg { col: column }, PopulationStatistic::TopK { .. } => unreachable!(), }; @@ -492,8 +493,8 @@ impl ASAPOp { query, } => { needs_state(summary_input, "SummaryEstimate")?; - // The family-capability table is the one source of which - // statistic a state's family answers. + // The capability table is the one source of which statistic + // a summary type answers. let states: Vec<_> = summary_input .schema() .fields @@ -502,8 +503,8 @@ impl ASAPOp { .collect(); let valid = match states.as_slice() { [field] => match &field.dtype { - family @ FieldDataType::Sketch(..) => { - FamilyCapability::of(family).answers(query) + summary_type @ FieldDataType::Sketch(..) => { + SummaryTypeCapability::of(summary_type).answers(query) } _ => false, }, @@ -511,7 +512,7 @@ impl ASAPOp { }; if !valid { return Err(SchemaDerivationError::InvalidScalarSignature( - "evaluation does not match its summary family".into(), + "evaluation does not match its summary type".into(), )); } Ok(()) @@ -580,14 +581,14 @@ impl ASAPOp { fn summary_agg_inputs(&self) -> Result<(), SchemaDerivationError> { let (ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, filter, .. } | ASAPOp::SummaryTumblingWindowChain { child, - family, + summary_type: family, input, filter, .. @@ -665,10 +666,9 @@ fn rolled_up_build( }; let ASAPOp::SummaryAgg { child, - family, + summary_type: family, input: update, reduction, - grouping, filter, } = build else { @@ -687,10 +687,9 @@ fn rolled_up_build( Some(( ASAPOp::SummaryAgg { child, - family, + summary_type: family, input: update, reduction, - grouping, filter, }, schema, @@ -698,11 +697,11 @@ fn rolled_up_build( } /// The schema of a `SummaryAgg` (or `SummaryTumblingWindowChain`) of -/// `family` by `reduction` over rows of schema `child`: the reduction's +/// `summary_type` by `reduction` over rows of schema `child`: the reduction's /// output columns, its value column replaced by the state. pub(crate) fn summary_agg_schema( child: &Schema, - family: &FieldDataType, + summary_type: &FieldDataType, reduction: &Reduction, ) -> Result { let mut schema = crate::ir::schema::aggregate_schema::aggregate_output_schema( @@ -719,7 +718,7 @@ pub(crate) fn summary_agg_schema( .map_err(|e| SchemaDerivationError::InvalidScalarSignature(e.to_string()))?, Reduction::Reduce(_) => schema.fields.len() - 1, }; - schema.fields[index] = Field::new("state", family.clone(), false); + schema.fields[index] = Field::new("state", summary_type.clone(), false); Ok(schema) } @@ -732,7 +731,7 @@ fn finalized_schema( ) -> Result { let value_result = if let Some(ASAPOp::SummaryAgg { child: source, - family: FieldDataType::ExactAggregate(kind, _), + summary_type: FieldDataType::ExactAggregate(kind, _), input, reduction, filter, @@ -755,9 +754,7 @@ fn finalized_schema( ExactKind::Sum => Some(AggIntent::Sum { col: column }), ExactKind::Min => Some(AggIntent::Min { col: column }), ExactKind::Max => Some(AggIntent::Max { col: column }), - ExactKind::Count => Some(AggIntent::Count { - accuracy: crate::ir::properties::AccuracyTarget::exact(), - }), + ExactKind::Count => Some(AggIntent::Count), ExactKind::Rate => Some(AggIntent::Rate), ExactKind::IRate => Some(AggIntent::IRate), ExactKind::Increase => Some(AggIntent::Increase), diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 7e5419dcb..a6591e6ad 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -1,9 +1,10 @@ //! Physical ASAP DAG (planner-layering stage 2 output). //! //! The flattened operators of [`super::flat`], plus the lifetime and -//! execution timing (data state) of every node and edge. The realizer -//! (`AssignmentRealizer::realize`) builds it from a `DagOfSelectedCandidates`, -//! each node's timing and lifetime from the `PhysicalCandidate` it realizes. +//! execution timing (data state) of every node and edge: the only plan +//! structure, in memory and, serialized, in the plan file. Stage 3 builds it +//! directly as its last step, one node per selected execution, each with the +//! timing and lifetime of the physical candidate it was built from. //! The deployment executes the plan as given //! (#509 Assumption 8): it does not change lifetime or timing. @@ -21,8 +22,9 @@ use crate::ir::operator::non_asap::NonASAPOp; use crate::ir::properties::execution_data_state::{ExecutionDataState, ExecutionTiming}; use crate::ir::properties::guarantee::AccuracyGuarantee; -use crate::ir::schema::{FieldDataType, Schema, SketchParams}; +use crate::ir::schema::{Schema, SketchParams}; use crate::physical::{Lifetime, PhysicalPropertiesError}; +use crate::stages::PhysicalCandidateId; /// A node's operator, with each child replaced by its node id. pub type PhysicalASAPOperatorPayload = Operator; @@ -67,7 +69,7 @@ pub struct PhysicalASAPDAGNode { /// lifetime the physical properties assignment chose for the sub-DAG this /// node produces. Any node may be materialized (#509 §Materialization); a node /// with no assignment has lifetime `None`. With `output_state.timing` it - /// is the node's [`PhysicalProperties`]. + /// is the node's `PhysicalProperties`. pub lifetime: Lifetime, /// θ of a summary build (`SummaryAgg`, `SummaryTumblingWindowChain`): its /// candidate's `LogicalProperties::parameters`, which the executor builds @@ -87,6 +89,11 @@ pub struct PhysicalASAPDAGNode { /// enumerates lifetimes before Stage 3 knows retention. #[serde(default, skip_serializing_if = "Option::is_none")] pub retention_ms: Option, + /// The physical candidate the node was built from: in-memory provenance + /// for validation and tools, not part of the plan file; `None` for a plan + /// built by a custom pass. + #[serde(skip)] + pub selected_physical_candidate: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -217,130 +224,36 @@ pub enum PhysicalASAPDAGValidationError { Cycle, #[error("physical ASAP node {0:?} is not reachable from the root")] UnreachableNode(PhysicalASAPNodeId), - #[error("summary aggregate node {node:?} output schema does not contain its declared family")] - SummaryFamilySchemaMismatch { node: PhysicalASAPNodeId }, #[error( - "summary aggregate node {node:?} declares grouping inconsistent with its sketch state" + "summary aggregate node {node:?} output schema does not contain its declared summary type" )] - SummaryGroupingMismatch { node: PhysicalASAPNodeId }, + SummaryTypeSchemaMismatch { node: PhysicalASAPNodeId }, } -/// Which rows of a `Scan` its consumers read, back from an evaluation time `T` -/// ([`PhysicalASAPDAG::scan_window`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum ScanWindow { - /// The rows with time in `(T - end_offset_ms - width_ms, T - - /// end_offset_ms]`. - Within { end_offset_ms: i64, width_ms: i64 }, - /// Rows at any time. - Unbounded, -} - -impl ScanWindow { - /// The smallest window holding both. - pub(crate) fn union(self, other: Self) -> Self { - let ( - Self::Within { - end_offset_ms: a_end, - width_ms: a_width, - }, - Self::Within { - end_offset_ms: b_end, - width_ms: b_width, - }, - ) = (self, other) - else { - return Self::Unbounded; - }; - let end_offset_ms = a_end.min(b_end); - let start_back_ms = a_end - .saturating_add(a_width) - .max(b_end.saturating_add(b_width)); - Self::Within { - end_offset_ms, - width_ms: start_back_ms.saturating_sub(end_offset_ms), - } - } - - /// How far back from `T` the oldest row read lies, `end_offset_ms + - /// width_ms` (0 for a window ending after `T` that reaches no further - /// back); `None` when unbounded. - pub fn lookback_ms(self) -> Option { - match self { - Self::Within { - end_offset_ms, - width_ms, - } => Some(u64::try_from(end_offset_ms.saturating_add(width_ms)).unwrap_or(0)), - Self::Unbounded => None, - } - } - - /// `(start, end]` at evaluation time `evaluation_time_ms`; `None` when - /// unbounded. - pub fn bounds_ms(self, evaluation_time_ms: i64) -> Option<(i64, i64)> { - match self { - Self::Within { - end_offset_ms, - width_ms, - } => { - let end = evaluation_time_ms.saturating_sub(end_offset_ms); - Some((end.saturating_sub(width_ms), end)) - } - Self::Unbounded => None, - } - } -} - -/// The rows the `TimeRange` `time_range` keeps, back from the evaluation -/// time `T`: `(T - s - w, T - s]` for a `TimeRange(w)` over `TimeShift`s -/// whose offsets sum to `s`, the window [`PhysicalASAPDAG::scan_window`] -/// counts for it. `Unbounded` when one of those shifts has an `@` anchor, -/// or `time_range` is not a `TimeRange`. `payload_of` gives each node's -/// operator by id; an indexed lookup keeps this O(shifts), so a caller -/// asking for every `TimeRange` of a tumbling-window-expanded DAG stays linear. -pub fn time_range_window<'a>( - time_range: PhysicalASAPNodeId, - payload_of: impl Fn(PhysicalASAPNodeId) -> Option<&'a PhysicalASAPOperatorPayload>, -) -> ScanWindow { - let Some(Operator::NonASAP(NonASAPOp::TimeRange { range, child, .. })) = payload_of(time_range) - else { - return ScanWindow::Unbounded; - }; - let mut end_offset_ms = 0i64; - let mut below = payload_of(*child); - while let Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, child })) = below { - if shift.at.is_some() { - return ScanWindow::Unbounded; - } - end_offset_ms = end_offset_ms.saturating_add(shift.offset_ms); - below = payload_of(*child); - } - ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(range.as_millis()).unwrap_or(i64::MAX), - } -} - -/// How long before the evaluation time `node` is evaluated: the sum of the -/// offsets of the `TimeShift`s from `node` down; `None` when one of them has -/// an `@` anchor or the sum overflows. -pub(crate) fn offset_below<'a>( - node: PhysicalASAPNodeId, - payload_of: impl Fn(PhysicalASAPNodeId) -> Option<&'a PhysicalASAPOperatorPayload>, -) -> Option { - let mut end_offset_ms = 0i64; - let mut below = payload_of(node); - while let Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, child })) = below { - if shift.at.is_some() { - return None; - } - end_offset_ms = end_offset_ms.checked_add(shift.offset_ms)?; - below = payload_of(*child); +impl PhysicalASAPDAG { + /// The plan's query roots as IR, in entry order: each operator root's + /// node, or the standalone scalar query keyed by its entry. Shared nodes + /// stay shared. For tests that inspect a plan as operator trees. + #[cfg(any(test, feature = "test-support"))] + pub fn query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { + use crate::ir::QueryRoot; + let nodes = self.operator_nodes()?; + let mut operator_roots = self.roots.iter(); + let count = self.roots.len() + self.scalar_roots.len(); + Ok((0..count) + .map( + |entry| match self.scalar_roots.iter().find(|(index, _)| *index == entry) { + Some((_, expr)) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |id| Rc::clone(&nodes[id]))) + } + None => QueryRoot::Operator(Rc::clone( + &nodes[operator_roots.next().expect("one operator root per entry")], + )), + }, + ) + .collect()) } - Some(end_offset_ms) -} -impl PhysicalASAPDAG { /// Every node of this DAG as IR, by node id, with its exported /// guarantee (an operator node has no physical properties): what a caller derives /// IR properties of a plan file's node from, such as its @@ -377,148 +290,6 @@ impl PhysicalASAPDAG { Ok(rebuilt) } - /// The rows of the `Scan` `scan` its consumers read, back from the - /// evaluation time `T`: the union of each consumer's window. A consumer - /// reaches the `Scan` through `TimeShift`s without an `@` anchor - /// (offsets summing to `s`) and is - /// - a `TimeRange(w)`, range or instant: `(T - s - w, T - s]`; - /// - a `SummaryTumblingWindowChain` of `n` tumbling windows of width `w`: its whole window `(T - s - - /// n·w, T - s]`, or only its newest tumbling window `(T - s - w, T - s]` for a - /// chain in `materialized_chains` (materialized across evaluations, so each - /// evaluation builds one tumbling window). - /// - /// `Unbounded` when a root, any other consumer, or a shift with an `@` - /// anchor reads the `Scan` or one of those shifts, or nothing reads it. - pub fn scan_window( - &self, - scan: PhysicalASAPNodeId, - materialized_chains: &HashSet, - ) -> ScanWindow { - self.scan_windows([scan], materialized_chains)[&scan] - } - - /// [`Self::scan_window`] of each of `scans`, by id, with the DAG - /// indexed once for all of them: O(nodes + edges + the consumers walked). - pub fn scan_windows( - &self, - scans: impl IntoIterator, - materialized_chains: &HashSet, - ) -> HashMap { - let payloads = self.payloads_by_id(); - let mut consumers: HashMap> = HashMap::new(); - for edge in &self.edges { - consumers - .entry(edge.producer) - .or_default() - .push(edge.consumer); - } - let roots: HashSet = self.roots.iter().copied().collect(); - let window_of = |scan: PhysicalASAPNodeId| { - let mut window: Option = None; - let mut pending = vec![(scan, 0i64)]; - while let Some((id, end_offset_ms)) = pending.pop() { - if roots.contains(&id) { - return ScanWindow::Unbounded; - } - for &consumer in consumers.get(&id).into_iter().flatten() { - let width_ms = match payloads.get(&consumer) { - Some(Operator::NonASAP(NonASAPOp::TimeShift { shift, .. })) - if shift.at.is_none() => - { - pending.push((consumer, end_offset_ms.saturating_add(shift.offset_ms))); - continue; - } - Some(Operator::NonASAP(NonASAPOp::TimeRange { range, .. })) => { - range.as_millis() - } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - tumbling_window_width_ms, - .. - })) if materialized_chains.contains(&consumer) => { - u128::from(*tumbling_window_width_ms) - } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( - "#1207 PR 6: the scan window of a chain read through a TimeRange over it" - ), - _ => return ScanWindow::Unbounded, - }; - let read = ScanWindow::Within { - end_offset_ms, - width_ms: i64::try_from(width_ms).unwrap_or(i64::MAX), - }; - window = Some(window.map_or(read, |window| window.union(read))); - } - } - window.unwrap_or(ScanWindow::Unbounded) - }; - scans - .into_iter() - .map(|scan| (scan, window_of(scan))) - .collect() - } - - /// The interval of data the output of `node` summarizes, back from the - /// evaluation time `T`: the union of the windows - /// ([`time_range_window`]) of every `TimeRange` below it, and of every - /// unexpanded `SummaryTumblingWindowChain` (all its tumbling windows). For an expanded tumbling window, its - /// tumbling window; for a view built from tumbling windows, all of them. `Unbounded` when no - /// `TimeRange` or chain is below it or one of them is unbounded (a whole - /// SQL table, an `@` anchor). O(nodes + edges). - #[cfg(any(test, feature = "test-support"))] - pub fn view_window(&self, node: PhysicalASAPNodeId) -> ScanWindow { - self.view_windows([node])[&node] - } - - /// `Self::view_window` of each of `nodes`, by id, with the DAG indexed - /// once for all of them: O(nodes + edges + the nodes walked below each). - /// It walks each operator's inputs (a valid DAG's edges), so it also - /// answers for a DAG of nodes alone. - pub fn view_windows( - &self, - nodes: impl IntoIterator, - ) -> HashMap { - let payloads = self.payloads_by_id(); - let window_of = |node: PhysicalASAPNodeId| { - let mut seen = HashSet::new(); - let mut pending = vec![node]; - let mut window: Option = None; - while let Some(id) = pending.pop() { - if !seen.insert(id) { - continue; - } - let read = match payloads.get(&id) { - Some(Operator::NonASAP(NonASAPOp::TimeRange { .. })) => { - time_range_window(id, |id| payloads.get(&id).copied()) - } - Some(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) => todo!( - "#1207 PR 6: the window a chain read through a TimeRange over it summarizes" - ), - Some(payload) => { - pending.extend(payload.children().into_iter().copied()); - continue; - } - None => continue, - }; - window = Some(window.map_or(read, |window| window.union(read))); - } - window.unwrap_or(ScanWindow::Unbounded) - }; - nodes - .into_iter() - .map(|node| (node, window_of(node))) - .collect() - } - - /// Each node's operator, by id. - pub(crate) fn payloads_by_id( - &self, - ) -> HashMap { - self.nodes - .iter() - .map(|node| (node.id, &node.payload)) - .collect() - } - pub fn validate(&self) -> Result<(), PhysicalASAPDAGValidationError> { let mut nodes = HashMap::new(); for node in &self.nodes { @@ -553,7 +324,8 @@ impl PhysicalASAPDAG { return Err(PhysicalASAPDAGValidationError::UnexpectedRetention(node.id)); } if let Operator::ASAP(ASAPOp::SummaryAgg { - family, grouping, .. + summary_type: family, + .. }) = &node.payload { let mut found_family = false; @@ -561,20 +333,11 @@ impl PhysicalASAPDAG { if &field.dtype == family { found_family = true; } - if let FieldDataType::Sketch(_, schema_grouping) = &field.dtype { - if schema_grouping != grouping { - return Err(PhysicalASAPDAGValidationError::SummaryGroupingMismatch { - node: node.id, - }); - } - } } if !found_family { - return Err( - PhysicalASAPDAGValidationError::SummaryFamilySchemaMismatch { - node: node.id, - }, - ); + return Err(PhysicalASAPDAGValidationError::SummaryTypeSchemaMismatch { + node: node.id, + }); } } } @@ -784,8 +547,7 @@ pub(crate) fn is_materializable_state_output(operator: &Operator) -> bool /// Test-only: `roots` exported as one DAG with every node at query time and /// not materialized, and no θ (every `parameters` is `None`), with each /// exported node's operator node by node id. Executor tests hand-build DAGs -/// this way; planning exports only through the realizer -/// (`AssignmentRealizer::realize`). +/// this way; planning builds the plan in Stage 3. #[cfg(any(test, feature = "test-support"))] pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( roots: &[crate::ir::QueryRoot], @@ -843,6 +605,7 @@ pub(crate) fn physical_asap_dag_at_query_time_with_node_ids( output_schema: flat_node.schema, guarantee: flat_node.guarantee, retention_ms: None, + selected_physical_candidate: None, }); } let dag = PhysicalASAPDAG { diff --git a/crates/types/src/ir/properties/accuracy.rs b/crates/types/src/ir/properties/accuracy.rs index 45f50b6b9..b3007e3fa 100644 --- a/crates/types/src/ir/properties/accuracy.rs +++ b/crates/types/src/ir/properties/accuracy.rs @@ -318,9 +318,7 @@ pub fn default_metric_for_root(root: &QueryRoot) -> Option { /// intents without a listed default. fn intent_metric(intent: &AggIntent) -> Option { match intent { - AggIntent::Count { .. } | AggIntent::TopK { .. } => { - Some(AccuracyMetric::ABSOLUTE_STREAM_L1) - } + AggIntent::Count | AggIntent::TopK { .. } => Some(AccuracyMetric::ABSOLUTE_STREAM_L1), AggIntent::Cardinality { .. } | AggIntent::FrequencyL2 { .. } => { Some(AccuracyMetric::Relative) } @@ -538,11 +536,7 @@ mod tests { } fn quantile() -> AggIntent { - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::max_error(0.01), - } + AggIntent::Quantile { col: None, q: 0.99 } } /// A target reads and writes as a plain struct: every field but @@ -629,9 +623,7 @@ mod tests { }); let count = node(NonASAPOp::Aggregate { reduction: Reduction::PerEntity, - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::max_error(0.01), - }], + measures: vec![AggIntent::Count], output_names: vec![], filters: vec![], having: None, @@ -652,43 +644,25 @@ mod tests { #[test] fn root_metric_defaults_follow_the_query_intent() { let metric = |intent: AggIntent| default_metric_for_root(&aggregate(vec![intent])).unwrap(); - let exact = AccuracyTarget::exact(); - assert_eq!( - metric(AggIntent::Count { - accuracy: exact.clone() - }), - AccuracyMetric::ABSOLUTE_STREAM_L1 - ); + assert_eq!(metric(AggIntent::Count), AccuracyMetric::ABSOLUTE_STREAM_L1); assert_eq!( metric(AggIntent::Sum { col: None }), AccuracyMetric::ABSOLUTE ); assert_eq!( - metric(AggIntent::TopK { - k: 5, - accuracy: exact.clone() - }), + metric(AggIntent::TopK { k: 5 }), AccuracyMetric::ABSOLUTE_STREAM_L1 ); assert_eq!( - metric(AggIntent::Cardinality { - cols: vec![], - accuracy: exact.clone() - }), + metric(AggIntent::Cardinality { cols: vec![] }), AccuracyMetric::Relative ); assert_eq!( - metric(AggIntent::FrequencyL2 { - col: None, - accuracy: exact.clone() - }), + metric(AggIntent::FrequencyL2 { col: None }), AccuracyMetric::Relative ); assert_eq!( - metric(AggIntent::FrequencyEntropy { - col: None, - accuracy: exact.clone() - }), + metric(AggIntent::FrequencyEntropy { col: None }), AccuracyMetric::ABSOLUTE ); assert_eq!( @@ -711,15 +685,10 @@ mod tests { use crate::ir::operator::operator_properties::JoinKind; use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::{Predicate, ProjectItem}; - let exact = AccuracyTarget::exact(); - let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { - col: None, - accuracy: exact.clone(), - }]) else { + let QueryRoot::Operator(l2) = aggregate(vec![AggIntent::FrequencyL2 { col: None }]) else { unreachable!() }; - let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count { accuracy: exact }]) - else { + let QueryRoot::Operator(count) = aggregate(vec![AggIntent::Count]) else { unreachable!() }; let joined = node(NonASAPOp::Join { diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs index ead2871e1..845e20b4b 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -8,8 +8,8 @@ //! *vocabulary* that fixes that — the typed metric, the symbolic bound and //! failure-probability expressions, the provenance trail, and the typed //! rejection reasons. The *algebra* that composes these (the built-in -//! conservative rules and budget allocation) and the deployment's -//! pluggable `AccuracyModel` trait live one layer up in `asap_accuracy`: +//! conservative rules and budget allocation) and the deployment's measured +//! accuracy (`AccuracyEvidence`) live one layer up in `asap_accuracy`: //! this crate defines the shapes, the planning crates decide. //! //! ## What a guarantee says diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 9a391957a..4813d510b 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -26,8 +26,7 @@ use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; use crate::ir::operator::operator_properties::Reduction; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; -use crate::ir::schema::family_capability::{FamilyCapability, MergeInputRule}; -use crate::ir::schema::sketch::GroupingStrategy; +use crate::ir::schema::summary_type_capability::{MergeInputRule, SummaryTypeCapability}; use crate::ir::schema::{ColumnId, Schema}; use crate::ir::SchemaDerivationError; use crate::stages::{MemoGroupAsInput, MemoGroupId}; @@ -264,12 +263,12 @@ pub enum CoverageError { /// none): bad input. #[error("regrouping merge key {key} is not a group key of its input states")] NotAGroupKey { key: ColumnId }, - /// A regrouping `SummaryMerge` of states whose family does not merge - /// different groups: counter states (`Rate`, `Increase`) merge - /// consecutive windows of one series, a family without a merge merges - /// nothing, and a Hydra's groups share one state. Bad input. - #[error("the inputs' family cannot merge different groups")] - FamilyCannotRegroup, + /// A regrouping `SummaryMerge` of states whose summary type does not + /// merge different groups: counter states (`Rate`, `Increase`) merge + /// consecutive windows of one series, a summary type without a merge + /// merges nothing, and a Hydra's groups share one state. Bad input. + #[error("the inputs' summary type cannot merge different groups")] + SummaryTypeCannotRegroup, /// The build a regrouping `SummaryMerge` stands for does not derive: /// bad input. #[error("the regrouped build does not derive: {reason}")] @@ -362,10 +361,9 @@ fn selected_build<'m>( } let Operator::ASAP(ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction: Reduction::Reduce(by), - grouping, filter, }) = states.operator() else { @@ -402,10 +400,9 @@ fn selected_build<'m>( summary_agg_schema(rows.schema(), &family, &reduction).map_err(regrouped_build)?; Ok(Some(ASAPOp::SummaryAgg { child: rows, - family, + summary_type: family, input, reduction, - grouping, filter: filter.map(|pred| Predicate(pred.0.map_operator_refs(&mut built))), })) } @@ -422,9 +419,10 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result { - FamilyCapability::of(family).merge_inputs - } + Some(Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: family, + .. + })) => SummaryTypeCapability::of(family).merge_inputs, _ => None, } .unwrap_or(MergeInputRule::Disjoint); @@ -466,20 +464,17 @@ fn regrouped( keys: &[ColumnId], ) -> Result { let Some(Operator::ASAP(ASAPOp::SummaryAgg { - family, + summary_type: family, input, reduction, - grouping, filter, .. })) = merged.definition.unlifted_operators.first() else { return Err(CoverageError::NotSummary); }; - if !FamilyCapability::merges_groups(family) - || *grouping != GroupingStrategy::PerSubpopulationInstance - { - return Err(CoverageError::FamilyCannotRegroup); + if !SummaryTypeCapability::merges_groups(family) { + return Err(CoverageError::SummaryTypeCannotRegroup); } let state = first.schema(); let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; @@ -517,10 +512,9 @@ fn regrouped( let mut definition = merged.definition.clone(); definition.unlifted_operators[0] = Operator::ASAP(ASAPOp::SummaryAgg { child: None, - family: family.clone(), + summary_type: family.clone(), input: input.clone(), reduction, - grouping: grouping.clone(), filter: filter.clone(), }); Ok(SummaryCoverage { @@ -554,10 +548,9 @@ fn build_input_schema(state: &I) -> Option { fn of_summary_agg(build: &ASAPOp>) -> Result { let ASAPOp::SummaryAgg { child, - family, + summary_type: family, input, reduction, - grouping, filter, } = build else { @@ -628,10 +621,9 @@ fn of_summary_agg(build: &ASAPOp>) -> Result> for Field { /// What a schema field carries: an ordinary readable value, or the summary / /// exact-accumulator state produced by a `SummaryAgg`. /// -/// Every non-`Plain` variant carries the physical state identity required by -/// that family (`Sketch` additionally carries its grouping layout), so the -/// type system can reject merges of incompatible summaries at plan -/// construction time — a `SummaryMerge` over `Sketch(Kll, …)` and -/// `Sketch(Cms, …)` inputs is a plan-time error, and a `Sketch(…)` can never +/// Every non-`Plain` variant is a summary type: its family (the variant) and +/// its algorithm, so the type system can reject merges of incompatible +/// summaries at plan construction time — a `SummaryMerge` over `Sketch(Kll)` +/// and `Sketch(Cms)` inputs is a plan-time error, and a `Sketch(…)` can never /// be confused for a `SummarySample(…)` even though both are "opaque summary /// state" at a glance. /// @@ -163,12 +162,12 @@ pub enum FieldDataType { /// Exact, mergeable accumulator state (`Sum`/`Count`/`Min`/`Max`/`Rate`/ /// `Increase`). Value consumers require an explicit finalization boundary. ExactAggregate(ExactKind, ExactParams), - /// Approximate sketch state (KLL/CMS/HLL/…), evaluated by a - /// `SummaryEstimate`: its family and grouping layout. The family's - /// parameters θ are not part of the state's type, so states of one family - /// and grouping are one memo group whatever their size; θ is a - /// candidate's `LogicalProperties::parameters` (concepts doc, section 3). - Sketch(SketchAlgorithm, GroupingStrategy), + /// Approximate sketch state (KLL/CMS/HLL/…, and the `Hydra*` algorithms, + /// one structure shared by every subpopulation), evaluated by a + /// `SummaryEstimate`. θ is not part of the type, so states of one + /// algorithm are one memo group whatever their size; θ is a candidate's + /// `LogicalProperties::parameters` (concepts doc, section 1). + Sketch(SketchAlgorithm), /// A summary sample: the subset of input rows a [`SamplingKind`] keeps, /// such as a reservoir. Not raw data samples. Serialized as `Sample`. #[serde(rename = "Sample")] diff --git a/crates/types/src/ir/schema/sketch.rs b/crates/types/src/ir/schema/sketch.rs index b4c4ace79..c94c4fd0f 100644 --- a/crates/types/src/ir/schema/sketch.rs +++ b/crates/types/src/ir/schema/sketch.rs @@ -52,7 +52,7 @@ pub enum ExactParams { /// A specific sketch algorithm — bounded error, sized by its /// [`SketchParams`]. It is `asap_sketchlib`'s family enum; the statistics /// each answers are its row of the family-capability table -/// ([`FamilyCapability`](crate::ir::schema::FamilyCapability)). The planner +/// ([`SummaryTypeCapability`](crate::ir::schema::SummaryTypeCapability)). The planner /// sizes and offers only the families [`SketchParams`] has a variant for. pub use asap_sketchlib::SketchAlgorithm; @@ -60,39 +60,20 @@ pub use asap_sketchlib::SketchAlgorithm; /// `k` at this (`MAX_CACHEABLE_K` in its `sketches::kll`). pub const KLL_MAX_K: u32 = 26_602; -/// Concrete, catalog-validated parameters for a specific [`SketchAlgorithm`] -/// instance. The variant must correspond to the associated `SketchAlgorithm`; -/// mismatches are caught at post-ASAP bind time, before any later, -/// deployment-specific stage ever sees the plan. -/// -/// Equality compares the sketch that is built: a KLL's `failure_probability` -/// and a UnivMon's `distinct_failure_probability` record what it was sized -/// for, not its structure, so two KLLs with the same `k` are equal (they -/// share, and match the same offline evidence), as are two UnivMons of the -/// same shape. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// The structure of a summary of one [`SketchAlgorithm`] (θ): its sizes, +/// from the algorithm's predefined list (`parameter_list`). No field records +/// the error or failure probability a configuration was sized for, so +/// equality compares every field. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum SketchParams { UnivMon { heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8, - /// The failure probability the heap was sized for when this - /// UnivMon answers a distinct count; its relative error follows from - /// it and the heap. `None`: not sized for a distinct count, which - /// then has no guarantee. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - distinct_failure_probability: Option, }, Kll { k: u32, - /// The failure probability `k` was sized for, when below the - /// empirical 99% calibration's 0.01; `None` is that calibration. - /// Sizing never records `Some(δ)` for δ ≥ 0.01; a hand-built one is - /// read as a claim at that δ with the calibration's error bound, and - /// one outside (0, 1) gives no guarantee. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - failure_probability: Option, }, Cms { width: u32, @@ -100,12 +81,6 @@ pub enum SketchParams { }, Hll { precision: u8, - /// The relative error `precision` was sized for (#694): the error - /// bound of its guarantee, at which its failure probability is - /// evaluated. `None`: not sized for a target; its error bound is - /// the relative standard error `1.04/√m`. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - relative_error: Option, }, DDSketch { alpha: f64, @@ -117,12 +92,6 @@ pub enum SketchParams { }, Kmv { k: u32, - /// The relative error `k` was sized for (#694): the error bound of - /// its guarantee, at which its failure probability is evaluated. - /// `None`: not sized for a target; its error bound is the relative - /// standard error `1/√(k-2)`. Ignored by equality. - #[serde(default, skip_serializing_if = "Option::is_none")] - relative_error: Option, }, CountSketch { width: u32, @@ -133,158 +102,11 @@ pub enum SketchParams { depth: u32, heap_size: u32, }, - /// The sizes of a Hydra structure shared across subpopulations - /// ([`GroupingStrategy::SharedMultiSubpopulation`]): θ, like every other - /// family's parameters, not part of the grouping. + /// The sizes of a Hydra structure shared across subpopulations, for the + /// `Hydra*` algorithms. Hydra(HydraParams), } -impl PartialEq for SketchParams { - fn eq(&self, other: &Self) -> bool { - use SketchParams as P; - match (self, other) { - (P::Kll { k: a, .. }, P::Kll { k: b, .. }) => a == b, - ( - P::UnivMon { - heap_size: a0, - sketch_rows: a1, - sketch_cols: a2, - layers: a3, - .. - }, - P::UnivMon { - heap_size: b0, - sketch_rows: b1, - sketch_cols: b2, - layers: b3, - .. - }, - ) => (a0, a1, a2, a3) == (b0, b1, b2, b3), - ( - P::Cms { - width: a0, - depth: a1, - }, - P::Cms { - width: b0, - depth: b1, - }, - ) - | ( - P::CountSketch { - width: a0, - depth: a1, - }, - P::CountSketch { - width: b0, - depth: b1, - }, - ) => (a0, a1) == (b0, b1), - ( - P::CmsWithHeap { - width: a0, - depth: a1, - heap_size: a2, - }, - P::CmsWithHeap { - width: b0, - depth: b1, - heap_size: b2, - }, - ) - | ( - P::CountSketchWithHeap { - width: a0, - depth: a1, - heap_size: a2, - }, - P::CountSketchWithHeap { - width: b0, - depth: b1, - heap_size: b2, - }, - ) => (a0, a1, a2) == (b0, b1, b2), - (P::Hll { precision: a, .. }, P::Hll { precision: b, .. }) => a == b, - (P::DDSketch { alpha: a }, P::DDSketch { alpha: b }) => a == b, - (P::Kmv { k: a, .. }, P::Kmv { k: b, .. }) => a == b, - (P::Hydra(a), P::Hydra(b)) => a == b, - // Different variants. Listed by `self`'s variant, not `_`, so a - // new variant fails to compile here until it is compared above. - (P::UnivMon { .. }, _) - | (P::Kll { .. }, _) - | (P::Cms { .. }, _) - | (P::Hll { .. }, _) - | (P::DDSketch { .. }, _) - | (P::CmsWithHeap { .. }, _) - | (P::Kmv { .. }, _) - | (P::CountSketch { .. }, _) - | (P::CountSketchWithHeap { .. }, _) - | (P::Hydra(_), _) => false, - } - } -} - -/// A committed sketch choice: the concrete [`SketchAlgorithm`] and the -/// [`SketchParams`] realizing it. Sits between -/// [`FieldDataType::Sketch`](crate::ir::schema::FieldDataType::Sketch) -/// (the `Sketch` family as a whole, sibling to `SummarySample`/`Wavelet`/ -/// `StatModel`) and the bare algorithm. Which statistics a state of the -/// kind answers is the family-capability table's -/// ([`FamilyCapability`](crate::ir::schema::FamilyCapability)). -/// -/// Construct through [`SketchKind::new`], which rejects parameters of a -/// different algorithm. See `asap_logical_asap_optimization::summary_candidates` for the -/// `AggIntent -> [SketchAlgorithm]` candidate list. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SketchKind { - algorithm: SketchAlgorithm, - params: SketchParams, -} - -impl SketchKind { - /// The kind of `algorithm` with `params`. Panics when `params` belongs - /// to a different algorithm; invalid committed sketch states cannot be - /// constructed. - pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self { - let valid_params = matches!( - (&algorithm, ¶ms), - (SketchAlgorithm::Kll, SketchParams::Kll { .. }) - | (SketchAlgorithm::UnivMon, SketchParams::UnivMon { .. }) - | (SketchAlgorithm::DDSketch, SketchParams::DDSketch { .. }) - | (SketchAlgorithm::Hll, SketchParams::Hll { .. }) - | (SketchAlgorithm::Kmv, SketchParams::Kmv { .. }) - | (SketchAlgorithm::Cms, SketchParams::Cms { .. }) - | ( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { .. } - ) - | ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { .. } - ) - | ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { .. } - ) - ); - assert!( - valid_params, - "SketchKind parameter mismatch: algorithm={algorithm:?}, params={params:?}" - ); - Self { algorithm, params } - } - - /// The algorithm this kind committed to. - pub fn algorithm(&self) -> &SketchAlgorithm { - &self.algorithm - } - - /// The parameters this kind committed to. - pub fn params(&self) -> &SketchParams { - &self.params - } -} - // ── Sampling summaries ─────────────────────────────────────────────────────── /// A sampling summary family — retains an actual (weighted) subset of input @@ -356,101 +178,20 @@ pub enum StatModelParams { }, } -// ── Grouping strategy: per-subpopulation vs. shared multi-subpopulation -// (Hydra) — issue #256 ────────────────────────────────────────────────── -// -// A grouped aggregate (`GROUP BY city, quantile(...)`) has always implicitly -// built one independent summary instance per distinct `by` key — there was -// no type anywhere expressing that as a *choice* rather than a foregone -// conclusion. Hydra (Manousis et al., VLDB 2022 — see `HydraKind`'s own doc -// for the full citation and which variants are its actual proven -// construction) is the alternative: one shared structure serving every -// subpopulation instead of N independent instances, trading memory/build -// cost against per-subpopulation isolation. -// -// This axis is deliberately modeled here, alongside `SketchKind`/ -// `SamplingKind`/`WaveletKind`/`StatModelKind`, rather than as a new -// `SketchKind` (or `SamplingKind`, etc.) entry: it is orthogonal to *which* -// family/kind answers an intent — any family could in principle grow its own -// per-subpopulation vs. shared-multi-subpopulation variant, so it is a -// second, independent axis, not a member of any one family's own kind -// vocabulary. It is legal only for a grouped aggregate (a non-empty `by`) -// whose family has a Hydra variant (CMS or Count-Sketch); the planner does -// not offer it as a candidate yet (#703). - -/// A shared-multi-subpopulation summary family — one physical structure -/// serving every subpopulation of a grouped aggregate instead of one -/// independent instance per distinct `by` key. Named after Hydra (Manousis, -/// Cheng, Ben Basat, Liu, Sekar. "Enabling Efficient and General -/// Subpopulation Analytics in Multidimensional Data Streams." VLDB 2022). -/// -/// Orthogonal to [`SketchKind`]/[`SamplingKind`]/[`WaveletKind`]/ -/// [`StatModelKind`] the same way [`GroupingStrategy`] as a whole is -/// orthogonal to them. -/// -/// ## Which variants are the paper's own proven construction, and which aren't -/// -/// Hydra's accuracy proof (paper §4.5, Theorem 2) is over one specific -/// construction: hash each subpopulation into one of a shared w×r grid of -/// **linear, mergeable frequency-vector sketches** — the paper's own -/// heavy-hitter substrate is Count-Sketch (§4.3); Count-Min Sketch is the -/// same collision algebra — and bound the noise a colliding subpopulation's -/// estimate picks up from the others sharing its cell. -/// [`HydraKind::HydraCms`]/[`HydraKind::HydraCountSketch`] are exactly that -/// construction over this crate's existing `SketchAlgorithm::Cms`/ -/// `CountSketch`, so the paper's bound applies to them directly. -/// -/// [`HydraKind::HydraKll`] is **not** an instance of that proven -/// construction: KLL is an order-statistics sketch, not a linear frequency -/// vector, and has no analogous "sum the colliding contributions, bound the -/// noise" algebra. The paper is explicit that its own construction cannot -/// serve quantiles at all (§4.3: "A statistic that cannot directly be -/// estimated by Hydra-sketch is quantiles."). `HydraKll` remains available -/// as an explicit experimental IR value, but must not be exposed to -/// semantics-preserving replacement search: no error bound is -/// modeled for it (see [`HydraParams::HydraKll`]'s own doc). Enable automatic -/// selection only alongside an actual proof and error model. -/// -/// Not yet modeled: `HydraUnivMon`. The paper's own named "Hydra-sketch" is -/// really the universal-sketch composition (L layers of Count-Sketch plus a -/// heavy-hitter heap, Theorems 1+2 combined) estimating entropy/L1-norm/ -/// L2-norm/cardinality/frequency-moments as one instance. Standalone UnivMon -/// and its frequency evaluations are represented here, but sharing a Hydra -/// grid across populations still needs its own collision/error contract; -/// standalone support does not establish that contract. -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum HydraKind { - /// Hydra over a KLL-family quantile sketch. See this type's own doc: - /// **not** an instance of the paper's proven construction — the paper - /// excludes quantiles from Hydra-sketch entirely. Kept only as an - /// explicit experimental representation; automatic replacement search - /// must not treat it as equivalent to independent instances. - HydraKll, - /// Hydra over Count-Min Sketch: a direct instance of the paper's proven - /// w×r shared-grid construction (§4.2/§4.5) — CMS is exactly the - /// linear frequency-vector substrate Theorem 2 is proved over. - HydraCms, - /// Hydra over Count-Sketch — the paper's own heavy-hitter substrate - /// (§4.3). A direct instance of the same proven construction as - /// `HydraCms`, with Count-Sketch's balanced/zero-mean-error trade - /// instead of CMS's one-sided bias. - HydraCountSketch, -} - -/// Parameters for a [`HydraKind`] instance. Unlike a plain [`SketchParams`] -/// (sized purely for one instance's own accuracy), a Hydra structure needs -/// both the per-subpopulation accuracy target it emulates *and* how big the -/// one shared structure itself is — the latter is the whole memory/accuracy -/// trade Hydra makes, and has no equivalent for an instance built -/// independently per subpopulation. One variant per [`HydraKind`] because -/// the knobs a "sketch of sketches" needs are specific to the inner -/// sketch's own parameter shape — a `HydraCms` instance is sized in -/// (`width`, `depth`), a `HydraKll` instance in `k`; there is no single knob -/// set general enough to cover every inner sketch type. +// ── Hydra: one structure shared by every subpopulation ───────────────────── + +/// θ of the `Hydra*` algorithms ([`SketchAlgorithm::HydraKll`], +/// [`SketchAlgorithm::HydraCms`], [`SketchAlgorithm::HydraCountSketch`]): one +/// structure shared by every subpopulation of a grouped aggregate (Manousis +/// et al., "Enabling Efficient and General Subpopulation Analytics in +/// Multidimensional Data Streams", VLDB 2022). It holds both the +/// per-subpopulation sketch's sizes and the shared grid's. HydraCms and +/// HydraCountSketch are the paper's proven construction (§4.5, Theorem 2); +/// HydraKll is not (the paper excludes quantiles, §4.3), so no error model +/// covers it. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum HydraParams { - /// See [`HydraKind::HydraKll`]: kept for legality, **no accuracy bound - /// is modeled for this variant**. `k`/`shared_buckets` give a bound + /// **No accuracy bound is modeled for this variant**. `k`/`shared_buckets` give a bound /// sketch a concrete size to build against; unlike `HydraCms`/ /// `HydraCountSketch`'s fields, they are not backed by the paper's /// Theorem 2. @@ -501,10 +242,7 @@ impl HydraParams { /// emulates, which its error bound and per-group state follow. pub fn per_subpopulation(&self) -> SketchParams { match *self { - HydraParams::HydraKll { k, .. } => SketchParams::Kll { - k, - failure_probability: None, - }, + HydraParams::HydraKll { k, .. } => SketchParams::Kll { k }, HydraParams::HydraCms { width, depth, .. } => SketchParams::Cms { width, depth }, HydraParams::HydraCountSketch { width, depth, .. } => { SketchParams::CountSketch { width, depth } @@ -513,45 +251,6 @@ impl HydraParams { } } -/// How a grouped aggregate's summary state is physically instantiated -/// across its `by` subpopulations — orthogonal to *which* -/// `SketchKind`/`SamplingKind`/`WaveletKind`/`StatModelKind` answers the -/// intent (that choice lives alongside it on `FieldDataType`). Lives here, -/// alongside `SketchKind`/`SketchParams` etc., rather than on any of those -/// enums themselves, for exactly the reason explained in this section's -/// module docs above. -/// -/// Carried both on `ASAPOp::SummaryAgg` (where planning consults it) -/// and on sketch-valued `FieldDataType` edges (where it prevents -/// incompatible shared and independent physical states from type-checking -/// as merge-compatible). -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum GroupingStrategy { - /// One independent summary instance per distinct `by` key — today's - /// only (implicit) behavior, and this type's `Default` (below), so - /// every existing caller that never chose this axis explicitly keeps - /// observing exactly the same behavior it always has. - PerSubpopulationInstance, - /// One shared structure serving every subpopulation (Hydra and its - /// per-family variants — see [`HydraKind`]), trading per-subpopulation - /// isolation for shared memory/build cost. - /// - /// Named `SharedMultiSubpopulation`, not `SharedMultiTenant`: this - /// shares across a *query's own* subpopulations/group-by keys, not - /// across tenants in a deployment-isolation sense — a different, and - /// unrelated, kind of "sharing". - /// Its sizes are θ ([`SketchParams::Hydra`]), not part of the grouping. - SharedMultiSubpopulation { kind: HydraKind }, -} - -impl Default for GroupingStrategy { - /// [`GroupingStrategy::PerSubpopulationInstance`] — see that variant's - /// own doc for why this is the only sound default. - fn default() -> Self { - Self::PerSubpopulationInstance - } -} - // ── Sketch evaluation queries ───────────────────────────────────────────────── /// One summary-state update. `item` identifies a keyed item when the family @@ -592,7 +291,7 @@ pub enum WeightDomain { /// records it without reading the data workload; plan selection confirms /// it from the declared source facts /// ([`SourceFacts::metric_type`](crate::workload::SourceFacts::metric_type), - /// [`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative)). + /// [`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). /// Unconfirmed, it counts as not proven. Serialized as /// `samples_of_metric`. #[serde(rename = "samples_of_metric")] @@ -612,44 +311,6 @@ pub enum WeightDomain { }, } -impl WeightDomain { - /// The domain once the declared facts have answered an open requirement. - /// [`RawDataSamplesOfMetric`](Self::RawDataSamplesOfMetric) is proven when the - /// metric is a counter ([`NonNegativeWeightProof::CounterRawDataSamples`]) or else - /// declared non-negative - /// ([`NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples`]); - /// [`CounterDerivativeOfMetric`](Self::CounterDerivativeOfMetric) only - /// when it is a counter - /// ([`NonNegativeWeightProof::ResetAwareCounterDerivative`]). Otherwise - /// unknown. Any other domain is returned unchanged. - pub(crate) fn confirmed( - &self, - is_counter: impl Fn(&str) -> bool, - is_declared_non_negative: impl Fn(&str) -> bool, - ) -> WeightDomain { - match self { - WeightDomain::RawDataSamplesOfMetric { metric } if is_counter(metric) => { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples, - } - } - WeightDomain::RawDataSamplesOfMetric { metric } if is_declared_non_negative(metric) => { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, - } - } - WeightDomain::CounterDerivativeOfMetric { metric } if is_counter(metric) => { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - } - } - WeightDomain::RawDataSamplesOfMetric { .. } - | WeightDomain::CounterDerivativeOfMetric { .. } => WeightDomain::UnknownOrSigned, - other => other.clone(), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum NonNegativeWeightProof { @@ -669,7 +330,7 @@ pub enum NonNegativeWeightProof { #[serde(rename = "counter_samples")] CounterRawDataSamples, /// Raw data samples of a metric declared non-negative - /// ([`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative)), + /// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), /// such as a gauge, or plain sums of them. Plan selection gives it on /// confirming a [`WeightDomain::RawDataSamplesOfMetric`] requirement for a /// metric not declared a counter. Serialized as @@ -760,40 +421,6 @@ mod tests { } } - #[test] - fn sketch_kind_keeps_a_valid_algorithm_and_params_pair() { - let kind = SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ); - assert_eq!(kind.algorithm(), &SketchAlgorithm::Kll); - } - - #[test] - #[should_panic(expected = "SketchKind parameter mismatch")] - fn sketch_kind_rejects_params_from_another_algorithm() { - SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Hll { - precision: 14, - relative_error: None, - }, - ); - } - - #[test] - fn grouping_strategy_default_is_per_subpopulation_instance() { - // Existing/default behavior must stay `PerSubpopulationInstance` — - // this must not change any existing test's observed behavior. - assert_eq!( - GroupingStrategy::default(), - GroupingStrategy::PerSubpopulationInstance - ); - } - /// The raw-data-sample and summary-sample renames keep their JSON names, /// so committed plans and workloads still read. #[test] diff --git a/crates/types/src/ir/schema/family_capability.rs b/crates/types/src/ir/schema/summary_type_capability.rs similarity index 56% rename from crates/types/src/ir/schema/family_capability.rs rename to crates/types/src/ir/schema/summary_type_capability.rs index 960403e36..a53c7f560 100644 --- a/crates/types/src/ir/schema/family_capability.rs +++ b/crates/types/src/ir/schema/summary_type_capability.rs @@ -1,31 +1,31 @@ -//! The generic family-capability table: what each summary family can do, -//! keyed by its [`FieldDataType`]. +//! The summary-type capability table: what each summary type can do, keyed +//! by its [`FieldDataType`] (family and algorithm). //! -//! The IR stays permissive: a `SummaryMerge` over any state family is +//! The IR stays permissive: a `SummaryMerge` over any state type is //! structurally valid. The planner rules and the executor consult this //! table to decide whether a composition is sound. It is the one source of -//! which statistic a family answers: the IR's `SummaryEstimate` validation, -//! the executor's evaluation check and its supported-summary list all read -//! it. One row per family says +//! which statistic a summary type answers: the IR's `SummaryEstimate` +//! validation, the executor's evaluation check and its supported-summary +//! list all read it. One row per summary type says //! how the inputs of a merge must relate ([`MergeInputRule`]) and which //! statistics ([`StatisticKind`]) one state answers, each with the error of //! that readout from a merged state ([`MergedError`]), so one state can //! serve every consumer whose statistic it answers. //! -//! A sketch family's row is `asap_sketchlib`'s -//! ([`SketchAlgorithm::capability`]), read unchanged: the family, +//! A sketch algorithm's row is `asap_sketchlib`'s +//! ([`SketchAlgorithm::capability`]), read unchanged: the algorithm, //! statistic and merged-error vocabulary is the library's. The planner //! writes the rows of the exact accumulators, which the library does not //! have. Which readouts an executor evaluates is its own declaration //! (`DeploymentCapabilities::supported_summaries`), not this table's. -use super::sketch::{ExactKind, GroupingStrategy, HydraKind, SketchAlgorithm, SketchStatistic}; +use super::sketch::{ExactKind, SketchAlgorithm, SketchStatistic}; use crate::ir::schema::FieldDataType; pub use asap_sketchlib::{MergedError, StatisticKind}; -/// What one summary family can do: its row of the family-capability table. +/// What one summary type can do: its row of the capability table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct FamilyCapability { +pub struct SummaryTypeCapability { /// How the inputs of a merge of the family's states must relate; `None` /// when the family has no sound merge. pub merge_inputs: Option, @@ -89,7 +89,7 @@ impl From for MergeInputRule { } #[cfg(any(test, feature = "test-support"))] -impl FamilyCapability { +impl SummaryTypeCapability { /// Each statistic one state of the family answers, with the error of /// that readout from a merged state. pub fn readouts(&self) -> &'static [(StatisticKind, MergedError)] { @@ -97,20 +97,28 @@ impl FamilyCapability { } } -impl FamilyCapability { - /// Whether a rollup may merge different groups' states of `family`: its - /// merge takes disjoint or overlapping inputs, and each group has its own - /// state. Counter states merge consecutive windows of one series; a - /// family without a merge merges nothing; a Hydra's groups share one - /// state. - pub fn merges_groups(family: &FieldDataType) -> bool { +impl SummaryTypeCapability { + /// Whether a rollup may merge different groups' states of + /// `summary_type`: its merge takes disjoint or overlapping inputs, and + /// each group has its own state. Counter states merge consecutive + /// windows of one series; a summary type without a merge merges nothing; + /// a Hydra's groups share one state. + pub fn merges_groups(summary_type: &FieldDataType) -> bool { matches!( - Self::of(family).merge_inputs, + Self::of(summary_type).merge_inputs, Some(MergeInputRule::Disjoint | MergeInputRule::MayOverlap) - ) && !matches!( - family, - FieldDataType::Sketch(_, grouping) - if *grouping != GroupingStrategy::PerSubpopulationInstance + ) && !Self::shares_one_state_across_groups(summary_type) + } + + /// Whether `summary_type` is a Hydra algorithm: one structure shared by + /// every group, not one state per group. + pub fn shares_one_state_across_groups(summary_type: &FieldDataType) -> bool { + use SketchAlgorithm as A; + matches!( + summary_type, + FieldDataType::Sketch( + A::HydraCms | A::HydraCountSketch | A::HydraKll | A::HydraHll | A::HydraUnivMon + ) ) } @@ -135,54 +143,38 @@ impl FamilyCapability { .map(|&(_, error)| error) } - /// The capability of a per-subpopulation state of `algorithm`, whatever - /// its parameters: `asap_sketchlib`'s row of the family, which the IR and - /// a deployment's executor read. - pub fn of_per_instance_sketch(algorithm: &SketchAlgorithm) -> Self { - let row = algorithm.capability(); - Self { - merge_inputs: row.merge_inputs.map(MergeInputRule::from), - readouts: row.readouts, - } - } - - /// The capability of `family`. Families not listed have no capability - /// (they fail closed). - pub fn of(family: &FieldDataType) -> Self { + /// The capability of `summary_type`. Summary types not listed have no + /// capability (they fail closed). + pub fn of(summary_type: &FieldDataType) -> Self { use ExactKind as E; use MergeInputRule::{Disjoint, DisjointInTime, MayOverlap}; - let (merge_inputs, readouts): (_, &'static [(StatisticKind, MergedError)]) = match family { - FieldDataType::ExactAggregate(kind, _) => match kind { - E::Sum | E::Count => (Some(Disjoint), &[]), - E::Min | E::Max => (Some(MayOverlap), &[]), - // The counter accumulator joins time-ordered windows with - // reset handling (`IncreaseAccumulator::merge_pair`). - E::Rate | E::Increase => (Some(DisjointInTime), &[]), - // IRate reads the last two samples of the window, which a - // tumbling window does not keep across its boundary. - E::IRate => (None, &[]), - }, - // A Hydra answers per subpopulation from a shared grid of the - // inner family's cells. HydraKll merges as sketchlib's row says: - // `Hydra::merge` requires the same k and m and otherwise returns - // an error (asap_sketchlib #251). Nothing builds one yet: Pass 1 - // offers no Hydra, and the executor builds only HydraCms - // (`create_hydra_cms`). - FieldDataType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { kind, .. }) => { - return Self::of_per_instance_sketch(&match kind { - HydraKind::HydraCms => SketchAlgorithm::HydraCms, - HydraKind::HydraCountSketch => SketchAlgorithm::HydraCountSketch, - HydraKind::HydraKll => SketchAlgorithm::HydraKll, - }) - } - FieldDataType::Sketch(algorithm, GroupingStrategy::PerSubpopulationInstance) => { - return Self::of_per_instance_sketch(algorithm) - } - FieldDataType::Plain(_) - | FieldDataType::SummarySample(..) - | FieldDataType::Wavelet(..) - | FieldDataType::StatModel(..) => (None, &[]), - }; + let (merge_inputs, readouts): (_, &'static [(StatisticKind, MergedError)]) = + match summary_type { + FieldDataType::ExactAggregate(kind, _) => match kind { + E::Sum | E::Count => (Some(Disjoint), &[]), + E::Min | E::Max => (Some(MayOverlap), &[]), + // The counter accumulator joins time-ordered windows with + // reset handling (`IncreaseAccumulator::merge_pair`). + E::Rate | E::Increase => (Some(DisjointInTime), &[]), + // IRate reads the last two samples of the window, which a + // tumbling window does not keep across its boundary. + E::IRate => (None, &[]), + }, + // A sketch algorithm's row is sketchlib's. HydraKll merges as + // that row says: `Hydra::merge` requires the same k and m and + // otherwise returns an error (asap_sketchlib #251). + FieldDataType::Sketch(algorithm) => { + let row = algorithm.capability(); + return Self { + merge_inputs: row.merge_inputs.map(MergeInputRule::from), + readouts: row.readouts, + }; + } + FieldDataType::Plain(_) + | FieldDataType::SummarySample(..) + | FieldDataType::Wavelet(..) + | FieldDataType::StatModel(..) => (None, &[]), + }; Self { merge_inputs, readouts, @@ -194,11 +186,7 @@ impl FamilyCapability { mod tests { use super::*; use crate::ir::scalar::expr_ir::ColumnRef; - use crate::ir::schema::{ExactParams, SketchParams}; - - fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(algorithm, GroupingStrategy::default()) - } + use crate::ir::schema::ExactParams; fn total() -> SketchStatistic { SketchStatistic::PointCount { @@ -218,16 +206,7 @@ mod tests { /// so it can serve all four; it answers no quantile or top-k. #[test] fn univmon_answers_its_total_and_every_frequency_moment() { - let univmon = FamilyCapability::of(&sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 256, - sketch_rows: 5, - sketch_cols: 1024, - layers: 16, - distinct_failure_probability: None, - }, - )); + let univmon = SummaryTypeCapability::of(&FieldDataType::Sketch(SketchAlgorithm::UnivMon)); for statistic in [ total(), SketchStatistic::Cardinality, @@ -249,40 +228,12 @@ mod tests { /// HLL answers the distinct count, not its total. #[test] fn summary_capability_facts() { - let kll = sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ); - let ddsketch = sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ); - let count_sketch = sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 64, - depth: 5, - }, - ); - let cms_heap = sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 5, - heap_size: 10, - }, - ); - let hll = sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ); - for (family, yes, no) in [ + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let ddsketch = FieldDataType::Sketch(SketchAlgorithm::DDSketch); + let count_sketch = FieldDataType::Sketch(SketchAlgorithm::CountSketch); + let cms_heap = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); + let hll = FieldDataType::Sketch(SketchAlgorithm::Hll); + for (summary_type, yes, no) in [ ( &kll, vec![ @@ -317,12 +268,18 @@ mod tests { vec![SketchStatistic::FrequencyL2, item(), total()], ), ] { - let capability = FamilyCapability::of(family); + let capability = SummaryTypeCapability::of(summary_type); for statistic in yes { - assert!(capability.answers(&statistic), "{family:?} {statistic:?}"); + assert!( + capability.answers(&statistic), + "{summary_type:?} {statistic:?}" + ); } for statistic in no { - assert!(!capability.answers(&statistic), "{family:?} {statistic:?}"); + assert!( + !capability.answers(&statistic), + "{summary_type:?} {statistic:?}" + ); } } } @@ -332,16 +289,9 @@ mod tests { /// per group: not counters, IRate or a Hydra. #[test] fn only_per_group_mergeable_families_merge_groups() { - use crate::ir::schema::HydraKind; - let kll = sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ); - assert!(FamilyCapability::merges_groups(&kll)); - assert!(FamilyCapability::merges_groups( + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + assert!(SummaryTypeCapability::merges_groups(&kll)); + assert!(SummaryTypeCapability::merges_groups( &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) )); for (kind, params) in [ @@ -349,37 +299,23 @@ mod tests { (ExactKind::Increase, ExactParams::Increase), (ExactKind::IRate, ExactParams::IRate), ] { - assert!(!FamilyCapability::merges_groups( + assert!(!SummaryTypeCapability::merges_groups( &FieldDataType::ExactAggregate(kind, params) )); } - let FieldDataType::Sketch(kind, _) = sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 64, - depth: 5, - }, - ) else { - unreachable!() - }; - let hydra = FieldDataType::Sketch( - kind, - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraCms, - }, - ); + let hydra = FieldDataType::Sketch(SketchAlgorithm::HydraCms); assert_eq!( - FamilyCapability::of(&hydra).merge_inputs, + SummaryTypeCapability::of(&hydra).merge_inputs, Some(MergeInputRule::Disjoint) ); - assert!(!FamilyCapability::merges_groups(&hydra)); + assert!(!SummaryTypeCapability::merges_groups(&hydra)); } /// An exact accumulator is read as its finalized value, not a readout. #[test] fn exact_accumulators_answer_no_readout() { let sum = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - assert!(FamilyCapability::of(&sum).readouts.is_empty()); - assert!(!FamilyCapability::of(&sum).answers(&total())); + assert!(SummaryTypeCapability::of(&sum).readouts.is_empty()); + assert!(!SummaryTypeCapability::of(&sum).answers(&total())); } } diff --git a/crates/types/src/physical/mod.rs b/crates/types/src/physical/mod.rs index 28db939f6..f8aa12efb 100644 --- a/crates/types/src/physical/mod.rs +++ b/crates/types/src/physical/mod.rs @@ -1,13 +1,8 @@ -//! #509 Stage 2 decision data: the physical properties of each candidate. -//! -//! - [`properties`] — when a candidate runs and how long its output lives -//! ([`PhysicalProperties`], [`Lifetime`], [`StorageMedium`]). -//! - [`workload_plan`] — one evaluated assignment rebuilt as a physical DAG -//! ([`PhysicalWorkloadPlan`]). +//! #509 Stage 2 decision data: the physical properties of each candidate: +//! when it runs and how long its output lives ([`PhysicalProperties`], +//! [`Lifetime`], [`StorageMedium`]). pub mod properties; -pub mod workload_plan; pub use properties::{ ExecutionTiming, Lifetime, PhysicalProperties, PhysicalPropertiesError, StorageMedium, }; -pub use workload_plan::PhysicalWorkloadPlan; diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 3bc12bf0a..2841c733d 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -52,7 +52,7 @@ pub enum PhysicalPropertiesError { /// $p^{\mathrm{phys}}_v = (\tau, \ell)$. [`Self::new`] rejects the invalid /// combination (concepts doc, section 2). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] pub struct PhysicalProperties { timing: ExecutionTiming, lifetime: Lifetime, diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs deleted file mode 100644 index ce8ed9653..000000000 --- a/crates/types/src/physical/workload_plan.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! One realized selection as a physical DAG. -use crate::ir::physical_export::PhysicalASAPDAG; -use crate::ir::QueryRoot; -#[cfg(any(test, feature = "test-support"))] -use crate::physical::Lifetime; - -/// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one -/// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; -/// display labels are derived from it by the devtools. -#[derive(Debug, Clone)] -pub struct PhysicalWorkloadPlan { - /// `P` (nothing materialized) or `P-m`, set by the caller. - pub id: String, - /// The id of the logical candidate it implements. - pub from_logical: String, - /// The query roots, one per query in workload order, that `dag` - /// exports. - pub roots: Vec, - pub dag: PhysicalASAPDAG, -} - -impl PhysicalWorkloadPlan { - /// Whether any node of `dag` is materialized: a summary build, or a shared - /// non-summary node built once per evaluation. - #[cfg(any(test, feature = "test-support"))] - pub fn materializes_any(&self) -> bool { - self.dag - .nodes - .iter() - .any(|node| node.lifetime != Lifetime::None) - } -} diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index fd2614201..f33adbe13 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; use crate::ir::properties::summary_coverage::SummaryCoverage; -use crate::ir::properties::AccuracyMetric; +use crate::ir::properties::{AccuracyMetric, AccuracyTarget}; use crate::ir::schema::{SketchAlgorithm, SketchParams}; use crate::ir::ASAPOp; use crate::ir::Operator; @@ -16,7 +16,7 @@ use crate::ir::SchemaDerivationError; /// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`]). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] pub struct CandidateForMemoGroupId { pub group: MemoGroupId, pub index_in_group: usize, @@ -32,6 +32,13 @@ pub struct Memo { groups: Vec, /// Each query's root over groups, by index in the workload. pub query_roots: Vec>, + /// Each query's accuracy target $A_q$, by index; `None` for a query + /// without one (every query when accuracy is not enforced). + pub query_accuracy_targets: Vec>, + /// Each query's $n_q$, the number of values one evaluation returns, by + /// index; `None` when the data workload does not bound it. Pass 2 turns + /// a whole-result failure probability into a per-value one with it. + pub query_values_per_evaluation: Vec>, } impl Memo { @@ -40,11 +47,17 @@ impl Memo { &self.groups } - /// Adds a query whose root is `root` and returns its index: the root's - /// groups and every group below them list it - /// ([`MemoGroup::reached_by_queries`]). Stage 0 adds each query once. - pub fn add_query(&mut self, root: QueryRoot) -> usize { - let _ = root; + /// Adds a query whose root is `root`, with its accuracy target and + /// $n_q$, and returns its index: the root's groups and every group below + /// them list it ([`MemoGroup::reached_by_queries`]). Stage 0 adds each + /// query once. + pub fn add_query( + &mut self, + root: QueryRoot, + accuracy_target: Option, + values_per_evaluation: Option, + ) -> usize { + let _ = (root, accuracy_target, values_per_evaluation); todo!("#1207 PR 2") } @@ -81,10 +94,9 @@ impl Memo { } /// Replaces candidate `id`'s input requirements $r^{\log}_{c,i}$, one - /// per input group. The only way to change a candidate's - /// $r^{\log}$: Pass 2 sets each parameter floor from the input's caps - /// and direct needs after the rules have run. Panics when the vector - /// does not have one entry per input group. + /// per input group. The only way to change a candidate's $r^{\log}$: + /// Pass 2 sets a readout's required accuracy after the rules have run. + /// Panics when the vector does not have one entry per input group. pub fn set_input_requirements( &mut self, id: CandidateForMemoGroupId, @@ -147,6 +159,8 @@ impl Memo { /// input group's `group_properties` and `expression` /// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression /// breaks them (schema derivation, input validation, merge check). +// `column_sign` is derived in #1207 PR 2; until then the result is unreachable. +#[allow(unreachable_code)] fn derive_group_properties( memo: &Memo, expression: &Operator, @@ -167,6 +181,7 @@ fn derive_group_properties( result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), + column_sign: todo!("#1207 PR 2: each output column's sign, per operator, from the input groups' and the sources' declared signs"), }) } @@ -203,9 +218,9 @@ pub struct CandidateForMemoGroup { } /// Which rule added a candidate. Nothing in planning reads it. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] pub enum OriginOfCandidateForMemoGroup { - /// Pass 1: exact, exact accumulator, or a summary family. + /// Pass 1: exact, exact accumulator, or a summary type. Pass1, /// Canonical value preprocessing feeding a summary build. SummaryInput, @@ -228,14 +243,14 @@ pub struct CandidateExpression { pub operator: Operator, } -/// $p^{\log}_v = \boldsymbol\theta_v$. The family is the operator's +/// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's /// argument, not a property, and a tumbling-window chain's width is the /// chain operator's argument. [`Self::new`] rejects parameters of another -/// family, not in the family's list, missing on a sketch build (a chain +/// algorithm, not in the algorithm's list, missing on a sketch build (a chain /// included) or present on an exact operator. #[derive(Debug, Clone)] pub struct LogicalProperties { - /// $\boldsymbol\theta_v$, from the family's predefined list; `None` for + /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for /// an exact operator. parameters: Option, } @@ -261,43 +276,45 @@ impl LogicalProperties { pub enum LogicalPropertiesError { #[error("an exact operator has no summary parameters")] ParametersOnExactOperator, - #[error("a sketch build needs parameters from its family's list")] + #[error("a sketch build needs parameters from its algorithm's list")] SketchBuildWithoutParameters, - #[error("the parameters are not in their family's predefined list")] + #[error("the parameters are not in their algorithm's predefined list")] ParametersNotInList, } -/// The family's predefined $\boldsymbol\theta$ list (#1207 step 6), each with -/// its error bound derived through `estimators::local_guarantee` for a given -/// statistic. -pub fn parameter_list(family: SketchAlgorithm) -> &'static [SketchParams] { - let _ = family; - todo!("#1207 PR 1: the predefined θ list per family") +/// The algorithm's predefined $\boldsymbol\theta$ list: structures only; the +/// error of each, for a statistic at a failure probability, is its guarantee +/// (`asap_accuracy::theoretical_guarantee`). +pub fn parameter_list(algorithm: SketchAlgorithm) -> &'static [SketchParams] { + let _ = algorithm; + todo!("#1207 PR 1: the predefined θ list per algorithm") } -/// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. -#[derive(Debug, Clone)] +/// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. Only a +/// readout's input (`SummaryEstimate`, `FinalizeExactAccumulator`, +/// `EvaluatePopulation` over an approximate state) carries one (concepts +/// doc, section 2, "Logical requirements"). +#[derive(Debug, Clone, Default, PartialEq)] pub struct LogicalRequirements { - /// $\boldsymbol\theta^{\min}_{c,i}$: the weakest parameters whose - /// $\varepsilon$, for the statistic the consumer reads, is at most - /// $\text{cap}_{c,i}$ = min(validity cap of $g_c$, pruning cap), or a - /// direct need such as heap_size ≥ k. Any value of the family's parameter - /// type; it need not be in the family's list. - parameter_floor_for_required_accuracy: Option, + /// `None` on every input that is not a readout's, and on a readout's + /// input when no accuracy target reaches it. + pub required_accuracy: Option, } -impl LogicalRequirements { - pub fn new(parameter_floor_for_required_accuracy: Option) -> Self { - Self { - parameter_floor_for_required_accuracy, - } - } - - /// $\boldsymbol\theta^{\min}_{c,i}$. - pub fn parameter_floor_for_required_accuracy(&self) -> Option<&SketchParams> { - self.parameter_floor_for_required_accuracy.as_ref() - } +/// The accuracy a readout's input needs from its producer: the producer's +/// guarantee for the readout's statistic, evaluated at +/// `failure_probability`, is at most `max_error` in `metric`. It is also +/// the readout's own output error. +#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize)] +pub struct RequiredAccuracy { + pub metric: AccuracyMetric, + /// $\varepsilon_r$, in the metric's units. + pub max_error: f64, + /// $\delta_r$, per value; 1 when the target sets none. + pub failure_probability: f64, +} +impl LogicalRequirements { /// $g_c$'s linear bound for errors in `metric`, each input's and the /// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the /// operator and its arguments. `None` when the consumer has no error @@ -324,14 +341,3 @@ pub struct ErrorPropagation { /// denominator). pub validity_cap_per_input: Vec>, } - -/// $\alpha^{\log}_{u,c,i}$: $\boldsymbol\theta_u \succeq -/// \boldsymbol\theta^{\min}_{c,i}$, compared per family on the parameters -/// that set $\varepsilon$. No floor, or an exact producer, always satisfies. -pub fn producer_satisfies_logical_requirements( - producer: &LogicalProperties, - requirement: &LogicalRequirements, -) -> bool { - let _ = (producer, requirement); - todo!("#1207 PR 1: per-family dominance") -} diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index aa13b2b29..9b21b38f3 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -3,9 +3,8 @@ //! the candidates dead-candidate elimination removed, each with its root //! causes. -use super::{CandidateForMemoGroupId, Memo}; +use super::{CandidateForMemoGroupId, Memo, RequiredAccuracy}; use crate::ir::properties::AccuracyMetric; -use crate::ir::schema::SketchParams; /// $(u, c, i)$ over logical candidates: `producer_candidate_id` is a /// candidate of $G_{c,i}$, the group input `input_index_of_consumer` of @@ -37,7 +36,8 @@ pub struct LogicalASAPMemoCandidates { pub struct PrunedLogicalCandidate { pub candidate: CandidateForMemoGroupId, /// Every cause: one per failing edge it is cut off by, so mixed causes - /// (a floor on one edge, the metric rule on another) are all listed. + /// (a missed requirement on one edge, the metric rule on another) are + /// all listed. pub causes: Vec, } @@ -52,13 +52,14 @@ pub struct PruneCause { /// Why the edge at the end of a [`PruneCause`]'s path fails, or why the /// candidate has no edge at all. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize)] pub enum RootPruneReason { - /// The producer's parameters are weaker than the consumer input's floor. - BelowAccuracyFloorRequirement { + /// The producer's guarantee, at the required failure probability, + /// exceeds the consumer input's required error. + BelowRequiredAccuracy { consumer: CandidateForMemoGroupId, input: usize, - floor: SketchParams, + required: RequiredAccuracy, }, /// The consumer's error contract carries none of the producer's metrics. MetricRule { diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index e7c3948cd..06f4333f4 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -10,7 +10,7 @@ use crate::ir::schema::Schema; use crate::ir::{Operator, OperatorResultKind, QueryRoot}; /// A group, by index in the memo's groups. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] pub struct MemoGroupId(pub u32); /// One distinct logical sub-expression of the workload: what it computes @@ -41,6 +41,8 @@ pub struct MemoGroup { pub struct GroupProperties { pub schema: Schema, pub result_kind: OperatorResultKind, + /// Per output column, in schema order: the sign of its values. + pub column_sign: Vec, /// What an aggregate group covers: a summary state, a chain, a merge /// over a chain's windows, or an exact `Aggregate` result; `None` for /// any other group. Derived today for summary states only; exact @@ -51,6 +53,18 @@ pub struct GroupProperties { pub shareable_when_equal: bool, } +/// The sign of one column's values, a group property derived from the +/// input groups like lineage (concepts doc, section 3, "Value sign"); a +/// `Scan`'s are declared per column in the data workload's source facts. +/// Error contracts that hold only for values of one sign read it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValueSign { + NonNegative, + NonPositive, + Unknown, +} + /// A memo group read as an operator's input: its `group_properties` and /// its `expression`, whose inputs are read the same way. #[derive(Clone, Copy)] diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index f032bacad..b87cee1e2 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -13,26 +13,25 @@ mod memo; mod per_computation_logical_memo_candidates; mod physical; mod pruning; -mod realizer; pub use candidates::{ - parameter_list, producer_satisfies_logical_requirements, CandidateError, CandidateExpression, - CandidateForMemoGroup, CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, - LogicalPropertiesError, LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, + parameter_list, CandidateError, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, LogicalPropertiesError, + LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, RequiredAccuracy, }; pub use logical_memo_candidates::{ LogicalASAPMemoCandidates, LogicalEdge, PruneCause, PrunedLogicalCandidate, RootPruneReason, }; pub(crate) use memo::MemoGroupAsInput; -pub use memo::{queries_rooted_at, root_groups, GroupProperties, MemoGroup, MemoGroupId}; +pub use memo::{ + queries_rooted_at, root_groups, GroupProperties, MemoGroup, MemoGroupId, ValueSign, +}; pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, }; pub use physical::{ - output_references, producer_satisfies_physical_requirements, DagOfSelectedCandidates, - ExecutionContext, ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidate, - PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, RefreshWindowId, SelectedEdge, - SelectedNodeId, + output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, + ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, + PhysicalEdge, PhysicalEdgeId, }; pub use pruning::{PruneReason, PrunedCandidate}; -pub use realizer::{AssignmentRealizer, PhysicalCandidateError, RealizedPlan}; diff --git a/crates/types/src/stages/physical/execution_context.rs b/crates/types/src/stages/physical/execution_context.rs index 0ab1630e8..a0bf2f6c7 100644 --- a/crates/types/src/stages/physical/execution_context.rs +++ b/crates/types/src/stages/physical/execution_context.rs @@ -1,36 +1,34 @@ -//! $\kappa$: where a candidate with $\ell$ = None or `OneEvaluation` runs -//! (concepts doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and -//! $y_{\kappa,u,c,i}$ by it; [`super::PhysicalCandidate::delivers_on`] reads -//! it, so it lives with the physical candidates. +//! $\kappa$: a class of executions that happen at the same times (concepts +//! doc, section 5). Stage 3 indexes $x_{\kappa,v}$ and $y_{\kappa,u,c,i}$ by +//! it; [`super::PhysicalCandidate::delivers_on`] reads it, so it lives with +//! the physical candidates. -use super::graph::PhysicalCandidateId; -use crate::workload::{QueryRecurrence, RangeEvaluation}; - -/// A refresh window of the workload's ingestion pipelines, by index. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct RefreshWindowId(pub usize); +use super::memo_candidates::PhysicalCandidateId; +use crate::ir::materialized_view::RefreshWindow; /// Index of an [`ExecutionContext`] in the workload's list of distinct -/// contexts. Contexts are deduplicated by equality: [`QueryRecurrence`] -/// holds an `f64` (`EstimatedRate`), so a context cannot be hashed. +/// contexts. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct ExecutionContextId(pub usize); -/// $\kappa$: a class of executions with a common owner and schedule, where -/// a candidate with $\ell$ = None or `OneEvaluation` runs. -#[derive(Debug, Clone, PartialEq)] +/// $\kappa$: within one context a computation runs once, and every consumer +/// in it shares that run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ExecutionContext { - /// The evaluation of every query with this recurrence and range - /// evaluation, the workload's own description of when it runs: queries - /// with equal ones are evaluated together and share its `OneEvaluation` - /// results. A and B both every 10 s at phase 0 give one context; a - /// one-off query's context is its `OneTime` recurrence. - Evaluation { - recurrence: QueryRecurrence, - range_evaluation: Option, - }, - /// $I_r$: the ingestion pipeline of a refresh window. - IngestionPipeline(RefreshWindowId), - /// $\kappa_m$: inside an `AcrossEvaluations` candidate's own computation. + /// At every time of an evaluation grid, answering queries: every query + /// evaluated on the grid, a range query's steps included. An alert + /// every 10 s at phase 0 is the grid 10 s @ 0. + Evaluation { grid: EvaluationGrid }, + /// As data arrives, in batches of one refresh window. + IngestionPipeline(RefreshWindow), + /// The own computation of a candidate materialized across evaluations, + /// on its own schedule, whoever reads it. Materialized(PhysicalCandidateId), } + +/// The evaluation times `phase_ms + k · period_ms`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct EvaluationGrid { + pub period_ms: u64, + pub phase_ms: u64, +} diff --git a/crates/types/src/stages/physical/graph.rs b/crates/types/src/stages/physical/graph.rs deleted file mode 100644 index e2f9afb85..000000000 --- a/crates/types/src/stages/physical/graph.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Selections over Stage 2's physical candidates, and their structural -//! validation. -use super::memo_candidates::PhysicalASAPMemoCandidates; -use std::collections::HashSet; - -/// Index into [`PhysicalASAPMemoCandidates::candidates`](super::PhysicalASAPMemoCandidates::candidates). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct PhysicalCandidateId(pub usize); - -/// Index into [`DagOfSelectedCandidates::candidate_of_node`]: one run of a -/// candidate. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct SelectedNodeId(pub usize); - -/// One edge of a [`DagOfSelectedCandidates`]: `producer`'s output is input -/// `input` of `consumer`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SelectedEdge { - pub producer: SelectedNodeId, - pub consumer: SelectedNodeId, - /// Index in the consumer's inputs: its logical candidate's - /// [`Operator::children`](crate::ir::operator::Operator::children). - pub input_index_of_consumer: usize, -} - -/// The selected plan as a DAG with one node per run of a candidate: what -/// Stage 3 selects and the facade builds through -/// [`AssignmentRealizer::realize`](crate::stages::AssignmentRealizer::realize). -/// A candidate that runs twice (recomputed by two consumers, or two shared -/// instances over different inputs) is two nodes. -#[derive(Debug, Clone, PartialEq)] -pub struct DagOfSelectedCandidates { - /// Node `i` runs candidate `candidate_of_node[i]`. - pub candidate_of_node: Vec, - /// For each input of each node, the node that produces it. - pub edges: Vec, - /// The node producing each output reference, one per output reference, - /// in [`output_references`](super::output_references) order. - pub node_of_output_reference: Vec, -} - -impl DagOfSelectedCandidates { - /// Structural check against `physical`, instead of trusting rounded - /// solver values: every node runs an existing candidate, every input of - /// every node has exactly one edge from a node of a producer on one of - /// its admissible edges, the graph is acyclic, every node is reached by - /// some output reference, each output reference comes from a candidate - /// of its group. Costs and accuracy are Stage 3's to check. - pub fn validate(&self, physical: &PhysicalASAPMemoCandidates) -> Result<(), String> { - let memo = &physical.logical_candidates.memo; - let output_groups: Vec<_> = super::output_references(&memo.query_roots) - .into_iter() - .map(|(_, group)| group) - .collect(); - let query_output_candidates: Vec> = output_groups - .iter() - .map(|&group| { - (0..physical.physical_candidates.len()) - .map(PhysicalCandidateId) - .filter(|id| { - physical.physical_candidates[id.0] - .logical_candidate_id - .group - == group - }) - .collect() - }) - .collect(); - let query_output_candidates: Vec<&[_]> = - query_output_candidates.iter().map(Vec::as_slice).collect(); - let edges: Vec<_> = physical.edges().collect(); - self.validate_against( - physical.physical_candidates.len(), - |candidate| { - memo.candidate(physical.physical_candidates[candidate.0].logical_candidate_id) - .expression - .operator - .children() - .len() - }, - |candidate, input_index_of_consumer, producer| { - physical - .admissible_edges_into(candidate, input_index_of_consumer) - .iter() - .any(|edge| edges[edge.0].1.producer_candidate_id == producer) - }, - &query_output_candidates, - ) - } - - /// [`Self::validate`] over a catalog given by its parts: - /// `candidate_count` candidates; `input_count(c)` inputs of candidate - /// `c`; `admissible(c, i, u)` whether an admissible edge `(u, c, i)` - /// exists; `query_output_candidates[r]` the candidates that may produce - /// output reference `r`. - fn validate_against( - &self, - candidate_count: usize, - input_count: impl Fn(PhysicalCandidateId) -> usize, - admissible: impl Fn(PhysicalCandidateId, usize, PhysicalCandidateId) -> bool, - query_output_candidates: &[&[PhysicalCandidateId]], - ) -> Result<(), String> { - let invalid = - || "decoded candidate selection violates dependency or root constraints".to_string(); - let count = self.candidate_of_node.len(); - if self - .candidate_of_node - .iter() - .any(|c| c.0 >= candidate_count) - || self.node_of_output_reference.len() != query_output_candidates.len() - { - return Err(invalid()); - } - for (node, allowed) in self - .node_of_output_reference - .iter() - .zip(query_output_candidates) - { - if node.0 >= count || !allowed.contains(&self.candidate_of_node[node.0]) { - return Err(invalid()); - } - } - // Per node, per input: the node producing it. - let mut producers: Vec>> = self - .candidate_of_node - .iter() - .map(|&candidate| vec![None; input_count(candidate)]) - .collect(); - let mut consumers = vec![Vec::new(); count]; - for edge in &self.edges { - let (producer, consumer) = (edge.producer.0, edge.consumer.0); - if producer >= count || consumer >= count { - return Err(invalid()); - } - let Some(target) = producers[consumer].get_mut(edge.input_index_of_consumer) else { - return Err(invalid()); - }; - if target.is_some() - || !admissible( - self.candidate_of_node[consumer], - edge.input_index_of_consumer, - self.candidate_of_node[producer], - ) - { - return Err(invalid()); - } - *target = Some(producer); - consumers[producer].push(consumer); - } - if producers.iter().flatten().any(Option::is_none) { - return Err(invalid()); - } - // Kahn's algorithm checks cycles without recursion on a potentially deep DAG. - let mut outstanding: Vec<_> = producers.iter().map(Vec::len).collect(); - let mut ready: Vec<_> = (0..count).filter(|&i| outstanding[i] == 0).collect(); - let mut visited = 0; - while let Some(producer) = ready.pop() { - visited += 1; - for &consumer in &consumers[producer] { - outstanding[consumer] -= 1; - if outstanding[consumer] == 0 { - ready.push(consumer); - } - } - } - if visited != count { - return Err(invalid()); - } - // The nodes each output reference reaches. - let output_reach: Vec> = self - .node_of_output_reference - .iter() - .map(|root| { - let mut reached = HashSet::new(); - let mut pending = vec![root.0]; - while let Some(node) = pending.pop() { - if reached.insert(node) { - pending.extend(producers[node].iter().flatten().copied()); - } - } - reached - }) - .collect(); - if (0..count).any(|node| !output_reach.iter().any(|reach| reach.contains(&node))) { - return Err(invalid()); - } - Ok(()) - } -} diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 7070ef3f3..0abda93cc 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -7,11 +7,14 @@ use std::collections::HashMap; use super::execution_context::ExecutionContext; -use super::graph::PhysicalCandidateId; use crate::ir::QueryRoot; -use crate::physical::PhysicalProperties; +use crate::physical::{ExecutionTiming, PhysicalProperties}; use crate::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, PrunedCandidate}; +/// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] +pub struct PhysicalCandidateId(pub usize); + /// Index into [`PhysicalASAPMemoCandidates::edges`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct PhysicalEdgeId(pub usize); @@ -46,9 +49,9 @@ impl PhysicalCandidate { /// [`PhysicalCandidate::delivers_on`]. pub fn producer_satisfies_physical_requirements( producer: &PhysicalCandidate, - consumer: &PhysicalCandidate, + consumer_timing: ExecutionTiming, ) -> bool { - let _ = (producer, consumer); + let _ = (producer, consumer_timing); todo!("#1207 PR 1: the timing x lifetime table of section 2") } diff --git a/crates/types/src/stages/physical/mod.rs b/crates/types/src/stages/physical/mod.rs index fd2d1ba0b..dc3b1bb3a 100644 --- a/crates/types/src/stages/physical/mod.rs +++ b/crates/types/src/stages/physical/mod.rs @@ -2,12 +2,10 @@ //! builds from the memo and Stage 3 selects from (#1047, #1054, #1200, #1201). mod execution_context; -mod graph; mod memo_candidates; -pub use execution_context::{ExecutionContext, ExecutionContextId, RefreshWindowId}; -pub use graph::{DagOfSelectedCandidates, PhysicalCandidateId, SelectedEdge, SelectedNodeId}; +pub use execution_context::{EvaluationGrid, ExecutionContext, ExecutionContextId}; pub use memo_candidates::{ output_references, producer_satisfies_physical_requirements, PhysicalASAPMemoCandidates, - PhysicalCandidate, PhysicalEdge, PhysicalEdgeId, + PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, PhysicalEdgeId, }; diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index 360621bcf..fb994d1e4 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -7,14 +7,13 @@ //! its selection MILP (dominated and uncostable candidates); its //! rejections of whole plans keep their own cause (`RejectionCause` in plan //! selection). The planner does not check whether an executor implements an -//! operator or family, so no reason names it. +//! operator or summary type, so no reason names it. use super::PhysicalCandidateId; -/// One candidate a stage removed, and why. `C` is the stage's candidate -/// type: Stage 2's [`PhysicalCandidate`](super::PhysicalCandidate), Stage -/// 3's [`PhysicalCandidateId`]. The candidate names its group through its -/// logical candidate, so there is no separate group field. +/// One candidate a stage removed, and why: Stage 2 and Stage 3 record the +/// [`PhysicalCandidate`](super::PhysicalCandidate). The candidate names its +/// group through its logical candidate, so there is no separate group field. #[derive(Debug, Clone, PartialEq)] pub struct PrunedCandidate { pub candidate: C, @@ -24,7 +23,7 @@ pub struct PrunedCandidate { /// Why a stage removed a candidate: first Stage 2's (the physical candidate /// cannot be built, run or read by a query output), then Stage 3's, before /// its selection MILP. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] pub enum PruneReason { /// Stage 2: it runs at ingestion time, and the source has no ingestion /// pipeline (its data is at rest or its arrival unknown). @@ -43,6 +42,10 @@ pub enum PruneReason { /// Stage 2: no query output can read it, directly or through other /// candidates, once each candidate's allowed consumers are applied. ReachesNoQueryOutput, + /// Stage 2: a Count-Min build (`Cms`, `CmsWithHeap`, `HydraCms`) whose + /// weight column's `column_sign` is not `NonNegative`: its estimates are + /// bounds only for non-negative weights. + CountMinWeightsNotProvenNonNegative, /// Stage 3: the candidate it names, in the same memo group with the /// same producers and consumers, is at least as good in every respect /// (guarantee, usage, latency, output, held state), so no optimal plan diff --git a/crates/types/src/stages/realizer.rs b/crates/types/src/stages/realizer.rs deleted file mode 100644 index 96ad555a1..000000000 --- a/crates/types/src/stages/realizer.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! The Stage 3 → Stage 2 callback (#1054 X1): Stage 3 proposes a DAG of -//! selected candidates, and Stage 2 realizes it. Stage 3 receives a `&dyn AssignmentRealizer` from the facade, -//! so it never depends on the Stage 2 crate. -use std::collections::BTreeMap; -use std::rc::Rc; - -use crate::ir::physical_export::PhysicalASAPNodeId; -use crate::ir::OperatorNode; -use crate::physical::PhysicalWorkloadPlan; - -use super::{DagOfSelectedCandidates, PhysicalASAPMemoCandidates}; - -/// Why a selected assignment could not be realized. Only rejected -/// assignments are safe to exclude from the finite search; adapter or -/// identity failures must stop with diagnostics, not become no-good cuts. -#[derive(Debug, thiserror::Error)] -pub enum PhysicalCandidateError { - #[error("selected physical assignment is infeasible: {0}")] - RejectedAssignment(String), - #[error("invalid or unsupported physical reconstruction: {0}")] - InvalidOrUnsupported(String), -} - -impl From for PhysicalCandidateError { - fn from(message: String) -> Self { - Self::InvalidOrUnsupported(message) - } -} -impl From<&str> for PhysicalCandidateError { - fn from(message: &str) -> Self { - Self::InvalidOrUnsupported(message.into()) - } -} - -/// A realized selection (was `SelectedPhysicalCandidate`): the physical -/// plan Stage 2 built for it, how long each materialized tumbling-window chain of that plan -/// keeps its tumbling windows, and the operator node each selected node became. -#[derive(Debug, Clone)] -pub struct RealizedPlan { - /// The selection rebuilt as one physical plan, one operator node per - /// selected node, each with its candidate's timing and lifetime. - pub plan: PhysicalWorkloadPlan, - /// For each tumbling-window chain of `plan` materialized beyond one evaluation - /// (maintained at ingestion time, or built at query time and materialized - /// across evaluations), by node id in `plan.dag`: how far back from the - /// newest evaluation the runtime must keep its completed tumbling windows, in - /// milliseconds. It is how far its tumbling windows reach back: the offset its - /// newest tumbling window ends before the evaluation (`[60m] offset 10m`: 10 min), - /// plus the chain's window, plus the longest range-evaluation span of - /// its queries, so the earliest step of the next request still finds - /// its tumbling windows; without an offset or range evaluations, just the window. - /// Empty when the plan keeps no tumbling windows. - pub tumbling_window_retention_ms: BTreeMap, - /// The operator node of `plan` each selected node became, indexed by - /// [`SelectedNodeId`](super::SelectedNodeId). - pub operator_node_of_selected_node: Vec>, -} - -/// Realizes a [`DagOfSelectedCandidates`] through Stage 2, one plan node -/// per selected node. Implemented by the -/// Stage 2 crate, which holds the planning context (the workload's -/// original query roots, its entries and data, and whether query-time -/// results are retained); the facade builds it, passes it to Stage 3, and -/// calls the same method to build the selected plan. -/// -/// Guarantees: a returned result has one operator node per selected node, each -/// rebuilt over its selected producers with its identity kept, guarantees -/// recomposed, and every output contract (result kind and schema) met; a -/// realized assignment also passes Stage 2's timing and dependency rules -/// and the physical DAG validator, with every node keeping its -/// candidate's timing and lifetime. -/// -/// It checks structure and execution legality only. It does not check cost, -/// and it does not check accuracy targets: Stage 3 does both. -pub trait AssignmentRealizer { - fn realize( - &self, - physical: &PhysicalASAPMemoCandidates, - selected: &DagOfSelectedCandidates, - ) -> Result; -} diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 6f852a44c..0860327ed 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -9,7 +9,7 @@ use crate::ir::operator::operator_properties::{ }; use crate::ir::operator::{AggIntent, GroupKeys, Reduction, Source}; use crate::ir::physical_export::PhysicalASAPOperatorPayload as Payload; -use crate::ir::schema::{Schema, SketchStatistic, WeightDomain}; +use crate::ir::schema::{Schema, SketchStatistic}; use crate::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, ScalarExpr, }; @@ -406,25 +406,14 @@ fn at_most_one_row(node: &OperatorNode) -> bool { } } -/// `domain` with an open requirement answered from `data`'s declared source -/// facts (`WeightDomain::confirmed`): samples are proven when the metric is -/// declared a counter or non-negative, a counter derivative only when it is -/// declared a counter; unknown otherwise. No type is inferred from a name. -pub fn confirmed_weights(domain: &WeightDomain, data: &DataWorkload) -> WeightDomain { - domain.confirmed( - |metric| declared_counter(data, metric), - |metric| declared_non_negative(data, metric), - ) -} - /// Whether `data` declares every sample of `metric` non-negative: the -/// metric is declared a counter, or a metric (a gauge, say) with -/// [`SourceFacts::values_non_negative`](crate::workload::SourceFacts::values_non_negative). +/// metric is declared a counter, or its `value` column is declared +/// non-negative ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). pub fn declared_non_negative(data: &DataWorkload, metric: &str) -> bool { declared_counter(data, metric) - || data - .series_facts(metric) - .is_some_and(|(facts, _)| facts.values_non_negative) + || data.series_facts(metric).is_some_and(|(facts, _)| { + facts.column_signs.get("value") == Some(&crate::stages::ValueSign::NonNegative) + }) } /// Whether `data` declares `metric` a counter, itself or as the `_sum` or @@ -436,70 +425,6 @@ fn declared_counter(data: &DataWorkload, metric: &str) -> bool { .is_some_and(|(_, series_type)| series_type == Some(PrometheusMetricType::Counter)) } -/// `roots` with every open [`WeightDomain::RawDataSamplesOfMetric`] requirement -/// replaced by its answer from `data` ([`confirmed_weights`]): the form of a -/// selected plan the executor runs, which reads no data workload. The same -/// `Rc` where nothing changes. -pub fn with_confirmed_weights(roots: &[QueryRoot], data: &DataWorkload) -> Vec { - with_confirmed_weights_at(roots, data, &|_| true) -} - -/// [`with_confirmed_weights`] at the summaries `confirm_at` names only: the -/// selected plan with the weight proofs Stage 3 made for its candidates -/// applied (`Selection::nodes_with_proven_non_negative_weights`). -pub fn with_confirmed_weights_at( - roots: &[QueryRoot], - data: &DataWorkload, - confirm_at: &dyn Fn(&Rc) -> bool, -) -> Vec { - fn confirm( - node: &Rc, - data: &DataWorkload, - confirm_at: &dyn Fn(&Rc) -> bool, - cache: &mut HashMap<*const OperatorNode, Rc>, - ) -> Rc { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut changed = false; - let mut operator = node.operator.map_children(|child| { - let rebuilt = confirm(child, data, confirm_at, cache); - changed |= !Rc::ptr_eq(&rebuilt, child); - rebuilt - }); - if let Operator::ASAP( - ASAPOp::SummaryAgg { input, .. } | ASAPOp::SummaryTumblingWindowChain { input, .. }, - ) = &mut operator - { - let confirmed = confirmed_weights(&input.weight_domain, data); - if confirmed != input.weight_domain && confirm_at(node) { - input.weight_domain = confirmed; - changed = true; - } - } - let rebuilt = if changed { - Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()), - ) - } else { - Rc::clone(node) - }; - cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); - rebuilt - } - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| match root { - QueryRoot::Operator(n) => QueryRoot::Operator(confirm(n, data, confirm_at, &mut cache)), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |n| confirm(n, data, confirm_at, &mut cache)), - ), - }) - .collect() -} - /// Rows per millisecond the sources in `sources` receive together, each /// at its own rate: `series_count` series each sampled every /// `sample_interval`, or, for a source that does not declare both, its diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index 3558df41c..d013fd66f 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -4,6 +4,7 @@ pub mod parsed_workload; use std::collections::BTreeMap; use crate::ir::properties::AccuracyTarget; +use crate::stages::ValueSign; use serde::{Deserialize, Serialize}; // ── Query surface ───────────────────────────────────────────────────────────── @@ -516,7 +517,7 @@ pub struct DataWorkload { pub arrival: DataArrival, /// Facts per source, keyed by PromQL metric name or SQL table name /// (default: none). A declared histogram or summary is keyed by its - /// family name; its `metric_type`, `values_non_negative` and + /// family name; its `metric_type`, `column_signs` and /// `sample_interval` apply to every series of the family /// ([`Self::series_facts`]). Counts, norms and bounds (`series_count`, /// `stream_norms`, …) differ per series name (a `foo_bucket` has one @@ -546,15 +547,14 @@ pub struct SourceFacts { /// `_bucket`; an undeclared metric's samples may take any value. #[serde(default)] pub metric_type: Option, - /// Whether every sample of the metric is non-negative (default `false`: - /// unknown). A counter's samples always are; this declares it for a - /// gauge, such as a latency or a queue length. The deployment must - /// ensure it. Plan selection reads it as the sign of the metric's - /// samples, which a relative-error bound needs to carry through a - /// `sum` or `avg`, and as proof that Count-Min weights over the raw - /// samples are non-negative. Omitted from JSON when `false`. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] - pub values_non_negative: bool, + /// The declared sign of each column's values, keyed by column name + /// (default: none declared), e.g. `{"value": "non_negative"}` for a + /// latency gauge. A `Scan`'s `column_sign` reads it; an undeclared + /// column is `Unknown`, except a PromQL counter's value, which is + /// non-negative without a declaration. The deployment must ensure it. + /// Omitted from JSON when empty. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub column_signs: BTreeMap, /// How often each series of the metric (or group of rows of a SQL /// table) receives a sample (default: unknown). A PromQL or MetricsQL /// bare instant selector over the metric reads the latest sample within it; lowering such a @@ -1633,7 +1633,10 @@ mod tests { SourceFacts { series_count: Some(10_000), metric_type: Some(PrometheusMetricType::Counter), - values_non_negative: true, + column_signs: BTreeMap::from([( + "value".to_string(), + ValueSign::NonNegative, + )]), sample_interval: Some(DurationMs(15_000)), stream_norms: Some(StreamNorms { l1: Some(1e9), @@ -1662,7 +1665,7 @@ mod tests { "sources": {"requests_total": { "series_count": 10000, "metric_type": "counter", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 15000, "stream_norms": {"l1": 1e9, "l2": null}, "max_distinct_per_evaluation": 100, diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 10a46370a..e9861ccaf 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -25,10 +25,9 @@ fn plan() -> Rc { .unwrap(); let state = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: scan, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named("value".into())), reduction: Reduction::by(vec![0]), - grouping: Default::default(), filter: None, })) .unwrap(); @@ -271,146 +270,6 @@ fn a_raw_data_window_ends_its_offset_before_the_evaluation() { assert_eq!(asap_types::test_support::lookback_ms(&window), 900_000); } -/// `TimeRange(Range, width_ms)` over `shift` (if any) over `scan`, untimed. -fn window_over( - scan: &Rc, - width_ms: u64, - shift: Option, -) -> Rc { - let child = match shift { - Some(shift) => OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeShift { - shift, - child: Rc::clone(scan), - })) - .unwrap(), - None => Rc::clone(scan), - }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { - range: std::time::Duration::from_millis(width_ms), - kind: asap_types::ir::TimeRangeKind::Range, - child, - })) - .unwrap() -} - -/// The exported DAG of `roots`, all at query time, and its one `Scan`'s id. -fn exported_with_scan( - roots: Vec>, -) -> ( - asap_types::ir::physical_export::PhysicalASAPDAG, - asap_types::ir::physical_export::PhysicalASAPNodeId, -) { - let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); - let dag = asap_types::test_support::physical_asap_dag_at_query_time_with_node_ids(&roots) - .map(|(dag, _)| dag) - .unwrap(); - let scans: Vec<_> = dag - .nodes - .iter() - .filter(|node| matches!(node.payload, Operator::NonASAP(NonASAPOp::Scan { .. }))) - .map(|node| node.id) - .collect(); - assert_eq!(scans.len(), 1, "one shared Scan"); - (dag, scans[0]) -} - -/// A time-series `Scan` of `m` with a time index. -fn series_scan() -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap() -} - -// A `Scan` read by `m[5m]` and by `m[1h] offset 10m` is read over `(T - 70m, -// T]`, the union of `(T - 5m, T]` and `(T - 70m, T - 10m]`. -#[test] -fn a_scan_window_is_the_union_of_its_readers_windows() { - use asap_types::ir::physical_export::ScanWindow; - let minute_ms = 60_000; - let scan = series_scan(); - let offset = asap_types::ir::operator::TimeShift { - offset_ms: 10 * minute_ms as i64, - at: None, - }; - let (dag, scan_id) = exported_with_scan(vec![ - window_over(&scan, 5 * minute_ms, None), - window_over(&scan, 60 * minute_ms, Some(offset)), - ]); - let window = dag.scan_window(scan_id, &Default::default()); - assert_eq!( - window, - ScanWindow::Within { - end_offset_ms: 0, - width_ms: 70 * minute_ms as i64, - } - ); - assert_eq!(window.lookback_ms(), Some(70 * minute_ms)); - assert_eq!( - window.bounds_ms(100 * minute_ms as i64), - Some((30 * 60_000, 100 * 60_000)) - ); -} - -// A node above `m[5m]` and `m[1h] offset 10m` summarizes `(T - 70m, T]`, -// the union of the windows below it; a node with no window below it, the -// `Scan` itself, summarizes an unbounded window. -#[test] -fn a_view_window_is_the_union_of_the_windows_below_it() { - use asap_types::ir::physical_export::ScanWindow; - let minute_ms = 60_000; - let scan = series_scan(); - let offset = asap_types::ir::operator::TimeShift { - offset_ms: 10 * minute_ms as i64, - at: None, - }; - let both = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { - children: vec![ - window_over(&scan, 5 * minute_ms, None), - window_over(&scan, 60 * minute_ms, Some(offset)), - ], - })) - .unwrap(); - let (dag, scan_id) = exported_with_scan(vec![both]); - assert_eq!( - dag.view_window(dag.roots[0]), - ScanWindow::Within { - end_offset_ms: 0, - width_ms: 70 * minute_ms as i64, - } - ); - assert_eq!(dag.view_window(scan_id), ScanWindow::Unbounded); -} - -// A window under an `@` anchor is not a fixed span back from the evaluation -// time, so the `Scan` it reads is read over an unbounded window. -#[test] -fn a_scan_read_under_an_at_anchor_has_an_unbounded_window() { - use asap_types::ir::physical_export::ScanWindow; - let scan = series_scan(); - let anchored = asap_types::ir::operator::TimeShift { - offset_ms: 0, - at: Some(asap_types::ir::operator::AtModifier::Timestamp(1_000)), - }; - let (dag, scan_id) = exported_with_scan(vec![ - window_over(&scan, 300_000, None), - window_over(&scan, 300_000, Some(anchored)), - ]); - assert_eq!( - dag.scan_window(scan_id, &Default::default()), - ScanWindow::Unbounded - ); -} - /// High-fan-in merge/concat validation is independent of wire edge order and /// preserves repeated input occurrences, rather than treating edges as a set. #[test] @@ -438,6 +297,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }) .collect(); let inputs: Vec<_> = (0..count).chain([0, 0]).collect(); @@ -452,6 +312,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { output_schema: schema.clone(), guarantee: None, retention_ms: None, + selected_physical_candidate: None, }); let mut dag = PhysicalASAPDAG { nodes, diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 611b0b316..7878f45be 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -2,7 +2,7 @@ use asap_types::ir::operator::{AggIntent, Reduction, Source}; use asap_types::ir::properties::AccuracyGuarantee; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; +use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; use std::rc::Rc; @@ -23,10 +23,9 @@ fn aggregate(child: Rc, asap: bool) -> Rc { let operator = if asap { Operator::ASAP(ASAPOp::SummaryAgg { child, - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named("value".into())), reduction: Reduction::by(vec![0]), - grouping: GroupingStrategy::default(), filter: None, }) } else { diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 47ce849bc..1129d0553 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -89,18 +89,15 @@ fn scalar_type_rules_fail_closed() { fn state_evaluations_and_passthrough_keep_their_contracts() { use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchStatistic, SummaryUpdate, - }; + use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchStatistic, SummaryUpdate}; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; - let family = FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()); + let family = FieldDataType::Sketch(SketchAlgorithm::Kll); let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: scan(), - family, + summary_type: family, input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(), @@ -139,17 +136,16 @@ fn shared_construction_derives_both_operator_categories() { use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::FieldDataType; - use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; + use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use asap_types::ir::{ASAPOp, Operator, ProjectItem}; let input = scan(); let state = Rc::new( OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: input.clone(), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), input: SummaryUpdate::column(ColumnRef::Named("x".into())), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap(), diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index f98eaa7ef..80340b905 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -3,10 +3,10 @@ use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, FamilyCapability, Field, FieldDataType, GroupingStrategy, - HydraKind, HydraParams, MergeInputRule, MergedError, SamplingKind, SamplingParams, Schema, - SketchAlgorithm, SketchParams, SketchStatistic, StatModelKind, StatModelParams, StatisticKind, - SummaryUpdate, WaveletKind, WaveletParams, + DataType, ExactKind, ExactParams, Field, FieldDataType, MergeInputRule, MergedError, + SamplingKind, SamplingParams, Schema, SketchAlgorithm, SketchStatistic, StatModelKind, + StatModelParams, StatisticKind, SummaryTypeCapability, SummaryUpdate, WaveletKind, + WaveletParams, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, @@ -15,10 +15,7 @@ use std::rc::Rc; /// KLL over `value` for one `region`, so states of different regions are /// disjoint. fn state(_k: u32, region: &str) -> Rc { - family_state( - FieldDataType::Sketch(SketchAlgorithm::Kll, Default::default()), - region, - ) + family_state(FieldDataType::Sketch(SketchAlgorithm::Kll), region) } /// A `family` state over `value` for one `region`. fn family_state(family: FieldDataType, region: &str) -> Rc { @@ -45,10 +42,9 @@ fn family_state(family: FieldDataType, region: &str) -> Rc { .unwrap(); OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { child: only_region, - family, + summary_type: family, input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), filter: None, })) .unwrap() @@ -80,50 +76,20 @@ fn incompatible_merge_inputs_fail() { } } -fn sketch(algorithm: SketchAlgorithm, _params: SketchParams) -> FieldDataType { - FieldDataType::Sketch(algorithm, Default::default()) -} -fn hydra( - algorithm: SketchAlgorithm, - _params: SketchParams, - kind: HydraKind, - _hydra_params: HydraParams, -) -> FieldDataType { - FieldDataType::Sketch( - algorithm, - GroupingStrategy::SharedMultiSubpopulation { kind }, - ) +fn sketch(algorithm: SketchAlgorithm) -> FieldDataType { + FieldDataType::Sketch(algorithm) } fn exact(kind: ExactKind, params: ExactParams) -> FieldDataType { FieldDataType::ExactAggregate(kind, params) } fn kll() -> FieldDataType { - sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - ) + sketch(SketchAlgorithm::Kll) } fn hll() -> FieldDataType { - sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - ) + sketch(SketchAlgorithm::Hll) } fn cms_with_heap() -> FieldDataType { - sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: 64, - depth: 4, - heap_size: 10, - }, - ) + sketch(SketchAlgorithm::CmsWithHeap) } fn merge(children: Vec>) -> Result, String> { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { @@ -133,15 +99,14 @@ fn merge(children: Vec>) -> Result, String> { .map_err(|e| e.to_string()) } -/// The family-capability table, row by row: how each family's merge +/// The summary-type capability table, row by row: how each type's merge /// inputs must relate, and which statistics one state answers with the /// error of each from a merged state. Every statistic kind is probed /// through `answers` and `merged_error`, so the table and both lookups are /// pinned together. #[test] -fn family_capability_table() { +fn summary_type_capability_table() { use MergeInputRule::{Disjoint, DisjointInTime, MayOverlap}; - let (width, depth, heap_size) = (64, 4, 10); // (family, how its merge inputs must relate, its readouts). type Row = ( FieldDataType, @@ -217,101 +182,30 @@ fn family_capability_table() { }; let sketches = [ (kll(), SketchAlgorithm::Kll), - ( - sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha: 0.01 }, - ), - SketchAlgorithm::DDSketch, - ), + (sketch(SketchAlgorithm::DDSketch), SketchAlgorithm::DDSketch), (hll(), SketchAlgorithm::Hll), + (sketch(SketchAlgorithm::Cms), SketchAlgorithm::Cms), ( - sketch(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), - SketchAlgorithm::Cms, - ), - ( - sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - ), + sketch(SketchAlgorithm::CountSketch), SketchAlgorithm::CountSketch, ), - ( - sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size, - sketch_rows: depth, - sketch_cols: width, - layers: 8, - distinct_failure_probability: None, - }, - ), - SketchAlgorithm::UnivMon, - ), - ( - sketch( - SketchAlgorithm::Kmv, - SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - ), - SketchAlgorithm::Kmv, - ), + (sketch(SketchAlgorithm::UnivMon), SketchAlgorithm::UnivMon), + (sketch(SketchAlgorithm::Kmv), SketchAlgorithm::Kmv), (cms_with_heap(), SketchAlgorithm::CmsWithHeap), ( - sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ), + sketch(SketchAlgorithm::CountSketchWithHeap), SketchAlgorithm::CountSketchWithHeap, ), ( - hydra( - SketchAlgorithm::Cms, - SketchParams::Cms { width, depth }, - HydraKind::HydraCms, - HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraCms), SketchAlgorithm::HydraCms, ), ( - hydra( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - HydraKind::HydraCountSketch, - HydraParams::HydraCountSketch { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraCountSketch), SketchAlgorithm::HydraCountSketch, ), ( - hydra( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - HydraKind::HydraKll, - HydraParams::HydraKll { - k: 200, - shared_buckets: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraKll), SketchAlgorithm::HydraKll, ), ]; @@ -338,7 +232,7 @@ fn family_capability_table() { SketchStatistic::TopK { k: 5 }, ]; for (family, merge_inputs, readouts) in rows { - let capability = FamilyCapability::of(&family); + let capability = SummaryTypeCapability::of(&family); assert_eq!(capability.merge_inputs, merge_inputs, "{family:?}"); assert_eq!(capability.readouts().to_vec(), readouts, "{family:?}"); for statistic in &statistics { @@ -376,16 +270,7 @@ fn merged_error_depends_on_the_readout() { key: ColumnRef::Named("k".into()), value: None, }; - let univmon = FamilyCapability::of(&sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: 10, - sketch_rows: 4, - sketch_cols: 64, - layers: 8, - distinct_failure_probability: None, - }, - )); + let univmon = SummaryTypeCapability::of(&sketch(SketchAlgorithm::UnivMon)); for (readout, error) in [ (FrequencyL2, MergedError::AsIfBuiltOverUnion), (point(), MergedError::Exact), @@ -395,7 +280,7 @@ fn merged_error_depends_on_the_readout() { assert_eq!(univmon.merged_error(&readout), Some(error), "{readout:?}"); } assert_eq!(univmon.merged_error(&TopK { k: 5 }), None); - let heap = FamilyCapability::of(&cms_with_heap()); + let heap = SummaryTypeCapability::of(&cms_with_heap()); assert_eq!( heap.merged_error(&TopK { k: 5 }), Some(MergedError::Approximate) @@ -403,13 +288,7 @@ fn merged_error_depends_on_the_readout() { assert_eq!(heap.merged_error(&point()), None); // Count Sketch's signed counters give no total: no merged error for a // bare point count, and one build's guarantee for an item's frequency. - let count_sketch = FamilyCapability::of(&sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: 64, - depth: 5, - }, - )); + let count_sketch = SummaryTypeCapability::of(&sketch(SketchAlgorithm::CountSketch)); assert!(!count_sketch.answers(&point())); assert_eq!(count_sketch.merged_error(&point()), None); let item = SketchStatistic::PointCount { @@ -422,17 +301,20 @@ fn merged_error_depends_on_the_readout() { ); // An exact accumulator is read as its finalized value, not a readout. let sum = exact(ExactKind::Sum, ExactParams::Sum); - assert_eq!(FamilyCapability::of(&sum).merged_error(&point()), None); + assert_eq!(SummaryTypeCapability::of(&sum).merged_error(&point()), None); assert_eq!( - FamilyCapability::of(&hll()).merged_error(&Cardinality), + SummaryTypeCapability::of(&hll()).merged_error(&Cardinality), Some(MergedError::AsIfBuiltOverUnion) ); assert_eq!( - FamilyCapability::of(&kll()).merged_error(&SketchStatistic::Quantile { q: 0.99 }), + SummaryTypeCapability::of(&kll()).merged_error(&SketchStatistic::Quantile { q: 0.99 }), Some(MergedError::AsIfBuiltOverUnion) ); let irate = exact(ExactKind::IRate, ExactParams::IRate); - assert_eq!(FamilyCapability::of(&irate).merged_error(&point()), None); + assert_eq!( + SummaryTypeCapability::of(&irate).merged_error(&point()), + None + ); } /// A structurally validated `SummaryEstimate` of `query` over a `family` @@ -452,17 +334,8 @@ fn estimate(family: FieldDataType, query: SketchStatistic) -> Result bool { use SketchAlgorithm as A; use SketchStatistic as Q; @@ -497,6 +370,8 @@ fn accepted_by_category(algorithm: &SketchAlgorithm, query: &SketchStatistic) -> A::Cms | A::CountSketch => matches!(query, Q::PointCount { .. }), A::CmsWithHeap | A::CountSketchWithHeap => matches!(query, Q::TopK { .. }), A::UnivMon => !matches!(query, Q::Quantile { .. }), + A::HydraKll => matches!(query, Q::Quantile { .. }), + A::HydraCms | A::HydraCountSketch => matches!(query, Q::PointCount { .. }), _ => unreachable!("{algorithm:?} is not probed"), } } @@ -509,94 +384,25 @@ fn accepted_by_category(algorithm: &SketchAlgorithm, query: &SketchStatistic) -> /// answer. #[test] fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { - let (width, depth) = (64, 4); + use SketchAlgorithm as A; let mut families: Vec = [ - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - SketchParams::DDSketch { alpha: 0.01 }, - SketchParams::Hll { - precision: 12, - relative_error: None, - }, - SketchParams::Kmv { - k: 1024, - relative_error: None, - }, - SketchParams::Cms { width, depth }, - SketchParams::CountSketch { width, depth }, - SketchParams::CmsWithHeap { - width, - depth, - heap_size: 10, - }, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size: 10, - }, - SketchParams::UnivMon { - heap_size: 10, - sketch_rows: depth, - sketch_cols: width, - layers: 8, - distinct_failure_probability: None, - }, + A::Kll, + A::DDSketch, + A::Hll, + A::Kmv, + A::Cms, + A::CountSketch, + A::CmsWithHeap, + A::CountSketchWithHeap, + A::UnivMon, ] .into_iter() - .map(|params| { - use SketchAlgorithm as A; - let algorithm = match ¶ms { - SketchParams::Kll { .. } => A::Kll, - SketchParams::DDSketch { .. } => A::DDSketch, - SketchParams::Hll { .. } => A::Hll, - SketchParams::Kmv { .. } => A::Kmv, - SketchParams::Cms { .. } => A::Cms, - SketchParams::CountSketch { .. } => A::CountSketch, - SketchParams::CmsWithHeap { .. } => A::CmsWithHeap, - SketchParams::CountSketchWithHeap { .. } => A::CountSketchWithHeap, - SketchParams::UnivMon { .. } => A::UnivMon, - SketchParams::Hydra(_) => unreachable!("no Hydra parameters in this list"), - }; - sketch(algorithm, params) - }) + .map(sketch) .collect(); families.extend([ - hydra( - SketchAlgorithm::Cms, - SketchParams::Cms { width, depth }, - HydraKind::HydraCms, - HydraParams::HydraCms { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), - hydra( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { width, depth }, - HydraKind::HydraCountSketch, - HydraParams::HydraCountSketch { - width, - depth, - shared_rows: depth, - shared_columns: width, - }, - ), - hydra( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: 200, - failure_probability: None, - }, - HydraKind::HydraKll, - HydraParams::HydraKll { - k: 200, - shared_buckets: width, - }, - ), + FieldDataType::Sketch(SketchAlgorithm::HydraCms), + FieldDataType::Sketch(SketchAlgorithm::HydraCountSketch), + FieldDataType::Sketch(SketchAlgorithm::HydraKll), ]); let statistics = [ SketchStatistic::PointCount { @@ -614,7 +420,7 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { SketchStatistic::TopK { k: 5 }, ]; for family in families { - let FieldDataType::Sketch(kind, _) = &family else { + let FieldDataType::Sketch(kind) = &family else { unreachable!() }; for query in &statistics { @@ -623,11 +429,13 @@ fn estimate_validation_matches_the_category_match_except_the_fixed_rows() { ( SketchAlgorithm::UnivMon, StatisticKind::ItemFrequency | StatisticKind::TopK - ) | (SketchAlgorithm::CountSketch, StatisticKind::Total) - | ( - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, - StatisticKind::ItemFrequency, - ) + ) | ( + SketchAlgorithm::CountSketch | SketchAlgorithm::HydraCountSketch, + StatisticKind::Total + ) | ( + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, + StatisticKind::ItemFrequency, + ) ); let before = accepted_by_category(kind, query); let after = estimate(family.clone(), query.clone()).is_ok(); @@ -648,5 +456,5 @@ fn summary_merge_is_permissive_about_the_family() { ]) .unwrap(); merged.validate_structure().unwrap(); - assert_eq!(FamilyCapability::of(&irate()).merge_inputs, None); + assert_eq!(SummaryTypeCapability::of(&irate()).merge_inputs, None); } diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 439a87edb..c695b7d25 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -6,7 +6,7 @@ use std::time::Duration; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; -use asap_types::ir::schema::{GroupingStrategy, SketchAlgorithm, SummaryUpdate}; +use asap_types::ir::schema::{SketchAlgorithm, SummaryUpdate}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; const MINUTE: u64 = 60_000; @@ -28,17 +28,16 @@ fn scan() -> Rc { } fn kll() -> FieldDataType { - FieldDataType::Sketch(SketchAlgorithm::Kll, GroupingStrategy::default()) + FieldDataType::Sketch(SketchAlgorithm::Kll) } /// 1-min KLL tumbling windows per series over `input`. fn chain(input: Rc) -> Rc { OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: input, - family: kll(), + summary_type: kll(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: MINUTE, })) @@ -61,10 +60,9 @@ fn a_chain_reads_an_instant_vector() { }; let empty = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { child: scan(), - family: kll(), + summary_type: kll(), input: SummaryUpdate::column(ColumnRef::SampleValue), reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), filter: None, tumbling_window_width_ms: 0, })) diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index 98823f488..052d2bd62 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -19,7 +19,7 @@ not turn an unknown guarantee into a satisfied accuracy requirement. ## Candidate state Here, **certified for a target** means Planner has a fully known -`AccuracyGuarantee` for the candidate and `AccuracyModel::satisfies(guarantee, +`AccuracyGuarantee` for the candidate and `asap_accuracy::satisfies(guarantee, target)` returns `true`. **Uncertified** means Planner cannot make that claim: the guarantee is absent, contains unknown terms, or is known not to meet the target. An uncertified summary may still be a well-formed logical candidate; @@ -73,7 +73,7 @@ not emit a `RejectedCandidate` for that case. The backend must still check summary family, window, and state-operation capabilities before deployment. -- Accuracy/domain: `AccuracyEvidenceProvider` supplies quantile domains and +- Accuracy/domain: `AccuracyEvidence` supplies measured guarantees; the data workload supplies quantile domains and propagation statistics. DDSketch ratio domains, Hydra shared-grid bounds, TopK margin intervals, HLL confidence, and composition row counts may be missing. A missing non-negative-value proof for relative-error composition @@ -134,7 +134,7 @@ All three start from PromQL text and exercise frontend lowering and planning. None runs a deployed query. For a workload containing `quantile_over_time(0.9,data[5m]) / -quantile_over_time(0.5,data[5m])`, search with `NoAccuracyEvidence` and a root +quantile_over_time(0.5,data[5m])`, search with no accuracy evidence and a root target may expose a direct DDSketch ratio candidate. Its missing domain proof is visible through `has_missing_accuracy_evidence()`. The backend examines the operand windows and its own data-domain facts, checks whether the denominator diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index a3fc422c1..b24531056 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -18,7 +18,7 @@ selected. | `PlanningWorkload.query_workload` | Query language and one-time/repeating query workloads | Yes | | `PlanningWorkload.data_workload` | How the data arrives, and facts about each source | No implicit default. Set `None` when unavailable; a PromQL query with a bare instant selector needs a nonzero `sample_interval` on the metric it reads. | | Frontend-specific dependencies (outside `PlanningWorkload`) | `SqlCatalog` for SQL | `SqlCatalog` is required for SQL lowering | -| `PlanningModels` | Accuracy model and evidence, `CostInputs` (cost model 1, 2 or 3, resource speed, measured operation costs), `DeploymentCapabilities` | `PlanningModels::builtin()` uses `BuiltinAccuracyModel`, no evidence, cost model 1 with illustrative unit costs, and a deployment that supports everything | +| `PlanningModels` | Accuracy evidence (`AccuracyEvidence`), `CostInputs` (cost model 1, 2 or 3, resource speed, measured operation costs), `DeploymentCapabilities` | `PlanningModels::builtin()` uses no evidence, cost model 1 with illustrative unit costs, and a deployment that supports everything | Frontend lowering (Stage 0) and Stages 1–3 run inside the workflow; they are not additional inputs. See [Inputs](#inputs) for the nested workload fields @@ -28,7 +28,7 @@ and [frontend dependencies](#frontend-specific-dependencies). | Output | Fields or contents | Meaning | |---|---|---| -| `PlanOutput` | `plans`: one `QueryPlan { entry_index, root }` per query; `selection`: Stage 3's `Selection` (`selected_dag`, `nodes_with_proven_non_negative_weights`, `cost`, `accepted_accuracy_per_query`, `pruned_candidates`, `search_report`), set by the default pass `Stage1ToStage3Pass`; `tumbling_window_retention_ms`: how long each kept tumbling-window chain keeps its tumbling windows; standalone scalar queries in `scalar_roots` and `scalar_plans` | The selected plan; `physical_asap_dag()` exports it as one `PhysicalASAPDAG` | +| `PlanOutput` | `plan`: the selected plan, one `PhysicalASAPDAG` (its roots in entry order, standalone scalar queries in `scalar_roots`) | The plan installed on the executor; Stage 3's `Selection` report is `run_planning`'s | | `PlanningRun` (from `asap_planner::run_planning`) | Every stage's output: Pass 1's per-computation candidates, the selected round's Pass 2 and Stage 2 outputs (with Stage 2's pruned candidates), the selection, the selected plan, the memo after each step and a report per tumbling-window-width round | For callers that inspect or display the alternatives | The selected plan is a planning artifact. ASAPPlanner does **not** deploy @@ -196,7 +196,7 @@ enum DataArrival { struct SourceFacts { series_count: Option, metric_type: Option, - values_non_negative: bool, + column_signs: BTreeMap, sample_interval: Option, stream_norms: Option, max_distinct_per_evaluation: Option, @@ -215,8 +215,8 @@ struct LabelSetDistinctValues { | `arrival` | Present; may be `Unknown` | Distinguishes data at rest, ingesting data, and mixed data. Ingestion-time builds are possible only for ingesting or mixed data. | | `rows_per_second` | With `Ingesting`/`Mixed` | Rows arriving per second over all sources, used to price scans and continuous maintenance. It must be finite and non-negative. A source with no `series_count` + `sample_interval` of its own is priced at its series share of it when every source declares `series_count`, else at all of it. | | `series_count` | Optional | Series in the source. Plan selection bounds a query's number of values `G`, and its input row count, by the series counts of the sources it reads (each branch of an `or` counted); unknown, they stay unbound. | -| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `values_non_negative` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | -| `values_non_negative` | Optional; default `false` | Declares every sample non-negative, for a gauge as well as a counter. It is the sign a relative-error bound needs to carry through `sum` and `avg` (a per-series DDSketch quantile under `avg by`, say); without it, and for a gauge, that bound is unknown. | +| `metric_type` | Optional | Declared Prometheus type (`PrometheusMetricType`): `counter`, `gauge`, `histogram`, `native_histogram` or `summary`; a counter proves samples non-negative. Never inferred from a name. A `histogram` or `summary` is keyed by its family name `foo` and supplies its series (`DataWorkload::series_facts`): `foo_bucket` (histogram) or `foo` (summary, per-series quantiles read as values), and the counters `foo_sum` and `foo_count`; any other name is looked up as is. The family entry's `metric_type`, `column_signs` and `sample_interval` apply to its series; counts, norms and bounds are read per series name. `histogram_quantile` reads float input as classic buckets, declared `histogram` or undeclared, and rejects a `native_histogram` or `summary`; any query reading a `native_histogram` is rejected. `rate`, `increase` or `irate` over a metric declared a gauge, and `sum`, `avg`, `quantile`, `stddev` or `stdvar` across a summary's quantile series, still plan, with a warning from the PromQL frontend (`metric_type_warnings`), which `asap_planner` reports. | +| `column_signs` | Optional; default none | The sign of each column's values (`non_negative`, `non_positive` or `unknown`), keyed by column name, e.g. `{"value": "non_negative"}` for a latency gauge. A `Scan`'s `column_sign` reads it; error contracts that need values of one sign (a relative error through `sum` or `avg`, DDSketch's relative error, Count-Min's estimates) read that. An undeclared column is `unknown`, except a PromQL counter's value. | | `sample_interval` | PromQL and MetricsQL; nonzero | How often each series of the metric is sampled. Bare instant selectors over it use it as their selection horizon; without it, such a query fails to lower. | | `stream_norms`, `max_distinct_per_evaluation` | Optional | Bounds that Count-Min/Count Sketch and HLL/UnivMon guarantees are bound with. | | `distinct_values_per_label` | Optional; default empty; each count nonzero | Distinct values each label (PromQL) or column (SQL) takes, e.g. `{"host": 1000, "job": 10}`. Plan selection prices a `by (k1, k2, ...)` aggregate or summary, and a partitioned sort or limit, at the product of its keys' counts, at most the `series_count` (series, or groups of rows) of the sources it reads and the rows it reads per evaluation. Over several sources a key's counts add. No count is assumed for a key some source read does not declare: the grouping is priced at the smaller of that `series_count` and those rows. Each count must be an upper bound, counting a missing label (or a NULL) as one more value: when every key of a grouped result declares one, their product (at most the series read) also bounds `G`, which sizes per-group sketches for a whole-result target; otherwise `G` keeps its series bound, or none for SQL. | @@ -248,15 +248,14 @@ They are explicit frontend function arguments, not one generic ### Planning evidence inputs **A model is planning logic; evidence is a scoped fact used by that logic.** -For example, `BuiltinAccuracyModel` knows how to combine error bounds, while -an operand-domain contract tells it whether a particular product or quotient -of absolute-error estimates has a bound. The former can be provided by a built-in default; the +For example, the accuracy rules know how to combine error bounds, while a +deployment's measurement (`AccuracyEvidence`) says what error a particular +summary configuration has. The former can be provided by a built-in default; the latter cannot be fabricated by one. | Input | Where it enters / default | Why it matters | |---|---|---| -| Accuracy model | `PlanningModels` (`AccuracyModel`; `BuiltinAccuracyModel` by default) | Composes candidate guarantees and checks them against requested accuracy. The model does not itself provide missing data-domain facts. | -| Accuracy/domain evidence | `PlanningModels` (`AccuracyEvidenceProvider`; `NoAccuracyEvidence` by default) | Input ranges, nonempty populations and similar facts can certify or rule out particular approximations. Missing facts remain unknown. | +| Accuracy evidence | `PlanningModels` (`AccuracyEvidence`; none by default) | Measured guarantees per summary configuration, read before the theoretical guarantee. Missing measurements leave the theoretical guarantee. | | Cost model | `CostInputs.cost_model` (cost model 1 by default) | Turns each valid candidate's resource usage into a choice. The illustrative unit costs are not a measured deployment cost. | | Measured cost evidence | `CostInputs.measured_operation_costs` | Measured CPU time per unit of each operator kind, at ingestion or query time, replaces the formulas for the nodes it prices (a sketch interpolated between measured sizes); the other nodes' formulas are scaled by one calibration factor fitted from the measured ones. | | Runtime capabilities | `DeploymentCapabilities` (`supported_summaries`, `raw_data_retained`, `query_time_results_retained`) | Prevents choosing a summary, raw-data scan or kept state the intended executor cannot provide. | @@ -279,21 +278,12 @@ candidate whose output has no guarantee for an approximate target. ### `PlanOutput` -`PlanOutput` holds the selected plan and Stage 3's record of the selection: - -* `plans`: one plan root per query, in `QueryWorkload::entries()` order; -* `selection.selected_dag`: the selected plan as a `DagOfSelectedCandidates`: - the physical candidate each node runs (one node per run, so a candidate - run twice is two nodes), the edge feeding each input of each node, and the - node producing each query output; -* `selection.cost`: the selected plan's resource usage and, under cost - models 1 and 2, its cost per second; -* `selection.accepted_accuracy_per_query`: per query, the accuracy metric - Stage 3 accepted; -* `selection.pruned_candidates`: the candidates Stage 3 ruled out; -* `selection.search_report`: how the search ended (`Optimal`: the MILP - runs to proven optimality, with no time limit), its lower bound, the - MILP's size and its times. +`PlanOutput` holds the selected plan, `plan`: one `PhysicalASAPDAG`, whose +`roots` answer the operator queries and `scalar_roots` the standalone scalar +queries, in `QueryWorkload::entries()` order. Its cost is +`asap_cost::plan_cost` of it. Stage 3's own report, `Selection` +(`accepted_accuracy_per_query`, `pruned_candidates` and the selection MILP's +size and times), is `run_planning`'s, beside the plan. Planner selects because it now has the inputs a backend used to own: the deployment declares its cost model, measurements and capabilities, and every diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index d8756480f..dfc46531b 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -127,7 +127,7 @@ tracked in #580. | Prepare roots | Entries in `entries()` order, the accuracy target from each entry's `requirements`; PromQL operator roots carry the series identity (`with_promql_series_identity`) where it has a realization | | Stage 1: logical ASAP-aware optimization | `pass1_memo` (Pass 1), then `generate_logical_asap_memo_candidates` (Pass 2) over every tumbling-window width | | Stage 2: physical ASAP-aware optimization | `generate_physical_asap_memo_candidates` | -| Stage 3: plan selection | `select_plan`: check accuracy and latency, price under the cost model, select one plan; the facade builds it with Stage 2's `AssignmentRealizer::realize` | +| Stage 3: plan selection | `select_plan`: select one plan under the cost model and build it (`PhysicalASAPDAG`) | `run_planning` runs these steps over the query-language frontends' roots (Stage 0) and returns `PlanningRun`: Pass 1's candidates, the Pass 2 and @@ -233,7 +233,7 @@ pass no longer runs these steps; another pass need not run any of them. | Crate | What it holds | |---|---| | `asap-types` | `ParsedWorkload` | -| `asap-plan-selection` | Stage 3: `select_plan`, `Selection`, `SearchReport`, `AccuracyCheckInputs` | +| `asap-plan-selection` | Stage 3: `select_plan`, `Selection` | | `asap-planner` *(new)* | `e2e_plan`, `UserInput`, `FrontendInput`, lowering dispatch; `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `optimize`, `PassRegistry`, `Stage1ToStage3Pass`, `run_planning` | ```text @@ -247,8 +247,8 @@ asap-planner ──┬──> asap-frontend-{sql, promql, metricsql} frontend; before it, the sole facade re-exporting more than one was `asap-devtools`, a developer-tools crate. It is also the only crate that depends on every stage, so it orchestrates them; `PlanningModels` lives in -it, and it gives each stage only its part (Stage 3 gets -`AccuracyCheckInputs` and `CostInputs`). +it, and it gives each stage only its part (Pass 2 gets the accuracy +evidence, Stage 3 `CostInputs`). --- diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 60aaa2d15..80517d7df 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -201,8 +201,10 @@ cost model. ## Composing guarantees through a DAG -`AccuracyModel` supplies local guarantees, propagation rules and target -satisfaction. Its default implementation is conservative: an unregistered +The accuracy crate supplies theoretical guarantees (`theoretical_guarantee`), +propagation rules and target satisfaction (`satisfies`); the deployment's +measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a +theoretical guarantee. The rules are conservative: an unregistered composition over approximate inputs is rejected rather than treated as exact. The principal composition rules are: @@ -264,8 +266,7 @@ configuration or candidate is needed. ## Evidence and trust boundaries -`AccuracyEvidenceProvider` supplies estimator contracts, operand domains -and propagation evidence. The evidence must cover the population to which the +`AccuracyEvidence` supplies the deployment's measured guarantees. The evidence must cover the population to which the claimed guarantee applies: sources, filters, grouping, evaluation windows and all merged tumbling windows. Evidence for a narrower population cannot silently certify a wider one. Data-workload facts are declared: the planner has no clock, so the caller @@ -377,7 +378,7 @@ The `asap_accuracy` crate separates these responsibilities: ```text accuracy/src/ ├── lib.rs # Public interfaces and unified entry point (satisfies, per-input targets) -├── model.rs # AccuracyModel, AccuracyEvidenceProvider and their built-in defaults +├── model.rs # AccuracyEvidence and producer_satisfies_logical_requirements (α^log) ├── evidence.rs # Evidence and source contracts ├── composition.rs # Error propagation across computations ├── budget.rs # Splitting a query's accuracy budget across approximate inputs @@ -408,8 +409,8 @@ Adding an estimator or composition requires: 5. Tests from target and evidence through sizing, composition and selection, including missing evidence and infeasible targets. Algorithm-specific mathematics belongs behind this common contract. It does -not require a new selection rule for every sketch. Deployment extensions to -`AccuracyModel` remain possible, but carry the same obligation to justify +not require a new selection rule for every sketch. A deployment's measured +guarantees (`AccuracyEvidence`) carry the same obligation to justify metrics, assumptions and propagation. For implementation details, see the [accuracy crate](../../../crates/accuracy/src/lib.rs), diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 5011ca6be..0eca0b93b 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -68,16 +68,18 @@ Based on our requirement, each field should contain the following information. 1. **What type of the ASAP Primitive is.** The field's type is a [`FieldDataType`](#63-schema-and-field-types-cratestypessrcirschemamodrs) (`crates/types/src/ir/schema/mod.rs`). A column is either a raw value or a summary state: - **Raw value**: `Plain(DataType)`, e.g., a number or a string. - - **Summary state**: described from coarse to fine by three levels: + - **Summary state**: described from coarse to fine by these terms: - | Level | Code | Values | KLL example | + | Term | Code | Values | KLL example | |---|---|---|---| | Family | `FieldDataType` variant | `ExactAggregate`, `Sketch`, `SummarySample`, `Wavelet`, `StatModel` | `Sketch` | - | Algorithm | `SketchAlgorithm` (other families: `ExactKind`, `SamplingKind`, …) | `Kll`, `Cms`, `Hll`, `DDSketch`, … | `Kll` | - | Parameters | `SketchParams` (other families: `ExactParams`, `SamplingParams`, …) | per algorithm | `Kll { k: 200 }` | + | Algorithm | `SketchAlgorithm` (other families: `ExactKind`, `SamplingKind`, …) | `Kll`, `Cms`, `Hll`, `DDSketch`, `HydraKll`, … | `Kll` | + | Summary type | family and algorithm: one `FieldDataType` value, the column's type | `FieldDataType::Sketch(SketchAlgorithm)`, … | `Sketch(Kll)` | + | Parameters θ | `SketchParams` (other families: `ExactParams`, `SamplingParams`, …), on the candidate's `LogicalProperties`, not in the type | per algorithm, from `parameter_list(algorithm)` | `Kll { k: 547 }` | - - For a sketch, algorithm and parameters are bundled as one `SketchKind` ([§6.4](#64-summary-family-parameters-cratestypessrcirschemasketchrs)). A sketch also records its `GroupingStrategy`: one instance per group, or one shared structure (Hydra) for all groups. - - So a quantile KLL sketch with `k = 200`, one instance per group, has the type `Sketch(SketchKind { Kll, Kll { k: 200 } }, PerSubpopulationInstance)`. Which statistics a family answers is the family-capability table's (`FamilyCapability`), not part of the type. + - The summary type is the operator's `summary_type` argument (`SummaryAgg`, the chain). θ is structure only (sizes, no sized-for error fields) and lives on the candidate, so builds that differ only in θ are candidates of one memo group. + - A shared multi-subpopulation structure (Hydra) is an algorithm of its own (`HydraKll`, `HydraCms`, `HydraCountSketch`, …), not a separate grouping dimension. + - So a quantile KLL sketch with `k = 547` has the type `Sketch(Kll)` and θ `Kll { k: 547 }`. Which statistics a summary type answers is its row of the capability table (`SummaryTypeCapability`), not part of the type. 2. **What query intent the summarized ASAP Primitive can support.** This is not stored in the field: it follows from the type in item 1. There are two kinds of intent: @@ -877,15 +879,15 @@ Every struct field and enum-variant field below has a comment saying what it hol OperatorNode §6.1 ├── operator: Operator::ASAP(ASAPOp) §6.2 │ ├── SummaryAgg -│ │ ├── family: FieldDataType ───────────┐ §6.3 → §6.4 +│ │ ├── summary_type: FieldDataType ─────┐ §6.3 → §6.4 │ │ └── input: SummaryUpdate │ §6.5 (what each row adds) │ ├── SummaryEstimate.query: SketchStatistic │ §6.5 (what is read out) │ ├── MaintainPopulation.population │ §6.5 │ └── EvaluatePopulation.evaluation │ §6.5 ├── schema: Schema │ §6.3 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output -│ └── Sketch(SketchKind, GroupingStrategy) `state` field has type `family`) -│ └── algorithm, params §6.4 +│ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) +│ (θ, `SketchParams`, is on the candidate) §6.4 ├── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) └── coverage() → SummaryCoverage §6.6 └── definition: OperatorNode (again a node, so the same structure) @@ -895,12 +897,12 @@ OperatorNode §6.1 |---|---|---| | §6.1 DAG node | `OperatorNode` | the DAG itself | | §6.2 ASAP operators | `ASAPOp` | `OperatorNode.operator` | -| §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.family` | -| §6.4 Summary-family parameters | `SketchKind`, `SketchParams`, `GroupingStrategy`, exact/sample/wavelet/model params | the payload of each non-plain `FieldDataType` variant | +| §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | +| §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | | §6.6 Summary coverage | `SummaryCoverage`, `SelectionBox`, `Constraint` | `OperatorNode::coverage()` on summary nodes | -**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `family = Sketch(SketchKind::new(Kll, Kll { k: 200 }), PerSubpopulationInstance)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5). Its `schema` (§6.3) is `(job Utf8, state Sketch(KLL k=200))`: the `state` field has the same type as `family`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). +**Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). ### 6.1 DAG node (`crates/types/src/ir/operator/node.rs`) @@ -948,14 +950,12 @@ pub enum ASAPOp> { /// The input rows. child: C, /// The summary type of the output `state` field. Never `Plain`. - family: FieldDataType, + summary_type: FieldDataType, /// What each input row adds to the state (item and weight). input: SummaryUpdate, /// The grouping: `Reduce(by columns)`, or `PerEntity` for one state per /// input series without grouping. reduction: Reduction, - /// Whether each group gets its own sketch or all groups share one. - grouping: GroupingStrategy, /// Rows to include, applied before updating the state. `None`: all rows. filter: Option>, }, @@ -1003,7 +1003,7 @@ pub enum ASAPOp> { outer: C, // one input state inner: C, // the other input state key: ColumnId, // the join key column - family: FieldDataType, // the summary type of the result + summary_type: FieldDataType, // the summary type of the result }, Extension { child: C, // the input @@ -1026,7 +1026,7 @@ impl ASAPOp { /// Whether the operator is reserved and cannot be built yet: Subtract, /// Delete, Join, Extension. pub fn is_unimplemented(&self) -> bool; - /// The summary type this operator outputs: `family` for SummaryAgg and + /// The summary type this operator outputs: `summary_type` for SummaryAgg and /// SummaryJoin, the inputs' state type for SummaryMerge, `None` otherwise. pub fn produced_state(&self) -> Option<&FieldDataType>; /// The output schema, derived from the operator and its inputs. An error @@ -1105,15 +1105,16 @@ impl Field { pub fn is_plain(&self) -> bool; } -/// A column's type: a plain value, or summary state of one family (§3). +/// A column's type: a plain value, or summary state of one summary type +/// (family and algorithm, §3). pub enum FieldDataType { /// A readable value of this type. Plain(DataType), /// Exact accumulator state: which accumulator, and its parameters. ExactAggregate(ExactKind, ExactParams), - /// Sketch state: the chosen sketch (algorithm, parameters), and - /// whether each group has its own sketch or all groups share one. - Sketch(SketchKind, GroupingStrategy), + /// Sketch state of one algorithm; θ is not part of the type. The + /// `Hydra*` algorithms share one structure across all groups. + Sketch(SketchAlgorithm), /// Summary sample state: the sampling method, and its parameters. #[serde(rename = "Sample")] SummarySample(SamplingKind, SamplingParams), @@ -1143,7 +1144,7 @@ pub enum DataType { } ``` -### 6.4 Summary-family parameters (`crates/types/src/ir/schema/sketch.rs`) +### 6.4 Summary algorithms and parameters (`crates/types/src/ir/schema/sketch.rs`) ```rust /// Which exact accumulator. None of them has parameters, so `ExactParams` @@ -1151,28 +1152,15 @@ pub enum DataType { pub enum ExactKind { Sum, Count, Min, Max, Increase, Rate, IRate } pub enum ExactParams { Sum, Count, Min, Max, Increase, Rate, IRate } -/// A chosen sketch. Built only through `new`, so the two fields always agree. -/// Which statistics it answers is its family's row of the family-capability -/// table (`FamilyCapability`). -pub struct SketchKind { - /// Which sketch algorithm. - algorithm: SketchAlgorithm, - /// That algorithm's size parameters; must match `algorithm`. - params: SketchParams, -} -impl SketchKind { - /// The only constructor. Panics if `params` belong to a different - /// algorithm. - pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self; - /// The sketch algorithm. - pub fn algorithm(&self) -> &SketchAlgorithm; - /// The algorithm's size parameters. - pub fn params(&self) -> &SketchParams; -} -// Theta is not offered as a candidate until asap_sketchlib has a Theta sketch. -pub enum SketchAlgorithm { UnivMon, Kll, Cms, Hll, DDSketch, CmsWithHeap, Kmv, Theta, CountSketch, CountSketchWithHeap } +/// `asap_sketchlib`'s algorithm enum, the Hydra algorithms (`HydraKll`, +/// `HydraCms`, `HydraCountSketch`, …) included. Which statistics each +/// answers is its row of the capability table (`SummaryTypeCapability`). +pub use asap_sketchlib::SketchAlgorithm; -/// Size parameters of each algorithm. Larger values: more memory, less error. +/// θ: the structure of each algorithm, from its predefined list +/// (`parameter_list`). Larger values: more memory, less error. No field +/// records what error a configuration was sized for; equality compares +/// every field. pub enum SketchParams { /// `layers`: number of sampling levels. Each level has a Count Sketch of /// `sketch_rows` hash rows × `sketch_cols` counters, and a heap of the @@ -1190,24 +1178,15 @@ pub enum SketchParams { CmsWithHeap { width: u32, depth: u32, heap_size: u32 }, /// `k`: number of smallest hash values kept. Kmv { k: u32 }, - /// `k`: number of hash values kept (nominal entries). - Theta { k: u32 }, /// `width`: counters per row. `depth`: number of hash rows. CountSketch { width: u32, depth: u32 }, /// Count Sketch `width` × `depth`, plus a heap of the `heap_size` heaviest items. CountSketchWithHeap { width: u32, depth: u32, heap_size: u32 }, + /// θ of a `Hydra*` algorithm. + Hydra(HydraParams), } -/// How a grouped sketch is laid out across the `by` groups. Independent of the family. -pub enum GroupingStrategy { - /// One separate sketch per group. The default. - PerSubpopulationInstance, - /// One shared structure for all groups (Hydra). `kind`: which shared - /// structure. `params`: its sizes. - SharedMultiSubpopulation { kind: HydraKind, params: HydraParams }, -} -pub enum HydraKind { HydraKll /* experimental, no error bound */, HydraCms, HydraCountSketch } -/// Sizes of the shared structure. +/// Sizes of a Hydra structure, one shared by all groups. pub enum HydraParams { /// `k`: KLL `k` each group sees. `shared_buckets`: size of the one structure /// shared by all groups. @@ -1218,11 +1197,6 @@ pub enum HydraParams { /// Same fields as `HydraCms`, for Count Sketch. HydraCountSketch { width: u32, depth: u32, shared_rows: u32, shared_columns: u32 }, } -/// The shared (Hydra) version of an algorithm, if one with a known error -/// bound exists: `Cms` → `HydraCms`, `CountSketch` → `HydraCountSketch`; -/// `None` for all others. -pub fn hydra_kind_for(a: &SketchAlgorithm) -> Option; - /// `size`: number of rows kept in the reservoir. pub enum SamplingKind { Reservoir } pub enum SamplingParams { Reservoir { size: u32 } } /// `coefficients`: number of wavelet coefficients kept. diff --git a/docs/design_docs/proposals/joint-planning-implementation-plan.md b/docs/design_docs/proposals/joint-planning-implementation-plan.md index 89b8dde7b..55831cfbd 100644 --- a/docs/design_docs/proposals/joint-planning-implementation-plan.md +++ b/docs/design_docs/proposals/joint-planning-implementation-plan.md @@ -14,14 +14,14 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc | --- | --- | --- | | Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`), stable initial query/group references. | | Computation and sharing rules | `logical-asap-optimization/src/pass2/memo/{candidates,rules,state_groups,shared_configurations,window_candidates}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | -| Physical legality | `physical-asap-optimization` | Executable operators, timing, materialization eligibility, and `PhysicalWorkloadPlan`. | -| Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning, the selected DAG (`DagOfSelectedCandidates`, one node per run of a candidate), and reconstruction. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | +| Physical legality | `physical-asap-optimization` | Executable operators, timing and materialization eligibility. | +| Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning; Stage 3 builds the selected plan (`PhysicalASAPDAG`) directly. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | | Solver | `plan-selection/src/search/milp` | Activation, producer/root assignment, reachability, acyclicity, resource constraints, and solver warm starts. | | Search and validation | `plan-selection/src/search/refinement` | Actual evaluator checks, sound cuts, objective-specific ranking, and certified component splitting. | | Pricing support | `plan-selection/src/search/costing` | Certified local bounds and consistent calibration. Production execution demand is owned by `cost/src/execution_demand.rs`; the search demand reference is test-only. | | Pipeline and lifecycle | `plan-selection/src/search/{pipeline,route}` | Initial groups, generation accounting, seed/full search budgets, archives, reports, and selected workload outputs. | -`PhysicalCandidate` is a local selectable computation. `PhysicalWorkloadPlan` is a complete Stage 2 workload DAG; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. +`PhysicalCandidate` is a local selectable computation; the plan (`PhysicalASAPDAG`) is the complete workload DAG Stage 3 builds; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. Pass 1 has one entry, `pass1_memo`, which populates the already-created memo groups (its target list, `PerComputationCandidates`, is a crate-private working form). This is not a second workload selector. Pass 2 can add intermediate state groups and consumer candidates; sharing a component does not merge distinct requested results: diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 13a1c6b07..0d195901d 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -506,12 +506,10 @@ into one plan and check and cost it with Stage 2 and Stage 3. The search is the group selection MILP (mixed-integer linear program, #1095): it chooses one candidate per memo group and execution context, with each shareable candidate charged once, and is solved once; it prices cost model 1 only (cost models 2 -and 3 are an `UnsupportedCostModel` error, #888). Stage 3 realizes the chosen -plan through Stage 2's `AssignmentRealizer`, which the facade builds and passes in, -so Stage 3 does not depend on the Stage 2 crate. The facade runs Pass 2, -Stage 2 and Stage 3 once, over the full tumbling-window-width domain, with no -time limit. Stage 3 checks that `plan_cost` of the realized plan -equals the MILP's objective, and that every query's accuracy and latency hold. +and 3 are an `UnsupportedCostModel` error, #888). Stage 3 builds the chosen +plan, a `PhysicalASAPDAG`, as its last step, so it does not depend on the +Stage 2 crate. The facade runs Pass 2, Stage 2 and Stage 3 once, over the +full tumbling-window-width domain, with no time limit. #### Memo code interface @@ -528,7 +526,7 @@ candidates is in `crates/plan-selection/src/` (`select.rs`, `groups/`, `milp/`). | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | -| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each, and builds the selected plan with Stage 2's `AssignmentRealizer::realize`. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | +| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index a1b24044c..26ad5b02e 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -17,10 +17,10 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | Crate | Role | | --- | --- | -| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `FamilyCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `PhysicalProperties` (timing and `Lifetime`), tumbling windows) | +| `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `SummaryTypeCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `PhysicalProperties` (timing and `Lifetime`), tumbling windows) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | | `asap-logical-asap-optimization` (`crates/logical-asap-optimization`) | Stage 1: Pass 1 and Pass 2 (the memo rules) | -| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-family estimators, composition, budget split, target check) and the deployment's `AccuracyModel` and `AccuracyEvidenceProvider`; depends only on `asap-types` | +| `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, composition, budget split, target check), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`producer_satisfies_logical_requirements`); depends only on `asap-types` | | `asap-physical-asap-optimization` (`crates/physical-asap-optimization`) | Stage 2: materialization and physical operator implementation | | `asap-cost` (`crates/cost`) | The cost model the stages share: resource-usage estimation (the analytical per-operator model, measured operation costs, state byte sizes) and the cost models (`ResourceUsage`, `CostModel`, `CostInputs`); depends only on `asap-types` | | `asap-plan-selection` (`crates/plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | @@ -53,14 +53,14 @@ LogicalASAPMemoCandidates │ asap_physical_asap_optimization::generate_physical_asap_memo_candidates Stage 2 ▼ PhysicalASAPMemoCandidates (operator nodes per timing/materialization + pruned_candidates) - │ asap_plan_selection::select_plan (with Stage 2's AssignmentRealizer) Stage 3 + │ asap_plan_selection::select_plan Stage 3 ▼ -Selection (selected_dag, cost, accepted_accuracy_per_query, search_report) +(PhysicalASAPDAG, Selection) — the plan, and accepted_accuracy_per_query, +pruned_candidates and the selection MILP's size and time ``` `asap_planner::run_planning` runs all of it over lowered roots: Pass 1, then -Pass 2 over every tumbling-window width, Stage 2 and Stage 3, and builds the -selected plan with `AssignmentRealizer::realize`. +Pass 2, Stage 2 and Stage 3, which builds the selected plan. It returns every stage's output (`PlanningRun`); `Stage1ToStage3Pass` is the same pipeline behind the `OptimizationPass` trait. @@ -70,14 +70,14 @@ same pipeline behind the `OptimizationPass` trait. | Which summary families serve an intent (`summary_candidates`) | `asap-logical-asap-optimization`, `pass1/realization.rs` | | Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-logical-asap-optimization`, `pass2/memo/rules/` and `pass2/memo/window_candidates.rs`, over the helpers in `pass2/asap_aware_cse.rs` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | `asap-logical-asap-optimization`, `pass2/memo/enumeration.rs` | -| What each summary family can merge and answer (`FamilyCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/family_capability.rs` | +| What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/summary_type_capability.rs` | | Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | | Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | | Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | -| The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyModel`, `BuiltinAccuracyModel`, `estimators/`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | +| The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `estimators/`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | | What the executor supports (`DeploymentCapabilities`) | `asap-logical-asap-optimization`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | -| The inputs every stage reads (`PlanningModels`: accuracy model, evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `pass.rs`; the facade gives Stage 3 only `AccuracyCheckInputs` and `CostInputs` | +| The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `pass.rs`; the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | The current production search organizes these computations in memo result/state groups and jointly selects physical candidates and connections; it does not diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 1b0e785e7..d46ee4c89 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -15,7 +15,7 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | --- | --- | --- | | Stage 0 (frontends) | `NonASAPOp` nodes only; aggregates carry `AggIntent` measures with each query's `AccuracyTarget` as written | none | | Stage 1 (`compose_logical_candidate`, Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | -| Stage 2 (`generate_physical_asap_memo_candidates`; a selected plan built by `Stage2AssignmentRealizer`) | the same operators, with a timing and a `Lifetime` on each summary build | every node timed from the candidate's `PhysicalPropertiesAssignment` | +| Stage 2 (`generate_physical_asap_memo_candidates`) and Stage 3 (`select_plan`, which builds the plan) | the same operators, with a timing and a `Lifetime` on each summary build | every node of the plan timed from the physical candidate it was built from (`selected_physical_candidate`) | - Structurally equal sub-DAGs are interned before Stage 1 (`stage0/cse.rs` in `asap_logical_asap_optimization`), so Stage 1 compares nodes by structure and Pass 2 @@ -62,11 +62,12 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types ## 3. Accuracy -- `AccuracyModel` is the extension point: `local_guarantee` (a family's - guarantee over exact input), `propagate` (composition through an operator) - and `satisfies` (a guarantee against a target). It fails closed: no rule, or - a missing input guarantee, means no guarantee, and the candidate is - rejected rather than assumed accurate. +- A summary's guarantee is the deployment's measurement + (`AccuracyEvidence::measured_guarantee`) when there is one, else its + algorithm's error formula (`theoretical_guarantee`); Pass 2 compares it + with each readout's `RequiredAccuracy` + (`producer_satisfies_logical_requirements`). It fails closed: no rule + means no guarantee, and the edge is not admissible. - A target with no `accuracy_metric` accepts a guarantee in any metric that meets its ε, δ and scope; Stage 3 reports the metric it accepted (`AcceptedAccuracy`). @@ -133,5 +134,5 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | What is shared, split into tumbling windows or reused | Pass 2 | | When each build runs and whether it is kept | Stage 2 | | Which plan is valid and which is chosen | Stage 3 | -| What the executor can build, retain and how it is priced | the deployment, through `PlanningModels` (`DeploymentCapabilities`, `CostInputs`, `AccuracyModel`) | +| What the executor can build, retain and how it is priced | the deployment, through `PlanningModels` (`DeploymentCapabilities`, `CostInputs`, `AccuracyEvidence`) | | Running the plan | the executor (`asap-executor`), outside the planner | diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index 1e4a98a05..dfc6a1ce2 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -52,23 +52,23 @@ Read the sections below when changing one of those contracts. Accuracy reasoning and allocation are separate from cost modeling: ```rust -trait AccuracyModel { - fn local_guarantee(/* family, query, parameters */) - -> Option; - - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, - ) -> Result; - - fn satisfies( +/// The theoretical guarantee of reading `statistic` from a state of +/// `summary_type` with parameters θ: the algorithm's error formula. +pub fn theoretical_guarantee( + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, +) -> Option; + +/// The deployment's measured guarantee for the same configuration, read +/// first when present. +impl AccuracyEvidence { + pub fn measured_guarantee( &self, - guarantee: &AccuracyGuarantee, - target: &AccuracyTarget, - ) -> bool; + summary_type: &FieldDataType, + parameters: &SketchParams, + statistic: &SketchStatistic, + ) -> Option; } ``` @@ -257,9 +257,9 @@ with probability at most `1/3`. Zero or even depth has no modeled guarantee. ASAPPlanner contains the guarantee algebra and parameter-derived contracts. Planner is not a query executor. Data- or runtime-dependent evidence enters -through an `AccuracyEvidenceProvider`, is exposed to propagation as typed -`PropagationStats`, and is recorded in provenance. `NoAccuracyEvidence` is the -default: it supplies no missing facts. Guarantee derivation remains conservative. +through `AccuracyEvidence` (measured guarantees) or the data workload's +declared facts, is exposed to propagation as typed `PropagationStats`, and +is recorded in provenance. The default evidence has no measurements. Guarantee derivation remains conservative. ### TopK membership diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index f59566dcd..c2ea719f2 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -10,15 +10,16 @@ owns the decision. A new sketch that Pass 1 can offer for an intent. -1. **Type.** `SketchAlgorithm` is `asap_sketchlib`'s family enum, so the - family must exist in the library first. Add its parameters to - `SketchParams` (`crates/types/src/ir/schema/sketch.rs`); a Hydra kind - maps to its library family in `FamilyCapability::of`. +1. **Type.** `SketchAlgorithm` is `asap_sketchlib`'s algorithm enum, so the + algorithm must exist in the library first; its summary type is + `FieldDataType::Sketch(algorithm)`. Add its parameters to `SketchParams` + (`crates/types/src/ir/schema/sketch.rs`) and its predefined list to + `parameter_list`. 2. **Capability.** Its row, which inputs it can merge and which statistics it answers with the merged error, is the library's family-capability table (`SketchAlgorithm::capability`), read unchanged by - `FamilyCapability::of_per_instance_sketch` - (`crates/types/src/ir/schema/family_capability.rs`). Pass 2's sharing, + `SummaryTypeCapability::of` + (`crates/types/src/ir/schema/summary_type_capability.rs`). Pass 2's sharing, tumbling window and rollup rules read only this table. Which readouts the executor evaluates is its own declaration, `EVALUATED_READOUTS` (`crates/executor/src/capability.rs`); add the family's readouts there @@ -29,8 +30,8 @@ A new sketch that Pass 1 can offer for an intent. Size it in `asap_accuracy::sizing::default_size_params`. 4. **Accuracy.** Add its error model under `crates/accuracy/src/estimators/` and return - it from `BuiltinAccuracyModel`'s `local_guarantee`, by inverting the same - sizing formulas. Without a guarantee, every candidate that uses it is + it from `theoretical_guarantee`, evaluated at the required failure + probability. Without a guarantee, every candidate that uses it is rejected. 5. **Cost.** Give it a state size (the library's `estimated_memory_bytes` for the type the executor builds; KLL keeps the planner's own formula) @@ -58,7 +59,7 @@ work. (for example `rollup.rs` or `topk.rs`): it adds candidates (and, when it needs them, groups) to the memo groups it implements, keeping every existing candidate. Read summary families only through - `FamilyCapability`. Shared rewrite helpers live in + `SummaryTypeCapability`. Shared rewrite helpers live in `pass2/asap_aware_cse.rs`. 2. Declare it in `pass2/memo/rules.rs` and call it from `apply_sharing_rules` (`pass2/logical_memo_candidates.rs`), after the diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 122eae3b8..b0f081d60 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -27,8 +27,8 @@ Inside this workspace, depend on the frontend you need, the stage crates (`asap-logical-asap-optimization` for Stage 1, `asap-physical-asap-optimization` for Stage 2, `asap-plan-selection` for Stage 3), `asap-planner` for the facade that runs the stages, `asap-cost` for the cost model and its inputs (`CostInputs`), -`asap-accuracy` for the accuracy model (`AccuracyModel`, -`AccuracyEvidenceProvider`), and `asap-types`, or on `asap-planner`, which wraps every frontend and +`asap-accuracy` for the accuracy rules and the deployment's measured +accuracy (`AccuracyEvidence`), and `asap-types`, or on `asap-planner`, which wraps every frontend and the planning stages. External users can use Git dependencies pinned to a compatible revision; use the same revision across these crates, and pick a revision at least as new as the APIs you call. Replace `REV` below with a @@ -187,26 +187,21 @@ asap_plan_selection::select_plan( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], data: &DataWorkload, - accuracy_checks: &AccuracyCheckInputs<'_>, // accuracy model and evidence cost_inputs: &CostInputs, - realizer: &dyn AssignmentRealizer, // Stage 2's, e.g. Stage2AssignmentRealizer -) -> Result +) -> Result<(PhysicalASAPDAG, Selection), ErrorSelectingPlan> ``` | Input or output | Meaning | | --- | --- | -| `PlanningModels::new(accuracy, evidence)` | Your `&dyn AccuracyModel` and `&dyn AccuracyEvidenceProvider`, with default cost inputs and deployment capabilities; `with_accuracy` and `with_evidence` replace one of them | -| `PlanningModels::builtin()` | Built-in accuracy model (`BuiltinAccuracyModel`) and no extra evidence (`NoAccuracyEvidence`). `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | +| `PlanningModels::new(evidence)` | Your `&AccuracyEvidence` (measured guarantees), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | +| `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | | `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | -| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::price_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | +| `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | | `CostModel::ClusterCapacity(ClusterLimits)` | Cost model 3: memory, disk and optional CPU cores are hard limits; `ClusterObjective::{LowestLatency, LeastMemory}` chooses among the candidates that fit | -| `PlanOutput.plans` | One `QueryPlan { entry_index, root }` per query, in `QueryWorkload::entries()` order | -| `PlanOutput.selection` | `Some` with Stage 3's `Selection` under the default pass: `selected_dag`, `nodes_with_proven_non_negative_weights`, `cost` (the selected plan's `PlanCost` with its `ResourceUsage` and `cost_per_second`, `None` under cost model 3), `accepted_accuracy_per_query`, `pruned_candidates` and `search_report` | -| `PlanOutput.tumbling_window_retention_ms` | For each tumbling-window chain the selected plan keeps beyond one evaluation, by node id, how long its tumbling windows are kept (`RealizedPlan::tumbling_window_retention_ms`) | -| `PlanningRun` | `per_computation_candidates`, `logical_asap_candidates`, `physical_asap_candidates`, `selection`, `selected_physical_plan` (`RealizedPlan`), `selected_plan` (the roots to run) and `report` (Pass 2's and Stage 2's counts and each stage's time); `per_computation_candidates` is kept only with `PlanSearchSettings::inspect`; `asap_planner::selected_chosen_candidates` reads the selected plan as one candidate per group of `logical_asap_candidates.memo` | -| `PlanOutput::physical_asap_dag()` | The planned queries as one `PhysicalASAPDAG`, timed with the default `PhysicalPropertiesAssignment` (see [Export](#export)) | +| `PlanOutput.plan` | The selected plan, one `PhysicalASAPDAG`: `roots` answer the operator queries and `scalar_roots` the standalone scalar queries, in `QueryWorkload::entries()` order; each materialized node carries its `retention_ms`. Its cost is `asap_cost::plan_cost` of it | +| `PlanningRun` | `pass1_memo`, `logical_asap_candidates`, `physical_asap_candidates`, `selection` (the plan and Stage 3's `Selection`: `accepted_accuracy_per_query`, `pruned_candidates`, `milp_variables`, `milp_constraints`, `milp_construction_ms`, `milp_solve_ms`) and `report` (Pass 2's and Stage 2's counts and each stage's time); `pass1_memo` is kept only with `PlanSearchSettings::inspect` | Each candidate's `ResourceUsage` holds its usage without prices: `cpu_ops_per_second`, `scanned_bytes_per_second`, `average_memory_bytes`, @@ -310,7 +305,7 @@ query's window comes from its text (range and offset). These facts are separate: `DataWorkload` holds `arrival` (`DataArrival`: `AtRest`, `Ingesting` or `Mixed` with `rows_per_second`, or `Unknown`) and `sources`, a map from PromQL metric or SQL table name to `SourceFacts`: `series_count`, -`metric_type`, `values_non_negative`, `sample_interval` (PromQL and +`metric_type`, `column_signs`, `sample_interval` (PromQL and MetricsQL), `stream_norms`, `max_distinct_per_evaluation` and `distinct_values_per_label` (distinct values per label or column, which plan selection prices a `by (...)` grouping's groups at: the product of its diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json index f6d18e95d..c53e1cc4d 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-1h-60s-300s.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json index cbbb225a3..cbcd46e14 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q05-q09-1h.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json index 0836b9c93..0666ce616 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t5-q09-15m-1h.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json index 35ab437f7..f1b9fe3a0 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t7-t4-1h.json @@ -53,7 +53,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json b/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json index c3b6d73b5..47eab5eb9 100644 --- a/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json +++ b/tools/dag-viewer/memo-examples/promql/rangeheavy-t8-t5-1h.json @@ -72,7 +72,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json b/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json index af180ad3b..ddf8d3755 100644 --- a/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json +++ b/tools/dag-viewer/memo-examples/promql/ratio-of-quantiles-5m.json @@ -33,7 +33,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 5 } } diff --git a/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json b/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json index 9da4f70c9..61566c302 100644 --- a/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json +++ b/tools/dag-viewer/memo-examples/sql/rollupheavy-t11-service-endpoint-5m.json @@ -55,7 +55,7 @@ "http": { "series_count": 10000, "sample_interval": 5, - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "max_distinct_per_evaluation": 1000000 } } diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index d30b2c307..dcd6f5842 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -320,7 +320,6 @@ function uniqueIds(list, where, errors) { // it is not priced. const FAILED_CHECK_CAUSES = [ 'accuracy_target_missed', - 'count_min_weights_not_proven_non_negative', 'no_accuracy_guarantee', 'latency_bound_exceeded', 'latency_over_repeat_interval', 'over_cluster_capacity', 'infeasible', 'cost_not_representable', ]; @@ -1035,7 +1034,7 @@ function dataWorkloadView(dataWorkload, queries) { name, series: typeof facts.series_count === 'number' ? facts.series_count : '—', type: facts.metric_type || '—', - nonNegative: facts.values_non_negative ? 'yes' : String(facts.metric_type).toLowerCase() === 'counter' ? 'yes (counter)' : '—', + nonNegative: (facts.column_signs || {}).value === 'non_negative' ? 'yes' : String(facts.metric_type).toLowerCase() === 'counter' ? 'yes (counter)' : '—', sampleInterval: facts.sample_interval === undefined || facts.sample_interval === null ? '—' : durationText(facts.sample_interval), streamNorms: norms(facts.stream_norms), maxDistinct: typeof facts.max_distinct_per_evaluation === 'number' ? facts.max_distinct_per_evaluation : '—', diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json index 05a8bddd0..8480e3aa9 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-01.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json index a32fb624e..23fde7efe 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-02.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json index 42cb5cebb..90ad67f7c 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-03.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json index e7a49f676..42fcd0d1b 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-04.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json index 97d5c8a7f..4693f5b31 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-05.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json index bad5fb16d..15574d528 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-06.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json index 690d9b987..b79ccfab7 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-07.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json index 951365d04..ded627a08 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-08.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json index 9c55b9869..1e5bbd1df 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-09.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json index 50ec85402..4937890e3 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-long/workload-10.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json index 3c6a1258d..f61726db0 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-01.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json index 29bc69aaa..85913b6d4 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-02.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json index 563bf1baf..162dafc10 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-03.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json index 176dca915..2ab1953f1 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-04.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json index 0100b98c7..5a604e05c 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-05.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json index 36f4f9cef..fc267e744 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-06.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json index 55e7850df..5de0750f0 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-07.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json index 0883c7b99..3ed4aa86d 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-08.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json index ba5415bde..b04a04c86 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-09.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json index 9bcfede93..0830cf521 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862-x5/workload-10.json @@ -186,7 +186,7 @@ "data": { "series_count": 50000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json index 966256bd4..838970ad6 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-01.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json index 099306fe2..928638d74 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-02.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json index 662ddbdf5..2b397b08b 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-03.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json index f6a18db5d..25c3318f9 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-04.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json index e1e63aa36..da1813dc4 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-05.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json index cd0c15a80..99e20ad82 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-06.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json index 771acf36c..f88ce4219 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-07.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json index 9db1b41d3..dfcebffd6 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-08.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json index c95ec5037..8fdcc8a7d 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-09.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null diff --git a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json index adc66bfdd..2773378c6 100644 --- a/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json +++ b/tools/dag-viewer/workloads/random-mixed-seed-862/workload-10.json @@ -186,7 +186,7 @@ "data": { "series_count": 10000, "metric_type": "gauge", - "values_non_negative": true, + "column_signs": {"value": "non_negative"}, "sample_interval": 100, "stream_norms": null, "max_distinct_per_evaluation": null From d82c1492e671bbe0cf5ea6d3fd0b231fd7ad1704 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:55:30 +0000 Subject: [PATCH 18/30] refactor(types): guarantees at a failure probability, AccuracyEvidence trait, MemoQuery, evaluation grids of groups (#1207 PR 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-11-09 decisions: - theoretical_guarantee and AccuracyEvidence::measured_guarantee take the failure probability and return the error bound that holds at it (None when θ cannot fail that rarely); bodies stay todo!() for PR 4. - AccuracyEvidence is a trait; NoMeasuredGuarantees has no measurement. PlanningModels::evidence and Pass 2 take &dyn AccuracyEvidence. - The memo holds its queries in one Vec { root, accuracy_target, values_per_evaluation }; add_query(query). - SupportedSummary::family is summary_type. - satisfies is deleted with metric_meets, scope_meets, native_target and AccuracyScope::covers, which only it used; target_limit is the comparison limit with the tolerance. - PhysicalCandidate::delivers_on(&PhysicalASAPMemoCandidates, context). - GroupProperties::depends_on_evaluation_time (derived in PR 2) and LogicalASAPMemoCandidates::evaluation_grid_of_group (filled in PR 4). --- crates/accuracy/src/composition.rs | 2 +- crates/accuracy/src/estimators/cms.rs | 8 +- .../accuracy/src/estimators/count_sketch.rs | 8 +- crates/accuracy/src/estimators/hll.rs | 12 +- crates/accuracy/src/estimators/kll.rs | 13 +- crates/accuracy/src/estimators/mod.rs | 19 +- crates/accuracy/src/estimators/univmon.rs | 27 +- crates/accuracy/src/lib.rs | 300 ++---------------- crates/accuracy/src/model.rs | 34 +- crates/devtools/src/bin/asap_planner.rs | 2 +- crates/devtools/src/planner_fuzz/mod.rs | 2 +- crates/devtools/src/planning_stages_text.rs | 4 +- crates/devtools/tests/asap_planner.rs | 4 +- crates/executor/src/supported_summaries.rs | 6 +- crates/integration-tests/src/lib.rs | 3 +- .../src/pass1/per_computation_candidates.rs | 4 +- .../src/pass2/logical_memo_candidates.rs | 4 +- crates/planner/src/installable.rs | 2 +- crates/planner/src/pass.rs | 11 +- crates/types/src/deployment_capabilities.rs | 34 +- crates/types/src/ir/properties/accuracy.rs | 31 -- crates/types/src/ir/properties/guarantee.rs | 5 +- .../src/ir/schema/summary_type_capability.rs | 10 +- crates/types/src/stages/candidates.rs | 45 +-- .../src/stages/logical_memo_candidates.rs | 9 +- crates/types/src/stages/memo.rs | 26 +- crates/types/src/stages/mod.rs | 2 +- .../src/stages/physical/memo_candidates.rs | 23 +- .../evidence-dependent-candidates.md | 4 +- docs/design_docs/concepts/accuracy-models.md | 4 +- .../end-to-end-accuracy-guarantees.md | 15 +- docs/develop_docs/library-api.md | 4 +- tools/dag-viewer/stages.js | 2 +- tools/dag-viewer/workload-sets.js | 2 +- 34 files changed, 204 insertions(+), 477 deletions(-) diff --git a/crates/accuracy/src/composition.rs b/crates/accuracy/src/composition.rs index 3776f485b..6c3d249d2 100644 --- a/crates/accuracy/src/composition.rs +++ b/crates/accuracy/src/composition.rs @@ -1135,7 +1135,7 @@ mod tests { s, GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" ))); - assert!(!satisfies(&out, &AccuracyTarget::max_error(1.0))); + assert!(!meets(&out, 1.0, None)); let known = PropagationStats { input_row_count: Some(4), diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/estimators/cms.rs index 629f3d583..11f78d4ac 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/estimators/cms.rs @@ -54,13 +54,7 @@ mod tests { ) .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.001) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L1) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); + assert!(meets(&g, 0.01, Some(0.001))); } #[test] diff --git a/crates/accuracy/src/estimators/count_sketch.rs b/crates/accuracy/src/estimators/count_sketch.rs index 433803db3..5cda666b1 100644 --- a/crates/accuracy/src/estimators/count_sketch.rs +++ b/crates/accuracy/src/estimators/count_sketch.rs @@ -172,13 +172,7 @@ mod tests { guarantee.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L2 ); - assert!(satisfies( - &guarantee, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::ABSOLUTE_STREAM_L2) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); + assert!(meets(&guarantee, 0.01, Some(0.01))); } /// The depth is the smallest odd one whose binomial tail meets δ. Each diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/estimators/hll.rs index 25ba433cd..efaef2a51 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/estimators/hll.rs @@ -486,15 +486,7 @@ mod tests { .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Relative) - )); - assert!(!satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative) - )); + assert!(meets(&g, 0.01, None)); + assert!(!meets(&g, 0.01, Some(0.01))); } } diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index ccbf54de0..a0303c103 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -159,18 +159,9 @@ mod tests { ) .unwrap(); assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01).accuracy_metric(AccuracyMetric::Rank) - )); + assert!(meets(&g, 0.01, None)); assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(satisfies( - &g, - &AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Rank) - .accuracy_scope(asap_types::ir::properties::AccuracyScope::PerValue) - )); + assert!(meets(&g, 0.01, Some(0.01))); assert_eq!(g.approximate_layer_count(), 1); assert!(g.provenance.iter().any(|source| matches!( source, diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 21649811d..3f4045d98 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -57,21 +57,24 @@ fn bounded_guarantee( ) } -/// The theoretical guarantee of reading `statistic` from a state of -/// `summary_type` with parameters `parameters`: the algorithm's error -/// formula (concepts doc, section 2, "Where the guarantee comes from"). -/// `None` when the summary type does not answer `statistic` (its -/// capability row) or has no error model (`SummarySample`, `Wavelet`, -/// `StatModel`, and sketch readouts without a certified bound). +/// The theoretical error bound of reading `statistic` from a state of +/// `summary_type` with parameters `parameters` at failure probability +/// `failure_probability` (the guarantee's own failure probability): the +/// algorithm's error formula (concepts doc, section 2, "Where the guarantee +/// comes from"). `None` when this θ cannot fail that rarely, when the +/// summary type does not answer `statistic` (its capability row) or has no +/// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch +/// readouts without a certified bound). pub fn theoretical_guarantee( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, + failure_probability: f64, ) -> Option { match summary_type { FieldDataType::Sketch(_) if SummaryTypeCapability::of(summary_type).answers(statistic) => { - let _ = parameters; - todo!("#1207 PR 4: the algorithm's error formula at θ, at the required failure probability") + let _ = (parameters, failure_probability); + todo!("#1207 PR 4: the algorithm's error formula at θ, at failure_probability") } _ => None, } diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index fac4a3951..2bf1d8a56 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -367,16 +367,6 @@ mod tests { ) } - fn meets(guarantee: &AccuracyGuarantee, epsilon: f64, delta: f64) -> bool { - satisfies( - guarantee, - &crate::native_target( - &AccuracyTarget::max_error(epsilon).failure_probability(delta), - guarantee, - ), - ) - } - fn with(params: &SketchParams, f: impl FnOnce(&mut u32, &mut u32, &mut u32)) -> SketchParams { let mut params = params.clone(); let SketchParams::UnivMon { @@ -410,17 +400,17 @@ mod tests { let guarantee = l2(¶ms).expect("an L2 guarantee"); assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); assert!( - meets(&guarantee, eps, delta), + meets(&guarantee, eps, Some(delta)), "({eps}, {delta}): {params:?}" ); let fewer_rows = with(¶ms, |_, rows, _| *rows -= 1); assert!( - l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, delta)), + l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, Some(delta))), "({eps}, {delta}): {fewer_rows:?}" ); let fewer_cols = with(¶ms, |_, _, cols| *cols -= 1); assert!( - l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, delta)), + l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, Some(delta))), "({eps}, {delta}): {fewer_cols:?}" ); } @@ -471,7 +461,10 @@ mod tests { hash_independent(sketch_rows, sketch_cols, layers), "{params:?}" ); - assert!(!meets(&l2(¶ms).unwrap(), eps, delta), "{params:?}"); + assert!( + !meets(&l2(¶ms).unwrap(), eps, Some(delta)), + "{params:?}" + ); } } @@ -505,12 +498,12 @@ mod tests { let guarantee = bind(¶ms, max_distinct); assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); assert!( - meets(&guarantee, eps, delta), + meets(&guarantee, eps, Some(delta)), "({eps}, {delta}): {params:?}" ); let smaller = with(¶ms, |heap, _, _| *heap -= 1); assert!( - !meets(&bind(&smaller, max_distinct), eps, delta), + !meets(&bind(&smaller, max_distinct), eps, Some(delta)), "({eps}, {delta}): {smaller:?}" ); } @@ -535,7 +528,7 @@ mod tests { let params = size_params(&default_cardinality(), 0.05, 0.01); let guarantee = distinct_guarantee(¶ms, 0.01, "m").unwrap(); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); - assert!(!meets(&guarantee, 0.05, 0.01)); + assert!(!meets(&guarantee, 0.05, Some(0.01))); let SketchParams::UnivMon { heap_size: k, .. } = params else { unreachable!() }; diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 817710727..9c480976d 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -1,7 +1,7 @@ //! `asap-accuracy` — the accuracy rules the planning stages share: the //! planner's built-in analytical rules (estimators, composition, guarantee -//! propagation through a DAG, sizing, budget split and target check), the -//! deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ +//! propagation through a DAG, sizing, budget split and a target's comparison +//! limit [`target_limit`]), the deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ //! of a logical edge ([`producer_satisfies_logical_requirements`]). //! //! Estimator models derive theoretical guarantees, composition propagates @@ -28,14 +28,11 @@ pub mod test_support; pub use budget::per_input_target; pub use estimators::theoretical_guarantee; pub(crate) use evidence::PropagationStats; -pub use model::{producer_satisfies_logical_requirements, AccuracyEvidence}; +pub use model::{producer_satisfies_logical_requirements, AccuracyEvidence, NoMeasuredGuarantees}; -#[cfg(test)] -use asap_types::ir::properties::AccuracyScope; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::properties::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, - GuaranteeSource, ProbabilityExpr, + AccuracyError, AccuracyGuarantee, AccuracyMetric, AccuracyTarget, BoundExpr, + CompositionOperator, GuaranteeSource, ProbabilityExpr, }; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; @@ -44,281 +41,38 @@ use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, Sketc /// by floating-point noise. const SATISFACTION_TOLERANCE: f64 = 1e-9; -/// The built-in conservative check of whether `guarantee` meets `target`: -/// a bound within its ε, and within its δ when it states one, over the -/// values the target's scope covers (a per-value δ does not meet a -/// whole-result target). When the target sets a metric, the bound must be -/// in that metric (or one with a sound direct comparison): a bound in -/// another metric does not satisfy, however small. When it sets none, a -/// bound in any metric does, each compared in its own metric. Unknown -/// required dimensions fail. -pub fn satisfies(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - let within = |value: Option, limit: f64| { - value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) - }; - if target.is_exact() || guarantee.is_exact() { - return guarantee.is_exact(); - } - target - .accuracy_metric - .is_none_or(|metric| metric_meets(guarantee.spec.accuracy_metric, metric)) - && scope_meets(guarantee, target) - && within(guarantee.spec.error.evaluate(), target.max_error) - && target - .failure_probability - .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) -} - -/// Whether an error bound stated in `guarantee` bounds the error `target` -/// states with the same ε, so the two numbers compare directly. The same -/// metric does; so does an L2-scaled bound for an L1-scaled target, since -/// both are scaled by the norm of the one stream the root reads and -/// `‖f‖₂ ≤ ‖f‖₁`. Any other pair needs a conversion backed by evidence -/// (Absolute ↔ Relative needs a bound on the true value's magnitude; a rank -/// error has no distribution-free value error), which the default model -/// does not make: it does not satisfy. The scope is compared separately -/// ([`scope_meets`]). -pub(crate) fn metric_meets(guarantee: AccuracyMetric, target: AccuracyMetric) -> bool { - guarantee == target - || (guarantee == AccuracyMetric::ABSOLUTE_STREAM_L2 - && target == AccuracyMetric::ABSOLUTE_STREAM_L1) -} - -/// Whether `guarantee`'s failure probability covers the values `target`'s -/// does ([`AccuracyScope::covers`](asap_types::ir::properties::AccuracyScope::covers)). A whole-result guarantee covers every -/// target, and one over the groups holding at least a share of the records -/// covers a target over a larger share (not a per-value one, whose value may -/// be a smaller group). A per-value one covers only a per-value target: it must first be stated over the -/// target's scope ([`AccuracyGuarantee::over_scope`]). The scope bounds -/// only the failure probability, so it does not matter for a target -/// without a δ requirement or for a guarantee that never fails. -fn scope_meets(guarantee: &AccuracyGuarantee, target: &AccuracyTarget) -> bool { - guarantee.spec.accuracy_scope.covers(target.accuracy_scope) - || target.failure_probability.is_none() - || guarantee.spec.failure_probability.is_zero() +/// The largest evaluated error (or failure probability) that meets a +/// required `limit`: `limit` widened by [`SATISFACTION_TOLERANCE`], so a +/// bound equal to it up to floating-point noise meets it. +pub fn target_limit(limit: f64) -> f64 { + limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON } -/// `target` restated in `guarantee`'s metric, per value: the check of one -/// summary against the target it was sized for. A summary is sized in its -/// own native terms (#690 §4) and for each value it answers; an aggregate -/// intent's metric and scope are not checked: only a query root's are, by -/// plan selection. +/// Whether `guarantee`'s evaluated error is within `max_error` and, when +/// `failure_probability` is set, its evaluated failure probability within +/// it ([`target_limit`]); unknown values do not meet. #[cfg(test)] -pub(crate) fn native_target( - target: &AccuracyTarget, +pub(crate) fn meets( guarantee: &AccuracyGuarantee, -) -> AccuracyTarget { - target - .clone() - .accuracy_metric(guarantee.spec.accuracy_metric) - .accuracy_scope(AccuracyScope::PerValue) + max_error: f64, + failure_probability: Option, +) -> bool { + let within = |value: Option, limit: f64| value.is_some_and(|v| v <= target_limit(limit)); + within(guarantee.spec.error.evaluate(), max_error) + && failure_probability + .is_none_or(|delta| within(guarantee.spec.failure_probability.evaluate(), delta)) } #[cfg(test)] mod tests { use super::*; - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - #[test] - fn satisfies_is_fail_closed_on_unknowns_and_exact() { - let mut unknown = abs(0.0, 0.0); - unknown.spec.error = BoundExpr::Unknown { - statistic: "x".into(), - }; - assert!(!satisfies(&unknown, &AccuracyTarget::max_error(1.0))); - assert!(!satisfies(&abs(0.0, 0.01), &AccuracyTarget::exact())); - assert!(satisfies( - &AccuracyGuarantee::exact("x"), - &AccuracyTarget::exact() - )); - } - - fn guarantee(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - - /// A bound in another metric does not satisfy a target, however small - /// its number: e.g. a Count-Min L1-scaled frequency bound never meets a - /// distinct count's relative target, and a rank bound never meets a - /// value target. An L2-scaled bound meets an L1-scaled target of the - /// same stream (`‖f‖₂ ≤ ‖f‖₁`), not the reverse; and an exact value - /// meets every metric. - #[test] - fn satisfies_requires_the_targets_metric() { - let target = |metric| { - AccuracyTarget::max_error(0.5) - .failure_probability(0.5) - .accuracy_metric(metric) - .accuracy_scope(AccuracyScope::PerValue) - }; - let (l1, l2) = ( - AccuracyMetric::ABSOLUTE_STREAM_L1, - AccuracyMetric::ABSOLUTE_STREAM_L2, - ); - for (bound, wanted, meets) in [ - (l1, l1, true), - (l2, l2, true), - (l2, l1, true), - (l1, l2, false), - (l1, AccuracyMetric::Relative, false), - (l1, AccuracyMetric::ABSOLUTE, false), - (AccuracyMetric::ABSOLUTE, l1, false), - (AccuracyMetric::Relative, AccuracyMetric::ABSOLUTE, false), - (AccuracyMetric::ABSOLUTE, AccuracyMetric::Relative, false), - (AccuracyMetric::Rank, AccuracyMetric::Relative, false), - (AccuracyMetric::Relative, AccuracyMetric::Rank, false), - (AccuracyMetric::Rank, AccuracyMetric::Rank, true), - ] { - assert_eq!( - satisfies(&guarantee(bound, 0.01, 0.01), &target(wanted)), - meets, - "{bound:?} against {wanted:?}" - ); - } - assert!(satisfies( - &AccuracyGuarantee::exact("x"), - &target(AccuracyMetric::Rank) - )); - } - - /// A target without a δ requirement checks only the error, even when - /// the guarantee's failure probability is unknown. - #[test] - fn a_target_without_delta_ignores_the_failure_probability() { - let mut g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.0); - g.spec.failure_probability = ProbabilityExpr::Unknown { - statistic: "x".into(), - }; - assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); - assert!(!satisfies( - &g, - &AccuracyTarget::max_error(0.01).failure_probability(0.5) - )); - } - - /// A per-value guarantee does not meet a whole-result δ target - /// directly: over `G` output values it holds for all of them with - /// δ_whole = G · δ_per ([`AccuracyGuarantee::over_scope`]), and an unbound `G` - /// leaves it unmet. A whole-result guarantee meets a per-value target - /// directly. - #[test] - fn per_value_guarantees_meet_whole_result_targets_over_g_values() { - let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); - assert_eq!(g.spec.accuracy_scope, AccuracyScope::PerValue); - let whole = AccuracyTarget::max_error(0.01).failure_probability(0.01); - assert_eq!(whole.accuracy_scope, AccuracyScope::WholeResult); - assert!(!satisfies(&g, &whole)); - let per_value = whole.clone().accuracy_scope(AccuracyScope::PerValue); - assert!(satisfies(&g, &per_value)); - let over = |values| g.over_scope(AccuracyScope::WholeResult, values); - assert_eq!( - over(Some(10)).spec.accuracy_scope, - AccuracyScope::WholeResult - ); - assert_eq!( - over(Some(10)).spec.failure_probability.evaluate(), - Some(0.01) - ); - assert!(satisfies(&over(Some(10)), &whole)); - assert!(!satisfies(&over(Some(11)), &whole)); - assert!(!satisfies(&over(None), &whole)); - assert!(satisfies(&over(Some(10)), &per_value)); - } - - /// A per-value guarantee does not meet a target on the groups holding at - /// least 5% of the records directly; stated over that scope (at most 20 - /// groups) it does when 20 · δ_per is within the target's δ. A guarantee - /// over a smaller share (more groups) meets it; one over a larger share - /// does not. - #[test] - fn a_record_share_target_is_met_over_one_over_share_groups() { - let share = AccuracyScope::GroupsWithRecordShareAtLeast; - let g = guarantee(AccuracyMetric::Relative, 0.05, 0.05 / 20.0); - let five_percent = AccuracyTarget::max_error(0.05) - .failure_probability(0.05) - .accuracy_scope(share(0.05)); - assert!(!satisfies(&g, &five_percent)); - assert!(satisfies(&g.over_scope(share(0.05), None), &five_percent)); - assert!(!satisfies(&g.over_scope(share(0.04), None), &five_percent)); - let one_percent_g = guarantee(AccuracyMetric::Relative, 0.05, 0.05 / 100.0); - assert!(satisfies( - &one_percent_g.over_scope(share(0.01), None), - &five_percent - )); - let over_five = g.over_scope(share(0.05), None); - assert!(!satisfies( - &over_five, - &five_percent.clone().accuracy_scope(share(0.01)) - )); - assert!(!satisfies( - &over_five, - &five_percent.clone().accuracy_scope(AccuracyScope::PerValue) - )); - } - - /// The scope bounds only the failure probability: a target without a δ - /// requirement, or a guarantee that never fails, meets a whole-result - /// target whatever the number of values. - #[test] - fn scope_matters_only_with_a_failure_probability() { - let g = guarantee(AccuracyMetric::ABSOLUTE, 0.01, 0.001); - assert!(satisfies(&g, &AccuracyTarget::max_error(0.01))); - let never_fails = AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Zero, - vec![], - ); - let whole = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_metric(AccuracyMetric::Relative); - assert!(satisfies(&never_fails, &whole)); - assert!(satisfies( - &never_fails.over_scope(AccuracyScope::WholeResult, None), - &whole - )); - } - /// A target whose metric the user did not set accepts a guarantee in - /// any metric whose own ε, δ and scope meet the target's numbers: a - /// rank, a relative or an absolute bound at any scale. A larger bound - /// or failure probability still misses it. + /// A bound equal to the limit up to floating-point noise meets it; a + /// bound one part in a million above it does not. #[test] - fn a_target_without_a_metric_accepts_any_metric() { - let target = AccuracyTarget::max_error(0.01) - .failure_probability(0.01) - .accuracy_scope(AccuracyScope::PerValue); - assert_eq!(target.accuracy_metric, None); - for metric in [ - AccuracyMetric::ABSOLUTE, - AccuracyMetric::ABSOLUTE_STREAM_L1, - AccuracyMetric::ABSOLUTE_STREAM_L2, - AccuracyMetric::Relative, - AccuracyMetric::Rank, - ] { - assert!( - satisfies(&guarantee(metric, 0.01, 0.01), &target), - "{metric:?}" - ); - assert!( - !satisfies(&guarantee(metric, 0.02, 0.01), &target), - "{metric:?}" - ); - assert!( - !satisfies(&guarantee(metric, 0.01, 0.02), &target), - "{metric:?}" - ); - } + fn target_limit_admits_only_floating_point_noise() { + assert!(0.01 * (1.0 + 1e-12) <= target_limit(0.01)); + assert!(0.01 * (1.0 + 1e-6) > target_limit(0.01)); + assert!(0.0 <= target_limit(0.0)); } } diff --git a/crates/accuracy/src/model.rs b/crates/accuracy/src/model.rs index 992688d14..26249ce64 100644 --- a/crates/accuracy/src/model.rs +++ b/crates/accuracy/src/model.rs @@ -9,21 +9,33 @@ use asap_types::stages::{CandidateForMemoGroupId, Memo}; /// else [`theoretical_guarantee`](crate::estimators::theoretical_guarantee); /// the source is decided before Pass 2 builds the logical edges (concepts /// doc, section 2, "Where the guarantee comes from"). -#[derive(Debug, Clone, Default)] -pub struct AccuracyEvidence {} - -impl AccuracyEvidence { - /// The measured guarantee of reading `statistic` from a state of - /// `summary_type` with parameters `parameters`; `None` when the - /// deployment has no measurement for it. No measurement source exists - /// yet, so every configuration has none. - pub fn measured_guarantee( +pub trait AccuracyEvidence { + /// The measured error bound of reading `statistic` from a state of + /// `summary_type` with parameters `parameters` at failure probability + /// `failure_probability` (the guarantee's own failure probability); + /// `None` when the deployment has no measurement for it, or when this θ + /// cannot fail that rarely. + fn measured_guarantee( &self, summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, + failure_probability: f64, + ) -> Option; +} + +/// Evidence with no measurement: every guarantee is the theoretical one. +#[derive(Debug, Clone, Copy)] +pub struct NoMeasuredGuarantees; + +impl AccuracyEvidence for NoMeasuredGuarantees { + fn measured_guarantee( + &self, + _summary_type: &FieldDataType, + _parameters: &SketchParams, + _statistic: &SketchStatistic, + _failure_probability: f64, ) -> Option { - let _ = (summary_type, parameters, statistic); None } } @@ -37,7 +49,7 @@ pub fn producer_satisfies_logical_requirements( producer: CandidateForMemoGroupId, consumer: CandidateForMemoGroupId, input_index_of_consumer: usize, - evidence: &AccuracyEvidence, + evidence: &dyn AccuracyEvidence, ) -> bool { let _ = (memo, producer, consumer, input_index_of_consumer, evidence); todo!("#1207 PR 4: the readout rule against RequiredAccuracy, else the metric rule") diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index ca50f72e8..6e3fa25bb 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -765,7 +765,7 @@ fn memo_groups_json(table: &mut MemoNodes, memo: &Memo, previous: Option<&Memo>) "children": group.expression.children().into_iter().map(|g| g.0).collect::>(), "parents": ids(group.parent_groups()), "queries": group.reached_by_queries(), - "roots_of": queries_rooted_at(&memo.query_roots, MemoGroupId(id as u32)), + "roots_of": queries_rooted_at(&memo.queries, MemoGroupId(id as u32)), "node": node, "candidate_count": candidates.len(), "candidate_count_by_origin": by_origin, diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index eee772af7..e01357c2f 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -207,7 +207,7 @@ impl FuzzWorkload { .filter(|supported| { self.sketch_families .as_ref() - .is_none_or(|only| only.contains(&supported.family)) + .is_none_or(|only| only.contains(&supported.summary_type)) }) .collect(), ), diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index 30478520d..cf8e1bf66 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -209,9 +209,9 @@ fn write_memo(out: &mut String, memo: &Memo, previous: Option<&Memo>, with_candi todo!("#1207 PR 9: print each candidate's expression, logical properties and input requirements") } let roots: Vec = memo - .query_roots + .queries .iter() - .map(|root| match root { + .map(|query| match &query.root { QueryRoot::Operator(group) => format!("G{}", group.0), scalar => format!("{scalar:?}"), }) diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 40d5d73e7..611b29c60 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -919,14 +919,14 @@ fn deployment_capabilities_flag_sets_the_capabilities_and_shows_them() { }) ); let kll_kept = with_capabilities( - r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}], + r#"{"supported_summaries": [{"summary_type": "Kll", "readouts": ["Quantile"]}], "query_time_results_retained": true}"#, &["--ignore-accuracy-guarantees"], ); assert_eq!( kll_kept["deployment_capabilities"], json!({ - "supported_summaries": [{ "family": "Kll", "readouts": ["Quantile"] }], + "supported_summaries": [{ "summary_type": "Kll", "readouts": ["Quantile"] }], "raw_data_retained": true, "query_time_results_retained": true, }) diff --git a/crates/executor/src/supported_summaries.rs b/crates/executor/src/supported_summaries.rs index 6aeed6b0c..54fa01548 100644 --- a/crates/executor/src/supported_summaries.rs +++ b/crates/executor/src/supported_summaries.rs @@ -12,8 +12,8 @@ use planner_types::ir::schema::SupportedSummary; pub fn supported_summary_families() -> Vec { EVALUATED_READOUTS .iter() - .map(|&(family, readouts)| SupportedSummary { - family, + .map(|&(summary_type, readouts)| SupportedSummary { + summary_type, readouts: readouts.to_vec(), }) .collect() @@ -189,7 +189,7 @@ mod tests { let executed = factory_builds(&family) && evaluation_answers(&family, readout); let declared = listed .iter() - .any(|s| &s.family == algorithm && s.readouts.contains(&readout)); + .any(|s| &s.summary_type == algorithm && s.readouts.contains(&readout)); assert_eq!(declared, executed, "{algorithm:?} {readout:?}"); let answered = SummaryTypeCapability::of(&family).answers(&statistic(readout)); assert_eq!( diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 192a32ab6..2a0f75c58 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -369,7 +369,8 @@ pub mod planning { .iter() .enumerate() .filter(|(group, _)| { - memo.query_roots[q] + memo.queries[q] + .root .operator_refs() .contains(&&asap_types::stages::MemoGroupId(*group as u32)) }) diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 0580f24d7..0a8af72f6 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -405,9 +405,9 @@ pub fn generate_per_computation_candidates( // a group several queries read is one node. let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); let roots: Vec = memo - .query_roots + .queries .iter() - .map(|root| match root { + .map(|query| match &query.root { QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), QueryRoot::Scalar(expr) => QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))), }) diff --git a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs index 3c90e7809..4da070c98 100644 --- a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs @@ -18,7 +18,7 @@ use asap_types::workload::QueryWorkloadEntry; pub fn generate_logical_asap_memo_candidates( pass1: Memo, entries: &[QueryWorkloadEntry], - evidence: &AccuracyEvidence, + evidence: &dyn AccuracyEvidence, ) -> Result { pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?, evidence) } @@ -41,7 +41,7 @@ pub fn pass2_local_window_composition( /// (`asap_accuracy::producer_satisfies_logical_requirements`). pub fn pass2_after_local_window_composition( mut memo: Memo, - evidence: &AccuracyEvidence, + evidence: &dyn AccuracyEvidence, ) -> Result { // 2. Scan widening. scan_labels::add_scan_label_candidates(&mut memo)?; diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index 64bfc668a..6ce0ac737 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -735,7 +735,7 @@ mod tests { assert!(!defaulted.supports(&SketchAlgorithm::CountSketch, StatisticKind::ItemFrequency)); let kll_only = DeploymentCapabilities { supported_summaries: Some(vec![SupportedSummary { - family: SketchAlgorithm::Kll, + summary_type: SketchAlgorithm::Kll, readouts: vec![StatisticKind::Quantile], }]), ..DeploymentCapabilities::default() diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 273a8b2b0..70608a2cf 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -19,13 +19,12 @@ use asap_types::workload::parsed_workload::ParsedWorkload; use asap_types::workload::WorkloadError; use crate::run::{SelectionError, Stage1ToStage3Pass}; -use asap_accuracy::AccuracyEvidence; +use asap_accuracy::{AccuracyEvidence, NoMeasuredGuarantees}; use asap_cost::CostInputs; use asap_logical_asap_optimization::PerComputationCandidateError; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; -static NO_ACCURACY_EVIDENCE: AccuracyEvidence = AccuracyEvidence {}; static DEFAULT_COST_INPUTS: std::sync::LazyLock = std::sync::LazyLock::new(CostInputs::default); static DEFAULT_DEPLOYMENT: std::sync::LazyLock = @@ -45,7 +44,7 @@ static DEFAULT_DEPLOYMENT: std::sync::LazyLock = pub struct PlanningModels<'a> { /// The deployment's measured accuracy, which Pass 2 reads before each /// summary's theoretical guarantee; none by default. - pub evidence: &'a AccuracyEvidence, + pub evidence: &'a dyn AccuracyEvidence, /// The deployment's cost inputs: the cost model (default /// cost model 1 with illustrative unit costs), the resource speed and /// the non-repeating query horizon. @@ -65,7 +64,7 @@ pub struct PlanningModels<'a> { impl<'a> PlanningModels<'a> { /// This evidence with the default plan selection settings. - pub fn new(evidence: &'a AccuracyEvidence) -> Self { + pub fn new(evidence: &'a dyn AccuracyEvidence) -> Self { Self { evidence, cost_inputs: &DEFAULT_COST_INPUTS, @@ -76,7 +75,7 @@ impl<'a> PlanningModels<'a> { /// The built-in models: no accuracy evidence. pub fn builtin() -> PlanningModels<'static> { - PlanningModels::new(&NO_ACCURACY_EVIDENCE) + PlanningModels::new(&NoMeasuredGuarantees) } /// These models, enforcing no accuracy target when `ignore` @@ -86,7 +85,7 @@ impl<'a> PlanningModels<'a> { self } - pub fn with_evidence(mut self, evidence: &'a AccuracyEvidence) -> Self { + pub fn with_evidence(mut self, evidence: &'a dyn AccuracyEvidence) -> Self { self.evidence = evidence; self } diff --git a/crates/types/src/deployment_capabilities.rs b/crates/types/src/deployment_capabilities.rs index 0ba1be478..f2a7be3ec 100644 --- a/crates/types/src/deployment_capabilities.rs +++ b/crates/types/src/deployment_capabilities.rs @@ -3,13 +3,13 @@ //! //! [`DeploymentCapabilities`] says three things: //! -//! * which sketch families the executor supports, and which readouts it -//! supports for each. A family is supported when the executor can build +//! * which sketch summary types the executor supports, and which readouts +//! it supports for each. A summary type is supported when the executor can build //! its state, and a readout of it when the executor can answer that //! statistic ([`StatisticKind`]) from the state. Stage 1 (Pass 1) offers -//! no summary candidate whose family or readout is unsupported, so no +//! no summary candidate whose summary type or readout is unsupported, so no //! Pass 2 candidate has one either (Pass 2 rules share, split into tumbling windows -//! or reuse existing builds; none adds a family or a readout). Exact +//! or reuse existing builds; none adds a summary type or a readout). Exact //! accumulators and plain exact execution are always supported; //! * whether raw data samples are retained. If they are, Stage 3 counts them on //! disk once per source, at the longest window a query-time scan of it @@ -31,7 +31,7 @@ //! and keeps no query-time result across evaluations, which is the //! planner's behavior without deployment inputs. -// The executor lists the families it builds as `SupportedSummary`s, so the +// The executor lists the summary types it builds as `SupportedSummary`s, so the // type lives with the readout vocabulary in `asap-types`. pub use crate::ir::schema::SupportedSummary; use crate::ir::schema::{SketchAlgorithm, StatisticKind}; @@ -46,8 +46,8 @@ use thiserror::Error; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct DeploymentCapabilities { - /// The sketch families the executor can build, each with the readouts - /// it can answer from one; `None` (the default) supports every family + /// The sketch summary types the executor can build, each with the + /// readouts it can answer from one; `None` (the default) supports every summary type /// and readout the planner can build. pub supported_summaries: Option>, /// Whether raw data samples are retained for query-time scans; default `true`. @@ -68,9 +68,9 @@ impl Default for DeploymentCapabilities { } impl DeploymentCapabilities { - /// Whether the executor can build `family` and answer `readout` from it. - pub fn supports(&self, family: &SketchAlgorithm, readout: StatisticKind) -> bool { - summary_supported(self.supported_summaries.as_deref(), family, readout) + /// Whether the executor can build `summary_type` and answer `readout` from it. + pub fn supports(&self, summary_type: &SketchAlgorithm, readout: StatisticKind) -> bool { + summary_supported(self.supported_summaries.as_deref(), summary_type, readout) } /// Checks these capabilities against the data workload: stored data @@ -90,18 +90,18 @@ impl DeploymentCapabilities { } /// Whether an executor that builds `supported_summaries` (`None`: every -/// family and readout the planner can build) can build `family` and answer -/// `readout` from it. Pass 1 reads only this field of -/// [`DeploymentCapabilities`]. +/// summary type and readout the planner can build) can build +/// `summary_type` and answer `readout` from it. Pass 1 reads only this +/// field of [`DeploymentCapabilities`]. pub fn summary_supported( supported_summaries: Option<&[SupportedSummary]>, - family: &SketchAlgorithm, + summary_type: &SketchAlgorithm, readout: StatisticKind, ) -> bool { supported_summaries.is_none_or(|supported| { supported .iter() - .any(|s| s.family == *family && s.readouts.contains(&readout)) + .any(|s| s.summary_type == *summary_type && s.readouts.contains(&readout)) }) } @@ -122,7 +122,7 @@ pub enum DeploymentCapabilitiesError { mod tests { use super::*; - /// The default supports every family and readout, retains raw data and + /// The default supports every summary type and readout, retains raw data and /// keeps no query-time result across evaluations; a JSON object with /// only some fields keeps the defaults for the rest. #[test] @@ -131,7 +131,7 @@ mod tests { assert!(default.supports(&SketchAlgorithm::Bloom, StatisticKind::TopK)); assert!(default.raw_data_retained && !default.query_time_results_retained); let parsed: DeploymentCapabilities = serde_json::from_str( - r#"{"supported_summaries": [{"family": "Kll", "readouts": ["Quantile"]}]}"#, + r#"{"supported_summaries": [{"summary_type": "Kll", "readouts": ["Quantile"]}]}"#, ) .unwrap(); assert!(parsed.supports(&SketchAlgorithm::Kll, StatisticKind::Quantile)); diff --git a/crates/types/src/ir/properties/accuracy.rs b/crates/types/src/ir/properties/accuracy.rs index b3007e3fa..a4eb1b576 100644 --- a/crates/types/src/ir/properties/accuracy.rs +++ b/crates/types/src/ir/properties/accuracy.rs @@ -171,21 +171,6 @@ impl AccuracyScope { } } } - - /// Whether a statement over this scope also covers every value `other` - /// covers: the whole result covers every scope, a record share covers a - /// larger share (not every value: a smaller group is outside it), and a - /// single value only itself. - pub fn covers(self, other: Self) -> bool { - match (self, other) { - (Self::WholeResult, _) | (Self::PerValue, Self::PerValue) => true, - ( - Self::GroupsWithRecordShareAtLeast(share), - Self::GroupsWithRecordShareAtLeast(other_share), - ) => share <= other_share, - _ => false, - } - } } /// The outer shape of an accuracy guarantee @@ -816,22 +801,6 @@ mod tests { ); } - /// The whole result covers every scope; a record share covers a larger - /// share but not a single value, which may be a smaller group; a single - /// value covers only itself. - #[test] - fn a_wider_scope_covers_a_narrower_one() { - use AccuracyScope::*; - let share = GroupsWithRecordShareAtLeast; - assert!(WholeResult.covers(share(0.01))); - assert!(share(0.01).covers(share(0.05))); - assert!(!share(0.05).covers(PerValue)); - assert!(PerValue.covers(PerValue)); - assert!(!share(0.05).covers(share(0.01))); - assert!(!share(0.05).covers(WholeResult)); - assert!(!PerValue.covers(share(0.05))); - } - /// Exact means zero error with certainty, whatever the metric; an /// ε of zero without a δ is not exact. #[test] diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs index 845e20b4b..a461d18cc 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -468,9 +468,8 @@ impl AccuracyGuarantee { /// never fails (`δ = 0`) holds for every value at once; an unbound `n` /// (the whole result of an unbound `G`) leaves the failure probability /// symbolic in [`VALUES_PER_RESULT`]. A guarantee over any other scope - /// is returned as it is: the check then accepts it only for a scope it - /// covers ([`AccuracyScope::covers`]). Plan selection uses it to check a - /// target over `scope`. + /// is returned as it is. Plan selection uses it to check a target over + /// `scope`. pub fn over_scope(&self, scope: AccuracyScope, values_per_result: Option) -> Self { let mut over = self.clone(); // Only a per-value statement extends to more values by the union diff --git a/crates/types/src/ir/schema/summary_type_capability.rs b/crates/types/src/ir/schema/summary_type_capability.rs index a53c7f560..a9c28ae51 100644 --- a/crates/types/src/ir/schema/summary_type_capability.rs +++ b/crates/types/src/ir/schema/summary_type_capability.rs @@ -37,15 +37,15 @@ pub struct SummaryTypeCapability { pub(crate) readouts: &'static [(StatisticKind, MergedError)], } -/// One sketch family a deployment's executor can build, with the readouts -/// it can answer from the family's state: an entry of +/// One sketch summary type a deployment's executor can build, with the +/// readouts it can answer from its state: an entry of /// `DeploymentCapabilities::supported_summaries`. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct SupportedSummary { - /// The family the executor can build. - pub family: SketchAlgorithm, - /// The statistics it can answer from one state of the family. + /// The summary type the executor can build. + pub summary_type: SketchAlgorithm, + /// The statistics it can answer from one state of the summary type. pub readouts: Vec, } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index f33adbe13..89ab3b284 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -30,15 +30,22 @@ pub struct Memo { /// The groups, by id ([`Self::groups`]); added only through /// [`Self::find_or_add_group`]. groups: Vec, - /// Each query's root over groups, by index in the workload. - pub query_roots: Vec>, - /// Each query's accuracy target $A_q$, by index; `None` for a query - /// without one (every query when accuracy is not enforced). - pub query_accuracy_targets: Vec>, - /// Each query's $n_q$, the number of values one evaluation returns, by - /// index; `None` when the data workload does not bound it. Pass 2 turns - /// a whole-result failure probability into a per-value one with it. - pub query_values_per_evaluation: Vec>, + /// The queries, by index in the workload. + pub queries: Vec, +} + +/// One query of the memo: its root over groups, its accuracy target and +/// $n_q$. +#[derive(Debug, Clone)] +pub struct MemoQuery { + pub root: QueryRoot, + /// $A_q$; `None` for a query without one (every query when accuracy is + /// not enforced). + pub accuracy_target: Option, + /// $n_q$, the number of values one evaluation returns; `None` when the + /// data workload does not bound it. Pass 2 turns a whole-result failure + /// probability into a per-value one with it. + pub values_per_evaluation: Option, } impl Memo { @@ -47,17 +54,11 @@ impl Memo { &self.groups } - /// Adds a query whose root is `root`, with its accuracy target and - /// $n_q$, and returns its index: the root's groups and every group below - /// them list it ([`MemoGroup::reached_by_queries`]). Stage 0 adds each - /// query once. - pub fn add_query( - &mut self, - root: QueryRoot, - accuracy_target: Option, - values_per_evaluation: Option, - ) -> usize { - let _ = (root, accuracy_target, values_per_evaluation); + /// Adds `query` and returns its index: its root's groups and every + /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 + /// adds each query once. + pub fn add_query(&mut self, query: MemoQuery) -> usize { + let _ = query; todo!("#1207 PR 2") } @@ -159,7 +160,8 @@ impl Memo { /// input group's `group_properties` and `expression` /// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression /// breaks them (schema derivation, input validation, merge check). -// `column_sign` is derived in #1207 PR 2; until then the result is unreachable. +// `column_sign` and `depends_on_evaluation_time` are derived in #1207 PR 2; +// until then the result is unreachable. #[allow(unreachable_code)] fn derive_group_properties( memo: &Memo, @@ -182,6 +184,7 @@ fn derive_group_properties( aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), column_sign: todo!("#1207 PR 2: each output column's sign, per operator, from the input groups' and the sources' declared signs"), + depends_on_evaluation_time: todo!("#1207 PR 2: whether the result changes with the evaluation time, from the operator and the input groups"), }) } diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index 9b21b38f3..c2d567f81 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -3,7 +3,7 @@ //! the candidates dead-candidate elimination removed, each with its root //! causes. -use super::{CandidateForMemoGroupId, Memo, RequiredAccuracy}; +use super::{CandidateForMemoGroupId, EvaluationGrid, Memo, RequiredAccuracy}; use crate::ir::properties::AccuracyMetric; /// $(u, c, i)$ over logical candidates: `producer_candidate_id` is a @@ -29,6 +29,13 @@ pub struct LogicalASAPMemoCandidates { /// The candidates dead-candidate elimination removed: no logical edge /// names them, and Stage 2 does not expand them. pub pruned: Vec, + /// Per group, by `MemoGroupId`: the grid a result of the group is + /// evaluated on, the coarsest grid holding every evaluation time of the + /// queries reaching it (X = `sum_over_time(m[5m])` read every 60 s and + /// every 120 s: 60 s); `None` when those times are not on one known + /// grid. A result of the group materialized across evaluations is + /// stored at these times. + pub evaluation_grid_of_group: Vec>, } /// A candidate eliminated by Pass 2. diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 06f4333f4..cce07008d 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -3,11 +3,11 @@ use std::collections::BTreeSet; -use super::{CandidateForMemoGroup, Memo}; +use super::{CandidateForMemoGroup, Memo, MemoQuery}; use crate::ir::operator::input::OperatorInput; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::schema::Schema; -use crate::ir::{Operator, OperatorResultKind, QueryRoot}; +use crate::ir::{Operator, OperatorResultKind}; /// A group, by index in the memo's groups. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -43,6 +43,12 @@ pub struct GroupProperties { pub result_kind: OperatorResultKind, /// Per output column, in schema order: the sign of its values. pub column_sign: Vec, + /// Whether the result changes with the evaluation time: true for a + /// group under a `TimeRange` relative to the evaluation time + /// (`sum_over_time(m[5m])`), false for one under a fixed `@ T` or + /// reading no time. A result that does not depend on it is the same at + /// every evaluation, so a stored copy delivers on every grid. + pub depends_on_evaluation_time: bool, /// What an aggregate group covers: a summary state, a chain, a merge /// over a chain's windows, or an exact `Aggregate` result; `None` for /// any other group. Derived today for summary states only; exact @@ -112,19 +118,19 @@ impl MemoGroup { } } -/// The groups some query's root reads, ascending: the groups of -/// `query_roots` that are a query's result or a scalar query's operand. -pub fn root_groups(query_roots: &[QueryRoot]) -> BTreeSet { - query_roots +/// The groups some query's root reads, ascending: the groups of the +/// `queries`' roots that are a query's result or a scalar query's operand. +pub fn root_groups(queries: &[MemoQuery]) -> BTreeSet { + queries .iter() - .flat_map(QueryRoot::operator_refs) + .flat_map(|query| query.root.operator_refs()) .copied() .collect() } /// The queries whose root reads `group`, ascending. -pub fn queries_rooted_at(query_roots: &[QueryRoot], group: MemoGroupId) -> Vec { - (0..query_roots.len()) - .filter(|&query| query_roots[query].operator_refs().contains(&&group)) +pub fn queries_rooted_at(queries: &[MemoQuery], group: MemoGroupId) -> Vec { + (0..queries.len()) + .filter(|&query| queries[query].root.operator_refs().contains(&&group)) .collect() } diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index b87cee1e2..728fab01b 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -17,7 +17,7 @@ mod pruning; pub use candidates::{ parameter_list, CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, ErrorPropagation, LogicalProperties, LogicalPropertiesError, - LogicalRequirements, Memo, OriginOfCandidateForMemoGroup, RequiredAccuracy, + LogicalRequirements, Memo, MemoQuery, OriginOfCandidateForMemoGroup, RequiredAccuracy, }; pub use logical_memo_candidates::{ LogicalASAPMemoCandidates, LogicalEdge, PruneCause, PrunedLogicalCandidate, RootPruneReason, diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 0abda93cc..f94b2c5ff 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -9,7 +9,7 @@ use std::collections::HashMap; use super::execution_context::ExecutionContext; use crate::ir::QueryRoot; use crate::physical::{ExecutionTiming, PhysicalProperties}; -use crate::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, PrunedCandidate}; +use crate::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, PrunedCandidate}; /// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize)] @@ -33,13 +33,20 @@ impl PhysicalCandidate { /// concepts doc's alignment table, section 2). True for $\ell$ = None, /// and for `OneEvaluation` in the same context. A chain producer /// (`SummaryTumblingWindowChain`) with $\ell$ = `AcrossEvaluations`, with - /// $w$ the chain operator's width in its expression in `memo`, needs every evaluation time of the context on its - /// grid: $w$ divides the context's evaluation grid (interval, range step - /// and phase), each scheduled time, or a one-off query's time; false - /// when the times are not known (an interval without a phase, an - /// estimated rate, an unknown recurrence). - pub fn delivers_on(&self, memo: &Memo, context: &ExecutionContext) -> bool { - let _ = (memo, context); + /// $w$ the chain operator's width, needs every time of the context on + /// its grid: $w$ divides the context's period and phase. Any other + /// producer with $\ell$ = `AcrossEvaluations` is stored on its group's + /// evaluation grid (`evaluation_grid_of_group`): it delivers on every + /// context when the group does not depend on the evaluation time, else + /// when that grid holds every time of the context. In + /// `Materialized(v)`, the context's times are v's, read from + /// `candidates`. + pub fn delivers_on( + &self, + candidates: &PhysicalASAPMemoCandidates, + context: &ExecutionContext, + ) -> bool { + let _ = (candidates, context); todo!("#1207 PR 1: the alignment table of section 2") } } diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index 052d2bd62..74bd32816 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -19,8 +19,8 @@ not turn an unknown guarantee into a satisfied accuracy requirement. ## Candidate state Here, **certified for a target** means Planner has a fully known -`AccuracyGuarantee` for the candidate and `asap_accuracy::satisfies(guarantee, -target)` returns `true`. **Uncertified** means Planner cannot make that claim: +`AccuracyGuarantee` for the candidate whose evaluated error is within the +target's (`asap_accuracy::target_limit`). **Uncertified** means Planner cannot make that claim: the guarantee is absent, contains unknown terms, or is known not to meet the target. An uncertified summary may still be a well-formed logical candidate; this label says nothing about whether the backend can physically execute it. diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 80517d7df..a5f1fe1f8 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -202,8 +202,8 @@ cost model. ## Composing guarantees through a DAG The accuracy crate supplies theoretical guarantees (`theoretical_guarantee`), -propagation rules and target satisfaction (`satisfies`); the deployment's -measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a +propagation rules and the comparison limit of a target (`target_limit`); the +deployment's measurements (`AccuracyEvidence::measured_guarantee`) take precedence over a theoretical guarantee. The rules are conservative: an unregistered composition over approximate inputs is rejected rather than treated as exact. diff --git a/docs/develop_docs/end-to-end-accuracy-guarantees.md b/docs/develop_docs/end-to-end-accuracy-guarantees.md index dfc6a1ce2..1b30230b2 100644 --- a/docs/develop_docs/end-to-end-accuracy-guarantees.md +++ b/docs/develop_docs/end-to-end-accuracy-guarantees.md @@ -52,22 +52,25 @@ Read the sections below when changing one of those contracts. Accuracy reasoning and allocation are separate from cost modeling: ```rust -/// The theoretical guarantee of reading `statistic` from a state of -/// `summary_type` with parameters θ: the algorithm's error formula. +/// The theoretical error bound of reading `statistic` from a state of +/// `summary_type` with parameters θ at `failure_probability`: the +/// algorithm's error formula. `None` when θ cannot fail that rarely. pub fn theoretical_guarantee( summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, + failure_probability: f64, ) -> Option; -/// The deployment's measured guarantee for the same configuration, read -/// first when present. -impl AccuracyEvidence { - pub fn measured_guarantee( +/// The deployment's measured error bound for the same configuration, read +/// first when present; `NoMeasuredGuarantees` has none. +pub trait AccuracyEvidence { + fn measured_guarantee( &self, summary_type: &FieldDataType, parameters: &SketchParams, statistic: &SketchStatistic, + failure_probability: f64, ) -> Option; } ``` diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index b0f081d60..52b2f114c 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -193,9 +193,9 @@ asap_plan_selection::select_plan( | Input or output | Meaning | | --- | --- | -| `PlanningModels::new(evidence)` | Your `&AccuracyEvidence` (measured guarantees), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | +| `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | | `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | -| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { family: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | +| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | | `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index dcd6f5842..131a02e27 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -1169,7 +1169,7 @@ function usageText(usage) { function deploymentText(capabilities) { if (!capabilities || typeof capabilities !== 'object') return ''; const summaries = Array.isArray(capabilities.supported_summaries) - ? `summaries: ${capabilities.supported_summaries.map((s) => `${s.family} (${(s.readouts || []).join(', ')})`).join(', ')}` + ? `summaries: ${capabilities.supported_summaries.map((s) => `${s.summary_type} (${(s.readouts || []).join(', ')})`).join(', ')}` : 'every summary the planner builds'; return [capabilities.raw_data_retained === false ? 'raw data not retained' : 'raw data retained', capabilities.query_time_results_retained ? 'query-time results retained' : 'query-time results not retained', summaries].join(' · '); diff --git a/tools/dag-viewer/workload-sets.js b/tools/dag-viewer/workload-sets.js index ece483103..28bf710da 100644 --- a/tools/dag-viewer/workload-sets.js +++ b/tools/dag-viewer/workload-sets.js @@ -19,7 +19,7 @@ const parts = []; if (capabilities.raw_data_retained === false) parts.push('raw data not retained'); if (capabilities.query_time_results_retained === true) parts.push('query-time results retained'); - if (Array.isArray(capabilities.supported_summaries)) parts.push(`${capabilities.supported_summaries.length} supported summary families`); + if (Array.isArray(capabilities.supported_summaries)) parts.push(`${capabilities.supported_summaries.length} supported summary types`); return parts.join(', ') || 'default deployment capabilities'; } From 55876077c2d2526bd1729a1f180961036ec0baf2 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:20:11 +0000 Subject: [PATCH 19/30] refactor: delete Pass 1's test-support whole-workload composition and the tests that use it (#1207 PR 0) pass1/composition.rs composed one whole-workload candidate DAG from one per-computation candidate per target, for tests only; planning adds every way to compute to the memo instead. It and the 74 tests and fixture helpers that compose through it are deleted (no interim code for PR 3). --- crates/devtools/src/candidate_labels.rs | 3 - crates/devtools/src/planning_stages_text.rs | 3 - crates/executor/tests/common/mod.rs | 37 - .../tests/external_corpus.rs | 83 -- crates/frontend-promql/Cargo.toml | 4 - .../frontend-promql/tests/count_planning.rs | 175 +-- crates/frontend-promql/tests/main.rs | 1 - .../observability/metrics_observability.rs | 163 --- .../tests/observability/promql_corpus.rs | 46 - crates/frontend-promql/tests/support.rs | 54 - .../tests/univmon_candidates.rs | 56 - crates/integration-tests/src/lib.rs | 67 -- .../tests/exact_composition.rs | 121 -- crates/integration-tests/tests/main.rs | 4 - .../tests/per_computation_candidates_sql.rs | 206 +--- .../tests/promql_numeric_regressions.rs | 332 ------ .../integration-tests/tests/sketch_rollup.rs | 371 ------ .../tests/sql_to_post_asap.rs | 459 +------ .../tests/stage1_logical_asap_optimization.rs | 750 ------------ crates/logical-asap-optimization/src/pass1.rs | 8 +- .../src/pass1/composition.rs | 1054 ----------------- .../src/pass1/per_computation_candidates.rs | 1010 +--------------- .../pass1/per_computation_candidates_tests.rs | 409 ------- .../src/pass1/realization.rs | 27 - .../src/test_support.rs | 86 +- .../asap-aware-mapping-architecture.md | 2 +- .../asap-aware-mapping-contracts.md | 2 +- 27 files changed, 21 insertions(+), 5512 deletions(-) delete mode 100644 crates/frontend-promql/tests/observability/metrics_observability.rs delete mode 100644 crates/frontend-promql/tests/univmon_candidates.rs delete mode 100644 crates/integration-tests/tests/exact_composition.rs delete mode 100644 crates/integration-tests/tests/promql_numeric_regressions.rs delete mode 100644 crates/integration-tests/tests/sketch_rollup.rs delete mode 100644 crates/integration-tests/tests/stage1_logical_asap_optimization.rs delete mode 100644 crates/logical-asap-optimization/src/pass1/composition.rs diff --git a/crates/devtools/src/candidate_labels.rs b/crates/devtools/src/candidate_labels.rs index 5bfe456b4..94267b698 100644 --- a/crates/devtools/src/candidate_labels.rs +++ b/crates/devtools/src/candidate_labels.rs @@ -149,6 +149,3 @@ fn duration_text(value: &Value) -> String { _ => value.to_string(), } } - -#[cfg(test)] -mod tests {} diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index cf8e1bf66..1f89f597a 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -238,6 +238,3 @@ fn group_label(group: &MemoGroup) -> String { let _ = group; todo!("#1207 PR 9: label a group by its expression") } - -#[cfg(test)] -mod tests {} diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index ea0af3b3d..4e720711c 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -21,43 +21,6 @@ pub fn selected_dag( Rc::clone(selected) } -/// Exercise each local implementation rule directly, keeping the other -/// computations at their default choice. A bounded solver archive need not -/// contain every algorithm that executor binding tests need to exercise. -pub fn computation_candidates(root: Rc) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .expect("Pass 1 accepts the query"); - let mut choices = vec![vec![0; inventory.targets.len()]]; - for (index, target) in inventory.targets.iter().enumerate() { - for choice in 1..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - choices.push(selected); - } - } - choices - .into_iter() - .map(|choice| { - let mut roots = compose_logical_candidate(&inventory, &choice) - .expect("a generated computation candidate composes"); - match roots.remove(0) { - QueryRoot::Operator(root) => root, - QueryRoot::Scalar(_) => panic!("one operator root"), - } - }) - .collect() -} - fn run_all_stages( root: Rc, accuracy: planner_types::ir::properties::AccuracyTarget, diff --git a/crates/frontend-metricsql/tests/external_corpus.rs b/crates/frontend-metricsql/tests/external_corpus.rs index 4e4f565dd..aafe36049 100644 --- a/crates/frontend-metricsql/tests/external_corpus.rs +++ b/crates/frontend-metricsql/tests/external_corpus.rs @@ -10,17 +10,6 @@ //! `ASAP_EXTERNAL_CORPUS_DIR= cargo test -p asap-frontend-metricsql //! --test metricsql external_corpus:: -- --nocapture` -use std::collections::HashMap; -use std::path::Path; -use std::time::Duration; - -use asap_frontend_metricsql::{lower_metricsql_query, MetricsqlError}; -use asap_types::ir::QueryRoot; - -use crate::support::logical_plan; - -const DIR_VAR: &str = "ASAP_EXTERNAL_CORPUS_DIR"; - fn queries(corpus: &str) -> impl Iterator { corpus .lines() @@ -36,14 +25,6 @@ fn floor(corpus: &str) -> Option { .and_then(|n| n.trim().parse().ok()) } -#[derive(Default, Debug)] -struct Tally { - lowered: usize, - unparseable: usize, - rejected: usize, - bind_errored: usize, -} - /// `msg` with sanitized identifiers replaced by `_` and digit runs by `N`, so /// the same failure in different queries groups under one message. fn normalize(msg: &str) -> String { @@ -68,70 +49,6 @@ fn normalize(msg: &str) -> String { out } -/// Tally `corpus`, counting each failure's normalized message in `errors`. -fn tally(corpus: &str, errors: &mut HashMap) -> Tally { - let mut t = Tally::default(); - for q in queries(corpus) { - match lower_metricsql_query(q, Duration::from_secs(15)) { - Ok(QueryRoot::Operator(dag)) => { - t.lowered += 1; - if logical_plan(&dag).is_err() { - t.bind_errored += 1; - } - } - Ok(QueryRoot::Scalar(_)) => t.lowered += 1, - Err(e) => { - if matches!(e, MetricsqlError::Parse(_)) { - t.unparseable += 1; - } else { - t.rejected += 1; - } - *errors.entry(normalize(&e.to_string())).or_default() += 1; - } - } - } - t -} - -/// Lowering and binding are total over every corpus file: a panic fails the -/// test, no lowerable query fails to bind, and coverage stays at its floor. -/// Also prints each unparseable/rejected cause with its count, most common -/// first, to show which unsupported features block the most queries. -#[test] -fn corpus_lowers_and_binds() { - let Some(dir) = std::env::var_os(DIR_VAR) else { - eprintln!("{DIR_VAR} unset; skipping the external corpus"); - return; - }; - let mut files: Vec<_> = std::fs::read_dir(Path::new(&dir)) - .unwrap_or_else(|e| panic!("{DIR_VAR}: {e}")) - .map(|entry| entry.unwrap().path()) - .filter(|path| path.extension().is_some_and(|ext| ext == "txt")) - .collect(); - files.sort(); - assert!(!files.is_empty(), "{DIR_VAR} has no *.txt files"); - - let mut failures = Vec::new(); - let mut errors = HashMap::new(); - for path in files { - let name = path.file_stem().unwrap().to_string_lossy(); - let corpus = std::fs::read_to_string(&path).unwrap(); - let min_lowered = floor(&corpus).unwrap_or_else(|| panic!("{name}: no `# floor: N` line")); - let t = tally(&corpus, &mut errors); - eprintln!("{name:24} {t:?}"); - if t.bind_errored > 0 || t.lowered < min_lowered { - failures.push(format!("{name}: {t:?} (floor {min_lowered})")); - } - } - let mut errors: Vec<_> = errors.into_iter().collect(); - errors.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); - eprintln!("\nFailures by cause:"); - for (msg, n) in errors { - eprintln!("{n:4} {msg}"); - } - assert!(failures.is_empty(), "{failures:#?}"); -} - #[test] fn floor_line() { assert_eq!(floor("# floor: 25\nrate(x[5m])"), Some(25)); diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index b7bc12ad6..541f1f97f 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -39,10 +39,6 @@ path = "tests/observability/promql_corpus.rs" name = "o11y_bench_promql" path = "tests/observability/o11y_bench_promql.rs" -[[test]] -name = "metrics_observability" -path = "tests/observability/metrics_observability.rs" - [[test]] name = "promql" path = "tests/main.rs" diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index d94b49e5c..a5496d1e4 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -1,48 +1,10 @@ //! Query text through summary selection: counts use observations, never value weights. use crate::planner_support::selected_dag; use crate::support; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{ - ExactKind, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr, - WeightDomain, -}; +use asap_types::ir::schema::{ExactKind, FieldDataType}; use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use std::rc::Rc; -use support::{logical_candidates, lower_promql, physical_asap_dag}; - -/// Whether `node` reads a Hydra state (one sketch shared by every group). -fn reads_hydra(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|node| { - matches!( - &node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch( - SketchAlgorithm::HydraCms - | SketchAlgorithm::HydraCountSketch - | SketchAlgorithm::HydraKll - | SketchAlgorithm::HydraHll - | SketchAlgorithm::HydraUnivMon - ), - .. - }) - ) - }) -} - -#[test] -#[ignore = "Pass 1 offers no Hydra state for a grouped count yet (#703)"] -fn grouped_count_offers_hydra_but_selection_keeps_per_group_state() { - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - // Hydra hashes a non-null item per row: the series identity (PromQL - // labels are nullable). - let root = asap_types::ir::schema::schema_support::with_promql_series_identity( - &lower_promql("count by(job)(up)", target.clone()).unwrap(), - ) - .unwrap(); - assert!(logical_candidates(Rc::clone(&root)).iter().any(reads_hydra)); - assert!(!reads_hydra(&selected_dag(root, target))); -} +use support::lower_promql; // Exact series and temporal counts must select a count accumulator, not distinct or sum. #[test] @@ -65,76 +27,6 @@ fn exact_counts_select_count_accumulators() { } } -// CMS/UnivMon count updates stay +1 even for zero or negative samples. -// CountSketch cannot answer totals and must not be offered for count. -#[test] -fn frequency_count_candidates_use_unit_weights() { - for query in ["count_over_time(up[5m])", "count(up)"] { - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_promql(query, accuracy.clone()).unwrap(); - let candidates = logical_candidates(root); - let mut algorithms = Vec::new(); - for node in &candidates { - let Some(summary_input) = - OperatorNode::reachable(node) - .into_iter() - .find_map(|node| match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - Some(Rc::clone(summary_input)) - } - _ => None, - }) - else { - continue; - }; - let Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(kind), - input, - .. - }) = &summary_input.operator - else { - continue; - }; - assert!( - !matches!(kind, SketchAlgorithm::Hll | SketchAlgorithm::CountSketch), - "{query}: {node:?}" - ); - if !matches!(kind, SketchAlgorithm::Cms | SketchAlgorithm::UnivMon) { - continue; - } - let dag = physical_asap_dag(node); - assert!( - dag.nodes.iter().any(|node| matches!( - &node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { input: actual, .. }) if actual == input - )), - "post-ASAP DAG must preserve the count update contract" - ); - algorithms.push(*kind); - assert!(input.item.is_some(), "frequency keys must be explicit"); - assert_eq!( - input.weight, - SummaryInputExpr::Constant(1.0), - "{query}: {node:?}" - ); - assert_eq!( - input.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount - } - ); - } - assert!( - algorithms.contains(&SketchAlgorithm::Cms), - "{query}: {candidates:?}" - ); - assert!( - algorithms.contains(&SketchAlgorithm::UnivMon), - "{query}: {candidates:?}" - ); - } -} - // Fixtures are already selected at one instant or within one five-minute window. // This narrow test oracle interprets the emitted aggregate, not Prometheus ingestion, // staleness, or scrape scheduling. Unsupported plan shapes fail explicitly. @@ -225,57 +117,6 @@ fn count_over_time_counts_scrapes_not_sample_values() { ); } -// Execute the emitted CMS update-weight expression on the reported values. -// This checks the planner's numerical update contract, not a sketch-library runtime. -#[test] -fn cms_count_updates_total_ten_for_zero_positive_and_negative_samples() { - use asap_types::ir::scalar::ColumnRef; - let accuracy = AccuracyTarget::max_error(0.02); - let root = lower_promql("count_over_time(up[5m])", accuracy.clone()).unwrap(); - let candidates = logical_candidates(root); - let dag = candidates - .iter() - .find_map(|node| { - let dag = physical_asap_dag(node); - dag.nodes - .iter() - .any(|node| { - matches!(&node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) - if kind == &SketchAlgorithm::Cms) - }) - .then_some(dag) - }) - .expect("CMS count candidate"); - let update = dag - .nodes - .iter() - .find_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { input, .. }) => Some(input), - _ => None, - }) - .unwrap(); - for value in [1.0, 0.0, 3.0, -3.0] { - let total: f64 = [value; 10] - .into_iter() - .map(|sample| { - let weight = match &update.weight { - SummaryInputExpr::Constant(value) => *value, - SummaryInputExpr::Column(ColumnRef::SampleValue) => sample, - SummaryInputExpr::Column(ColumnRef::Named(name)) if name == "value" => sample, - other => panic!("unsupported update weight: {other:?}"), - }; - assert!( - weight >= 0.0, - "CMS must not receive a negative update for {sample}" - ); - weight - }) - .sum(); - assert_eq!(total, 10.0, "ten samples of {value}"); - } -} - // The assertion helper must fail on an injected composition error; its coverage // must not depend on a real planner bug remaining unfixed. #[test] @@ -286,15 +127,3 @@ fn logical_candidates_fails_on_a_candidate_that_does_not_compose() { roots: Err("invalid choice".into()), }]); } - -// Ranked-row schemas let an aggregate consume TopK's numeric output (#672). -#[test] -fn aggregate_over_topk_candidates_compose() { - let accuracy = AccuracyTarget::max_error(0.01); - let root = lower_promql( - "sum by (job) (topk by (job, instance) (1, rate(m[5m])))", - accuracy.clone(), - ) - .unwrap(); - assert!(!logical_candidates(root).is_empty()); -} diff --git a/crates/frontend-promql/tests/main.rs b/crates/frontend-promql/tests/main.rs index 428e8a089..b962cb00b 100644 --- a/crates/frontend-promql/tests/main.rs +++ b/crates/frontend-promql/tests/main.rs @@ -12,4 +12,3 @@ mod promql_conformance; mod promql_equivalence; mod promql_lowering; mod scalar_design; -mod univmon_candidates; diff --git a/crates/frontend-promql/tests/observability/metrics_observability.rs b/crates/frontend-promql/tests/observability/metrics_observability.rs deleted file mode 100644 index 39815ca2e..000000000 --- a/crates/frontend-promql/tests/observability/metrics_observability.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Corpus totality checks over the metrics-observability benchmark exports. -//! -//! The source dump mixes Prometheus query logs, Grafana dashboard JSON, and -//! Prometheus rule YAML. The preprocessing script produces the line-oriented -//! fixtures consumed here. - -use asap_frontend_promql::PromqlError; -#[path = "../support.rs"] -mod support; -use asap_types::ir::properties::AccuracyTarget; -use support::{logical_plan, lower_promql}; - -/// Today's coverage of one corpus, as a ratchet: a change that plans fewer -/// queries with an ASAP operator, or fails to plan more of them, fails the -/// test. Raise `min_asap_candidates` (or lower `max_planning_errors`) when -/// coverage improves. -struct CoverageRatchet { - /// Queries whose logical plan binds an ASAP operator below the root. - min_asap_candidates: usize, - /// Lowered queries whose logical planning returns an error. - max_planning_errors: usize, -} - -const fn ratchet(min_asap_candidates: usize, max_planning_errors: usize) -> CoverageRatchet { - CoverageRatchet { - min_asap_candidates, - max_planning_errors, - } -} - -const CORPORA: &[(&str, &str, CoverageRatchet)] = &[ - ( - "CMU query logs", - include_str!("data/metrics_observability/cmu_chad_logs.txt"), - ratchet(162, 0), - ), - ( - "CMU dashboards", - include_str!("data/metrics_observability/cmu_chad_dashboards.txt"), - ratchet(174, 0), - ), - ( - "CMU rules", - include_str!("data/metrics_observability/cmu_chad_rules.txt"), - ratchet(202, 0), - ), - ( - "Claude query logs", - include_str!("data/metrics_observability/from_claude_logs.txt"), - ratchet(71, 0), - ), - ( - "Grafana dashboards", - include_str!("data/metrics_observability/grafana_dashboards.txt"), - ratchet(164, 0), - ), - ( - "Kubernetes mixin dashboards", - include_str!("data/metrics_observability/kubernetes_mixin_dashboards.txt"), - ratchet(147, 0), - ), - ( - "Kubernetes mixin alerts", - include_str!("data/metrics_observability/kubernetes_mixin_alerts.txt"), - ratchet(27, 2), - ), - ( - "Kubernetes mixin rules", - include_str!("data/metrics_observability/kubernetes_mixin_rules.txt"), - ratchet(66, 0), - ), - ( - "Promset", - include_str!("data/metrics_observability/promset.txt"), - ratchet(1505, 4), - ), -]; - -fn queries(corpus: &str) -> impl Iterator { - corpus - .lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) -} - -#[test] -fn benchmark_corpora_are_total_and_keep_their_coverage() { - for (name, corpus, ratchet) in CORPORA { - let mut total = 0; - let mut lowered = 0; - let mut parse_failures = 0; - let mut rejected = 0; - let mut post_asap_candidates = 0; - let mut post_asap_unchanged = 0; - let mut post_asap_errors = 0; - - for query in queries(corpus) { - total += 1; - // Approximate accuracy exercises the sketch-replacement boundary - // used by the existing post-ASAP corpus binding test. - match lower_promql(query, AccuracyTarget::max_error(0.01)) { - Ok(expr) => { - lowered += 1; - match logical_plan(&expr) { - // An ASAP operator bound somewhere below the root. - Ok(node) if node.contains_asap() => post_asap_candidates += 1, - Ok(_) => { - post_asap_unchanged += 1; - if std::env::var_os("METRICS_OBSERVABILITY_REPORT").is_some() { - eprintln!("POST_ASAP_MISS\t{name}\t{query}"); - } - } - Err(error) => { - post_asap_errors += 1; - if std::env::var_os("METRICS_OBSERVABILITY_REPORT").is_some() { - eprintln!("POST_ASAP_ERROR\t{name}\t{query}\t{error}"); - } - } - } - } - Err(PromqlError::Parse(_)) => parse_failures += 1, - Err(_) => rejected += 1, - } - } - - eprintln!("{name}: total={total}, parse_errors={parse_failures}, lowering_errors={rejected}, pre_asap={lowered}, post_asap_candidates={post_asap_candidates}, post_asap_unchanged={post_asap_unchanged}, post_asap_errors={post_asap_errors}"); - assert!(total > 0, "{name} fixture is empty"); - // Dashboard variables and panel expressions can contain Grafana - // template syntax rather than standalone PromQL. They are still - // counted above so extraction drift is visible, but are not expected - // to parse as PromQL. - assert_eq!(total, lowered + rejected + parse_failures); - assert!( - post_asap_candidates >= ratchet.min_asap_candidates, - "{name}: {post_asap_candidates} queries bind an ASAP operator, fewer than {}", - ratchet.min_asap_candidates - ); - assert!( - post_asap_errors <= ratchet.max_planning_errors, - "{name}: {post_asap_errors} queries fail logical planning, more than {}", - ratchet.max_planning_errors - ); - } -} - -/// The two Kubernetes mixin alerts whose candidates do not compose: each -/// aggregates over a `topk` output, which the frontend declares as -/// `[keys.., topk_k]` with a `Utf8` placeholder and no value column, so the -/// outer `sum`'s accumulator has no value to read (and the plain plan would -/// sum a string). The ranked-row TopK contract now exposes its numeric value -/// and retained identity columns (#672). -#[test] -fn kubernetes_mixin_aggregates_over_topk_compose() { - for query in [ - r#"sum without (id, metrics_path, name, image, endpoint, job, node) ( topk by (cluster, namespace, pod, container, instance) (1, increase( container_cpu_cfs_throttled_periods_total{container!="", job="cadvisor", } [5m]) ) ) / on (cluster, namespace, pod, container, instance) group_left sum without (id, metrics_path, name, image, endpoint, job, node) ( topk by (cluster, namespace, pod, container, instance) (1, increase( container_cpu_cfs_periods_total{job="cadvisor", } [5m]) ) ) > ( 25 / 100 )"#, - r#"histogram_quantile(0.99, sum by (cluster, instance, le) ( topk by (cluster, instance, le, operation_type) (1, rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m]) ) ) ) * on(cluster, instance) group_left(node) topk by (cluster, instance, node) (1, kubelet_node_name{job="kubelet"} ) > 60"#, - ] { - let root = lower_promql(query, AccuracyTarget::max_error(0.01)).unwrap(); - if let Err(error) = logical_plan(&root) { - panic!("{query}: {error}"); - } - } -} diff --git a/crates/frontend-promql/tests/observability/promql_corpus.rs b/crates/frontend-promql/tests/observability/promql_corpus.rs index cd0aab26c..461a8a4f4 100644 --- a/crates/frontend-promql/tests/observability/promql_corpus.rs +++ b/crates/frontend-promql/tests/observability/promql_corpus.rs @@ -17,7 +17,6 @@ use asap_frontend_promql::PromqlError as LoweringError; #[path = "../support.rs"] mod support; use asap_types::ir::properties::AccuracyTarget; -use support::{logical_plan, lower_promql}; const DOCS: &str = include_str!("data/promql_corpus_docs.txt"); const TESTDATA: &str = include_str!("data/promql_corpus_testdata.txt"); @@ -67,51 +66,6 @@ fn tally(corpus: &str) -> Tally { t } -/// Every query that lowers, additionally run through logical ASAP-aware -/// optimization ([`logical_plan`]: Pass 1 and composition), at an -/// approximate accuracy target so summary candidates actually appear (an -/// `Exact` target would only ever exercise the exact-accumulator arm). -#[derive(Default, Debug)] -struct BindTally { - /// An ASAP operator was bound somewhere below the root — the pass did - /// something. - transformed: usize, - /// The kept pre-ASAP dag — the pass left the query untouched. - unchanged: usize, - /// [`logical_plan`] returned `Err`: Pass 1 rejected the query, or its - /// exact candidate did not compose. - errored: usize, -} - -fn bind_tally(corpus: &str, accuracy: AccuracyTarget) -> BindTally { - let mut t = BindTally::default(); - for q in queries(corpus) { - let Ok(dag) = lower_promql(q, accuracy.clone()) else { - continue; - }; - match logical_plan(&dag) { - Ok(bound) if !bound.contains_asap() => t.unchanged += 1, - Ok(_) => t.transformed += 1, - Err(_) => t.errored += 1, - } - } - t -} - -#[test] -fn binding_is_total_over_the_entire_corpus() { - let accuracy = AccuracyTarget::max_error(0.01); - let docs = bind_tally(DOCS, accuracy.clone()); - let td = bind_tally(TESTDATA, accuracy); - eprintln!("docs corpus post-ASAP binding: {docs:?}"); - eprintln!("testdata corpus post-ASAP binding: {td:?}"); - - // Same totality guarantee as lowering: reaching here means `logical_plan` - // never panicked over any lowerable query in the corpus. - assert_eq!(docs.errored, 0, "post-ASAP binding errored: {docs:?}"); - assert_eq!(td.errored, 0, "post-ASAP binding errored: {td:?}"); -} - #[test] fn lowering_is_total_over_the_entire_corpus() { let docs = tally(DOCS); diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 816706b6c..fe2867e3f 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -137,48 +137,6 @@ pub fn declared_data(root: &Rc) -> DataWorkload { } } -/// Local computation candidates for frontend binding tests. Other computations -/// keep their default choices; a timed MILP archive is not a candidate catalog. -#[allow(dead_code)] -pub fn logical_candidates(root: Rc) -> Vec> { - local_computation_candidates(root).expect("generated computation candidates compose") -} - -fn local_computation_candidates(root: Rc) -> Result>, String> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::ir::QueryRoot; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .map_err(|error| error.to_string())?; - let mut choices = vec![vec![0; inventory.targets.len()]]; - for (index, target) in inventory.targets.iter().enumerate() { - for choice in 1..target.choice_count() { - let mut selected = vec![0; inventory.targets.len()]; - selected[index] = choice; - choices.push(selected); - } - } - choices - .into_iter() - .map(|choice| { - let mut roots = compose_logical_candidate(&inventory, &choice) - .map_err(|error| error.to_string())?; - match roots.remove(0) { - QueryRoot::Operator(root) => Ok(root), - QueryRoot::Scalar(_) => Err("expected an operator root".into()), - } - }) - .collect() -} - /// A composed logical candidate, or why it does not compose. #[allow(dead_code)] pub struct LogicalCandidate { @@ -205,15 +163,3 @@ pub fn require_composed_operator_candidates( }) .collect() } - -/// One generated ASAP candidate, or the exact computation if no local rule -/// applies. Corpus coverage tests exercise binding, not cost-based selection. -#[allow(dead_code)] -pub fn logical_plan(root: &Rc) -> Result, String> { - let candidates = local_computation_candidates(Rc::clone(root))?; - Ok(candidates - .iter() - .find(|node| node.contains_asap()) - .unwrap_or(&candidates[0]) - .clone()) -} diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs deleted file mode 100644 index 5c9856f4e..000000000 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::rc::Rc; - -use crate::planner_support::run_all_stages; -use crate::support; - -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; -use support::{logical_candidates, lower_promql}; - -/// The summary evaluation over a UnivMon state that `root` reads, if any. -fn univmon_estimate(root: &Rc) -> Option> { - OperatorNode::reachable(root).into_iter().find(|node| { - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &node.operator else { - return false; - }; - matches!(&summary_input.operator, Operator::ASAP(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) - if kind == &SketchAlgorithm::UnivMon) - }) -} - -#[test] -fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { - // An unmeasured heuristic remains inspectable but is never certified or - // automatically selected for a caller-visible bounded-error result. - // L2 is certified; entropy is not. - for query in ["entropy_over_time(m[5m])"] { - for target in [ - AccuracyTarget::exact(), - AccuracyTarget::max_error(0.02), - AccuracyTarget::max_error(0.02).failure_probability(0.01), - ] { - let root = lower_promql(query, target.clone()).unwrap(); - let generated = logical_candidates(Rc::clone(&root)); - let run = run_all_stages(root, target.clone()); - let unknown: Vec<_> = generated - .iter() - .filter(|node| univmon_estimate(node).is_some() && node.guarantee.is_none()) - .collect(); - if target.is_exact() { - assert!(unknown.is_empty(), "{query}"); - } else { - assert!(!unknown.is_empty(), "{query}"); - let (plan, _) = run - .selection - .as_ref() - .expect("a certified plan is selected"); - let roots = plan.query_roots().unwrap(); - let [asap_types::ir::QueryRoot::Operator(selected)] = roots.as_slice() else { - panic!("one operator result"); - }; - assert!(selected.guarantee.is_some(), "{query}"); - } - } - } -} diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 2a0f75c58..d7086e81a 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -237,73 +237,6 @@ pub mod planning { .collect() } - /// Exact computation and each explicit single-target rule application. - /// This fixture catalog is independent of the bounded MILP's visited plans. - pub fn explicit_local_candidates(root: Rc) -> Vec> { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let mut choices = std::collections::BTreeSet::from([vec![0; local.targets.len()]]); - for (target, options) in local.targets.iter().enumerate() { - for choice in 1..options.choice_count() { - let mut selected = vec![0; local.targets.len()]; - selected[target] = choice; - choices.insert(selected); - } - } - choices - .into_iter() - .map(|choice| { - match compose_logical_candidate(&local, &choice) - .expect("explicit local rule composes") - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator fixture"), - } - }) - .collect() - } - - /// One explicit plan using `family` wherever a local target supports it, - /// keeping all other computations exact. No workload Cartesian enumeration. - pub fn explicit_summary_candidate( - root: Rc, - family: asap_types::ir::schema::SketchAlgorithm, - ) -> Rc { - use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, - }; - use asap_types::stages::HowToComputeAggregateMeasure; - let local = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let choices = local.targets.iter().map(|target| target.ways_to_compute.iter() - .position(|candidate| matches!(candidate, HowToComputeAggregateMeasure::Summary(found) if *found == family)).unwrap_or(0)).collect::>(); - match compose_logical_candidate(&local, &choices) - .expect("explicit family choices compose") - .remove(0) - { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator fixture"), - } - } - /// The selected plan's operator roots for `root`. This is not the /// generated candidate catalog: rule-existence tests should use explicit /// fixtures. diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs deleted file mode 100644 index 4ed6ff21a..000000000 --- a/crates/integration-tests/tests/exact_composition.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! Issue #171 — composing exact operators with summary plans across -//! explicit update/evaluation boundaries, through the planner stages' -//! logical candidates (Stage 1). -//! -//! Covers grouped fine-to-coarse and identity folds over a summary -//! evaluation, phase-aware summary construction and pre/post-ASAP schemas, -//! along with pins for every exact-accumulator nesting under an outer -//! sketch. - -use std::rc::Rc; - -use asap_integration_tests::physical_export::physical_asap_dag; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::ExecutionTiming; -use asap_types::ir::schema::{DataType, Field, Schema}; - -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; - -// ── fixtures ──────────────────────────────────────────────────────────── - -fn node(op: NonASAPOp) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(op)) - .expect("fixture node derives its schema") -} - -fn metric_scan(labels: &[&str]) -> Rc { - let mut columns = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ]; - columns.extend( - labels - .iter() - .map(|n| Field::plain(*n, DataType::Utf8, true)), - ); - node(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: "latency".into(), - }, - predicates: vec![], - schema: Schema::with_time_index(columns, 0, vec![]), - }) -} - -fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { - node(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child, - }) -} - -/// `quantile by (zone, host) (latency)` — the fine-grained inner summary. -fn fine_quantile() -> Rc { - agg( - vec![2, 3], - default_quantile(0.99), - metric_scan(&["zone", "host"]), - ) -} - -/// The Stage 1 logical candidate of the single query `root` (planned at -/// ε = 0.01) whose outer aggregate stays exact over the inner summary -/// evaluation: the read-time composition. -fn composed(root: &Rc) -> Rc { - explicit_local_candidates(Rc::clone(root)) - .into_iter() - .find(|candidate| { - !matches!( - below_projects(candidate).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - ) && summary_evaluation(candidate).is_some() - }) - .expect("an exact fold over a summary evaluation") -} - -/// `node` with every `Project` above it skipped: Stage 1 restores a -/// rewritten aggregate's output names with a `Project`. -fn below_projects(mut node: &Rc) -> &Rc { - while let Some(NonASAPOp::Project { child, .. }) = node.non_asap() { - node = child; - } - node -} - -/// The first summary evaluation in `node`'s DAG. -fn summary_evaluation(node: &Rc) -> Option<&Rc> { - if matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. })) { - return Some(node); - } - node.children().into_iter().find_map(summary_evaluation) -} - -// ── step 1: pin every already-supported exact-accumulator nesting ─────── - -// ── direction 1: outer exact fold over an inner summary evaluation ──────── - -// ── summary construction phase ────────────────────────────────────────── - -// ── Physical export: explicit timing ───────────────────────────────────── - -#[test] -fn physical_export_carries_explicit_timing_for_a_composed_plan() { - let root = agg(vec![0], AggIntent::Max { col: None }, fine_quantile()); - let composed = composed(&root); - // Timing is explicit in the wire-6 DAG: the root finalizes the exact - // `max` state over the quantile evaluation, at query time. - let wire = physical_asap_dag(&composed); - let wire_root = wire.nodes.iter().find(|n| n.id == wire.roots[0]).unwrap(); - assert!(matches!( - wire_root.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - )); - assert_eq!(wire_root.output_state.timing, ExecutionTiming::QueryTime); -} diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index dfbe200d2..fa844281e 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -8,7 +8,6 @@ mod planner_layering_common; mod aggregate; mod binary_op; mod crate_graph; -mod exact_composition; mod filtered_aggregates; mod finer_aggregate_reuse; mod frontend_timestamps; @@ -23,15 +22,12 @@ mod operator_sharing; mod per_computation_candidates_sql; mod per_series_exact_execution; mod production_rollup_workloads; -mod promql_numeric_regressions; mod promql_to_post_asap; mod rollup_controlled; mod scan; mod schema; mod sharing_issue_regressions; -mod sketch_rollup; mod sql_cardinality; mod sql_frequency; mod sql_to_post_asap; -mod stage1_logical_asap_optimization; mod time_range; diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index f67394f8c..d6192608f 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -3,15 +3,11 @@ //! ranks. use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; +use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SummaryInputExpr}; -use asap_types::ir::{ASAPOp, OperatorNode, QueryRoot}; -use asap_types::stages::{ComputationTarget, HavingPlacement, HowToComputeAggregateMeasure}; +use asap_types::ir::QueryRoot; +use asap_types::stages::{ComputationTarget, HavingPlacement}; async fn targets(sql: &str) -> Vec { let catalog = SqlCatalog::new().with_table( @@ -144,199 +140,3 @@ async fn a_filter_or_group_key_over_an_estimate_has_no_guarantee() { } } } - -/// Whether `node` reads a DDSketch anywhere below it. -fn reads_ddsketch(node: &std::rc::Rc) -> bool { - use asap_types::ir::schema::SketchAlgorithm; - asap_types::ir::OperatorNode::reachable(node) - .iter() - .any(|node| { - matches!( - node.asap(), - Some(asap_types::ir::ASAPOp::SummaryAgg { - summary_type: asap_types::ir::schema::FieldDataType::Sketch(kind), - .. - }) if matches!(kind, SketchAlgorithm::DDSketch) - ) - }) -} - -/// A SQL fold over the groups of an approximate subquery counts rows the -/// series count of its source does not bound, so its δ names its own -/// row-count statistic (`group_row_count`), not the source's -/// `input_row_count`. -#[tokio::test] -async fn a_sql_fold_over_estimates_leaves_its_row_count_unbound() { - let catalog = SqlCatalog::new().with_table( - "flows", - Schema::new(vec![ - Field::plain("src", DataType::Utf8, false), - Field::plain("bytes", DataType::Float64, false), - ]), - ); - // q=1 is an endpoint, so this isolates the fold's row-count statistic - // without assuming a sign for SQL bytes or interpolating across zero. - let sql = "SELECT max(p) FROM \ - (SELECT src, approx_percentile_cont(bytes, 1.0) AS p FROM flows GROUP BY src) s"; - let root = lower_sql(sql, &catalog).await.unwrap(); - let candidates = vec![ - asap_integration_tests::planning::explicit_summary_candidate( - root, - SketchAlgorithm::DDSketch, - ), - ]; - assert!(candidates.iter().all(reads_ddsketch)); - assert!(!candidates.is_empty()); - for candidate in candidates { - let g = candidate.guarantee.clone().expect("a relative max"); - let delta = format!("{:?}", g.spec.failure_probability); - assert!(delta.contains("group_row_count"), "{delta}"); - assert!(!delta.contains("input_row_count"), "{delta}"); - } -} - -/// The item and weight columns of every heap sketch build in each composed -/// candidate of `sql` over `flows(key: key_type, bytes)` that picks a heap -/// sketch for its top-k. -async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, String)> { - heap_updates_over( - vec![ - Field::plain(key, key_type, false), - Field::plain("bytes", DataType::Float64, false), - ], - sql, - ) - .await -} - -/// [`heap_updates`] over `flows(fields)`. -async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String)> { - let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); - let root = lower_sql(sql, &catalog).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let name = |expr: &SummaryInputExpr| match expr { - SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), - SummaryInputExpr::Column(ColumnRef::Qualified { table, name }) => { - assert_eq!(table, "flows"); - name.clone() - } - // The ranked value when it is the child's one sample value. - SummaryInputExpr::Column(ColumnRef::SampleValue) => "".into(), - SummaryInputExpr::Tuple(items) => match items.as_slice() { - [SummaryInputExpr::Column(ColumnRef::Named(name))] => name.clone(), - [SummaryInputExpr::Column(ColumnRef::Qualified { table, name })] => { - assert_eq!(table, "flows"); - name.clone() - } - other => panic!("one item column: {other:?}"), - }, - SummaryInputExpr::Constant(value) => value.to_string(), - other => panic!("a named column or constant: {other:?}"), - }; - let mut updates = Vec::new(); - for (t, target) in inventory.targets.iter().enumerate() { - for c in 0..target.choice_count() { - let (candidate, _) = target.computation_for(c).unwrap(); - if !matches!( - candidate, - HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ) - ) { - continue; - } - let mut choice = vec![0; inventory.targets.len()]; - choice[t] = c; - let composed = compose_logical_candidate(&inventory, &choice) - .unwrap_or_else(|e| panic!("{candidate:?} composes: {e}")); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - for node in OperatorNode::reachable(root) { - if let Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - input, - .. - }) = node.asap() - { - updates.push((name(input.item.as_ref().unwrap()), name(&input.weight))); - } - } - } - } - updates -} - -const COUNT_RANKED_TOPK: &str = - "SELECT KEY, count(*) FROM flows GROUP BY KEY ORDER BY count(*) DESC LIMIT 10"; - -/// An Int64 key is not mistaken for the ranked value: both heap sketches -/// compose over raw rows, keyed by the key with unit weight. -#[tokio::test] -async fn topk_heap_over_an_int64_key_counts_raw_rows() { - let updates = heap_updates( - "port", - DataType::Int64, - &COUNT_RANKED_TOPK.replace("KEY", "port"), - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "port" && weight == "1"), - "{updates:?}" - ); -} - -/// A key named `value` is not the ranked value: the heap sketches are -/// fed raw rows with unit weight. -#[tokio::test] -async fn topk_heap_over_a_key_named_value_counts_raw_rows() { - let updates = heap_updates( - "value", - DataType::Utf8, - &COUNT_RANKED_TOPK.replace("KEY", "value"), - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "value" && weight == "1"), - "{updates:?}" - ); -} - -/// SQL's per-group top-k idiom (`ROW_NUMBER() OVER (PARTITION BY city -/// ORDER BY COUNT(*) DESC) … WHERE rn <= 5`) is a heavy-hitter top-k, as -/// PromQL `topk by (city)` is: both heap sketches compose, keyed by the -/// host and fed raw rows with unit weight. -#[tokio::test] -async fn rownumber_per_group_topk_offers_heap_sketches() { - let updates = heap_updates_over( - vec![ - Field::plain("city", DataType::Utf8, false), - Field::plain("host", DataType::Utf8, false), - ], - "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, ROW_NUMBER() OVER \ - (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM flows GROUP BY city, host) \ - WHERE rn <= 5", - ) - .await; - assert_eq!(updates.len(), 2, "{updates:?}"); - assert!( - updates - .iter() - .all(|(item, weight)| item == "host" && weight == "1"), - "{updates:?}" - ); -} diff --git a/crates/integration-tests/tests/promql_numeric_regressions.rs b/crates/integration-tests/tests/promql_numeric_regressions.rs deleted file mode 100644 index f941653b0..000000000 --- a/crates/integration-tests/tests/promql_numeric_regressions.rs +++ /dev/null @@ -1,332 +0,0 @@ -//! Numeric regression fixtures: actual PromQL lowering plus numeric update/evaluation checks. -//! The count/sum interpreter below verifies planner update semantics, not a deployed backend. -use asap_integration_tests::fixtures::lower_promql; -use asap_integration_tests::physical_export::physical_asap_dag; -use asap_integration_tests::planning::{explicit_local_candidates, explicit_summary_candidate}; -use asap_types::ir::operator::Reduction; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{ExactKind, FieldDataType, SummaryInputExpr, SummaryUpdate}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use std::rc::Rc; - -/// The first explicitly generated local-rule fixture containing an ASAP -/// operator, or its exact computation when no summary is applicable. -fn plan(query: &str, accuracy: AccuracyTarget) -> Rc { - let pre = lower_promql(query, accuracy.clone()).unwrap(); - let candidates = explicit_local_candidates(pre); - candidates - .iter() - .find(|node| node.contains_asap()) - .unwrap_or(&candidates[0]) - .clone() -} -fn aggregate(node: &OperatorNode) -> (&FieldDataType, &SummaryUpdate, &Reduction) { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - input, - reduction, - .. - }) => (family, input, reduction), - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => aggregate(summary_input), - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => aggregate(child), - // A value operation (Project/Filter/Sort/Limit/...) over the state. - Operator::NonASAP(_) if node.children().len() == 1 && node.contains_asap() => { - aggregate(node.children()[0]) - } - other => panic!("not a maintained accumulator: {other:?}"), - } -} -fn contribution(family: &FieldDataType, update: &SummaryUpdate, value: f64) -> f64 { - if matches!(family, FieldDataType::ExactAggregate(ExactKind::Count, _)) { - return 1.; - } - match update.weight { - SummaryInputExpr::Constant(v) => v, - SummaryInputExpr::Column(ColumnRef::SampleValue) => value, - ref other => panic!("unexpected update {other:?}"), - } -} - -/// Target count depends on series multiplicity, never distinct values or health. -#[test] -fn count_up_counts_targets_even_when_values_repeat_or_change_sign() { - let node = plan("count(up)", AccuracyTarget::exact()); - let (family, update, _) = aggregate(&node); - assert!(matches!( - family, - FieldDataType::ExactAggregate(ExactKind::Count, _) - )); - for values in [[1., 1., 1.], [1., 1., 0.], [0., 0., 0.], [-1., -1., -1.]] { - assert_eq!( - values - .into_iter() - .map(|v| contribution(family, update, v)) - .sum::(), - 3. - ); - } - physical_asap_dag(&node); -} - -/// Ten samples give count ten, whereas sum retains the signed sample values. -#[test] -fn window_counts_and_sums_distinguish_one_zero_three_and_negative_values() { - for (query, kind, is_count) in [ - ("count_over_time(up[5m])", ExactKind::Count, true), - ("sum_over_time(up[5m])", ExactKind::Sum, false), - ] { - let node = plan(query, AccuracyTarget::exact()); - let (family, update, reduction) = aggregate(&node); - assert!(matches!(family, FieldDataType::ExactAggregate(k, _) if *k == kind)); - assert_eq!(*reduction, Reduction::PerEntity); - for value in [1., 0., 3., -3.] { - let got: f64 = (0..10).map(|_| contribution(family, update, value)).sum(); - assert_eq!(got, if is_count { 10. } else { value * 10. }); - } - physical_asap_dag(&node); - } -} - -/// Exact summary candidates must exist, rather than merely retaining the original query. -#[test] -fn sum_rate_and_increase_have_real_exact_accumulator_nodes() { - for (query, kind) in [ - ("sum by(job)(up)", ExactKind::Sum), - ("rate(requests_total[5m])", ExactKind::Rate), - ("increase(requests_total[5m])", ExactKind::Increase), - ] { - let node = plan(query, AccuracyTarget::exact()); - let (family, _, _) = aggregate(&node); - assert!(matches!(family, FieldDataType::ExactAggregate(k, _) if *k == kind)); - assert!(node.guarantee.as_ref().unwrap().is_exact()); - physical_asap_dag(&node); - } -} - -/// The PromQL quantile over a DDSketch: the linear interpolation at rank -/// `q (n - 1)` between the sketch's two neighbouring order statistics. -fn interpolated_quantile(sketch: &asap_sketchlib::DdSketch, q: f64) -> f64 { - sketch.quantile_interpolated(q).unwrap().unwrap() -} - -/// A finite, nonzero approximate denominator does not certify a ratio bound. -#[test] -fn checked_ratio_must_not_certify_cross_zero_interpolation() { - let root = lower_promql( - "quantile_over_time(0.5, data[5m]) / quantile_over_time(0.9, data[5m])", - AccuracyTarget::max_error(0.01), - ) - .unwrap(); - let node = explicit_summary_candidate(root, asap_types::ir::schema::SketchAlgorithm::DDSketch); - assert_eq!(OperatorNode::reachable(&node).iter().filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(kind), .. }) if *kind == asap_types::ir::schema::SketchAlgorithm::DDSketch)).count(), 2, - "the signed interpolation counterexample requires two explicit DDSketch builds"); - assert!( - matches!(node.operator, Operator::NonASAP(NonASAPOp::BinaryOp { .. })) - && node.contains_asap(), - "direct quantile ratio should remain an available candidate" - ); - assert!(node.guarantee.is_none(), "{:?}", node.guarantee); - use asap_accuracy::propagation::bind_guarantees; - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - let root = asap_types::ir::QueryRoot::Operator(node.clone()); - assert!(bind_guarantees(&root, &Clone::clone).is_none()); - let proven = bind_guarantees(&root, &|guarantee| { - guarantee.bind(&|statistic| { - (statistic == raw_data_samples_sign_statistic("data")).then_some(1.0) - }) - }) - .expect("a declared nonnegative source recovers the ratio contract"); - assert!(proven.spec.error.evaluate().unwrap() <= 0.01, "{proven:?}"); - assert!( - node.guarantee.is_none(), - "conditional sizing must not attach a hypothetical sign proof" - ); - physical_asap_dag(&node); - // Keep the actual signed-sketch counterexample: division guards alone pass - // even though the quantile interpolation does not preserve relative error. - // The sketch reads its minimum and maximum exactly, so the interpolated - // order statistics are interior ones. - let alpha = (0.01 - 8.0 * f64::EPSILON) / 2.01; - let values = [-2., -1., 1.011, 2.]; - let estimate = |q| { - let mut sketch = asap_sketchlib::DdSketch::new(alpha); - for value in values { - sketch.try_add(&value).unwrap(); - } - interpolated_quantile(&sketch, q) - }; - let numerator = estimate(0.5); - let denominator = estimate(0.9); - let got = numerator / denominator; - assert!( - numerator.is_finite() && denominator.is_finite() && denominator != 0. && got.is_normal() - ); - let exact_quantile = |q: f64| { - let rank = q * (values.len() - 1) as f64; - let (lower, upper) = (rank.floor() as usize, rank.ceil() as usize); - values[lower] * (1. - rank.fract()) + values[upper] * rank.fract() - }; - let want = exact_quantile(0.5) / exact_quantile(0.9); - let error = (got - want).abs() / want.abs(); - assert!( - error > 0.01, - "fixture must expose cancellation beyond the requested budget" - ); -} - -/// Interpolated DDS quantiles require a source-domain proof; extrema do not -/// interpolate, and a computed value must not inherit its source's sign. -#[test] -fn ddsketch_interpolation_requires_a_nonnegative_domain() { - use asap_accuracy::propagation::bind_guarantees; - use asap_accuracy::test_support::raw_data_samples_sign_statistic; - use asap_types::ir::{schema::SketchAlgorithm, QueryRoot}; - for (query, endpoint, raw_samples) in [ - ("quantile_over_time(0, data[5m])", true, true), - ("quantile_over_time(1, data[5m])", true, true), - ("quantile_over_time(0.5, data[5m])", false, true), - ("quantile(0.5, data - 100)", false, false), - ] { - let node = explicit_summary_candidate( - lower_promql(query, AccuracyTarget::max_error(0.01)).unwrap(), - SketchAlgorithm::DDSketch, - ); - let bound = node - .guarantee - .as_ref() - .and_then(|g| g.spec.error.evaluate()); - assert_eq!(bound.is_some(), endpoint, "{query}"); - let proven = bind_guarantees(&QueryRoot::Operator(node), &|guarantee| { - guarantee.bind(&|statistic| { - (statistic == raw_data_samples_sign_statistic("data")).then_some(1.0) - }) - }); - assert_eq!( - proven.and_then(|g| g.spec.error.evaluate()).is_some(), - raw_samples, - "{query}" - ); - } -} - -/// Rewriting the inner average (its own target, as sum and count -/// accumulators) must not invalidate an otherwise usable outer sketch. -#[test] -fn quantile_over_temporal_average_keeps_a_legal_candidate() { - for query in [ - "quantile(0.9, avg_over_time(a[5m]))", - "quantile(0.9, avg_over_time(a[5m]) + avg_over_time(b[5m]))", - ] { - let accuracy = AccuracyTarget::max_error(0.01); - let pre = lower_promql(query, accuracy.clone()).unwrap(); - let node = explicit_local_candidates(pre) - .into_iter() - .find(|node| { - OperatorNode::reachable(node).iter().any(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .unwrap_or_else(|| panic!("outer sketch candidate must survive: {query}")); - physical_asap_dag(&node); - } -} - -#[test] -fn sketch_counts_use_unit_weights_and_signed_sums_keep_value_weights() { - use asap_types::ir::schema::{NonNegativeWeightProof, SketchAlgorithm, WeightDomain}; - for is_count in [true, false] { - let query = if is_count { - "topk(1, count_over_time(up[5m]))" - } else { - "topk(1, sum_over_time(up[5m]))" - }; - let accuracy = AccuracyTarget::max_error(0.01); - let pre = lower_promql(query, accuracy.clone()).unwrap(); - let candidates = explicit_local_candidates(pre); - // The heap state each summarized candidate builds. - let heap = |node: &Rc| { - OperatorNode::reachable(node).into_iter().find(|n| { - matches!( - n.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - .. - }) - ) - }) - }; - let wanted = if is_count { - SketchAlgorithm::CmsWithHeap - } else { - SketchAlgorithm::CountSketchWithHeap - }; - let (node, state) = candidates - .iter() - .find_map(|node| { - let state = heap(node)?; - let (family, _, _) = aggregate(&state); - matches!(family, FieldDataType::Sketch(kind) if kind == &wanted) - .then_some((node, state)) - }) - .expect("weighted sketch candidate"); - let (family, update, _) = aggregate(&state); - if is_count { - assert_eq!(update.weight, SummaryInputExpr::Constant(1.)); - assert_eq!( - update.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount - } - ); - } else { - assert_eq!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ); - // A signed sum's Count-Min heap carries no non-negative proof, - // so plan selection rejects it. - for state in candidates.iter().filter_map(heap) { - let (family, update, _) = aggregate(&state); - if matches!(family, FieldDataType::Sketch(kind) if kind == &SketchAlgorithm::CmsWithHeap) - { - assert!(!matches!( - update.weight_domain, - WeightDomain::NonNegative { .. } - )); - } - } - } - for value in [1., 0., 3., -3.] { - let key = asap_sketchlib::DataInput::Str("a"); - let mut cms: asap_sketchlib::CountMinF64 = - asap_sketchlib::CountMinSketch::with_dimensions(5, 128); - let mut cs = - asap_sketchlib::CountSketch::>::with_dimensions( - 5, 128, - ); - for _ in 0..10 { - let weight = contribution(family, update, value); - if is_count { - cms.update_weighted(&key, weight); - } else { - cs.update_weighted(&key, weight); - } - } - let got = if is_count { - cms.estimate(&key) - } else { - cs.estimate(&key) - }; - assert_eq!(got, if is_count { 10. } else { 10. * value }); - } - physical_asap_dag(node); - } -} diff --git a/crates/integration-tests/tests/sketch_rollup.rs b/crates/integration-tests/tests/sketch_rollup.rs deleted file mode 100644 index 2422d8ae3..000000000 --- a/crates/integration-tests/tests/sketch_rollup.rs +++ /dev/null @@ -1,371 +0,0 @@ -//! A regrouping `SummaryMerge` (rollup) answers a coarse grouping from the -//! states of a finer one: `GROUP BY job` from states built `GROUP BY region, -//! job`, merging each job's per-region states. For every mergeable sketch -//! the planner offers a SQL grouped query, the rolled-up answer equals the -//! direct build's (deterministic merges) or is within the sketch's bound -//! (KLL), and its coverage is the direct build's. - -use std::collections::BTreeMap; -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_integration_tests::planning::explicit_local_candidates; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::summary_coverage::CoverageError; -use asap_types::ir::schema::{ - DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SketchAlgorithm, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; -use futures::StreamExt; - -const REGIONS: [&str; 3] = ["us", "eu", "ap"]; -const JOBS: [&str; 4] = ["api", "web", "db", "cache"]; - -fn catalog() -> SqlCatalog { - SqlCatalog::new().with_table( - "http", - Schema::new(vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("job", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("user_id", DataType::Float64, false), - ]), - ) -} - -/// 6,000 rows over every (region, job): a heavy-tailed latency and a user -/// id from 800, from a fixed linear congruential generator. -fn rows() -> Vec<(&'static str, &'static str, f64, f64)> { - let mut state: u64 = 0x2545_f491_4f6c_dd1d; - let mut next = move || { - state = state - .wrapping_mul(6_364_136_223_846_793_005) - .wrapping_add(1_442_695_040_888_963_407); - (state >> 33) as f64 / (1u64 << 31) as f64 - }; - (0..6_000) - .map(|_| { - let region = REGIONS[(next() * 3.0) as usize % 3]; - let job = JOBS[(next() * 4.0) as usize % 4]; - let latency = 10.0 / (1.0 - next()).max(1e-6).sqrt(); - let user = (next() * 800.0).floor(); - (region, job, latency, user) - }) - .collect() -} - -/// `node` with every Int64 column typed Float64. The Planner types a SQL -/// `COUNT(DISTINCT)` estimate Int64 and the executor returns Float64, a -/// known SQL execution gap this test does not cover. -fn as_float64(node: &Rc) -> Rc { - let rebuilt = node.with_new_children(as_float64).unwrap(); - let mut schema = node.schema.clone(); - for field in &mut schema.fields { - if field.dtype == FieldDataType::Plain(DataType::Int64) { - field.dtype = FieldDataType::Plain(DataType::Float64); - } - } - Rc::new( - OperatorNode::with_schema(rebuilt.operator, schema).with_guarantee(node.guarantee.clone()), - ) -} - -/// The answers of the logical DAG `root` over [`rows`], by group key. -fn execute(root: &Rc) -> BTreeMap { - let root = &as_float64(root); - let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) - }) - .expect("one scan"); - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) = &scan.payload else { - unreachable!() - }; - let schema = Arc::new(scan.output_schema.clone()); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let table: Vec> = rows() - .into_iter() - .map(|(region, job, latency, user)| { - vec![ - Value::Timestamp(0), - Value::Utf8(region.into()), - Value::Utf8(job.into()), - Value::Float64(latency), - Value::Float64(user), - ] - }) - .collect(); - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), table).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - // The UnivMon states planned here are large: the merge's copy of - // twelve of them exceeds the default limit. - Limits { - max_bytes: 1 << 32, - ..Limits::default() - }, - ) - .unwrap(); - futures::executor::block_on(async { - let mut stream = bound.execute(plan.roots(), context).unwrap().remove(0); - let mut answers = BTreeMap::new(); - while let Some(batch) = stream.next().await { - for row in batch.unwrap().to_rows() { - let (keys, value) = row.split_at(row.len() - 1); - let key = keys - .iter() - .map(|value| match value { - Value::Utf8(text) => text.to_string(), - other => format!("{other:?}"), - }) - .collect::>() - .join(","); - let value = match value[0] { - Value::Float64(value) => value, - Value::Int64(value) => value as f64, - ref other => panic!("not a number: {other:?}"), - }; - assert!(answers.insert(key, value).is_none(), "one row per group"); - } - } - answers - }) -} - -/// The only `SummaryAgg` of `root`. -fn build(root: &Rc) -> Option> { - let builds: Vec<_> = OperatorNode::reachable(root) - .into_iter() - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - match builds.as_slice() { - [build] => Some(Rc::clone(build)), - _ => None, - } -} - -/// `node` with `from` replaced by `to`. -fn replace( - node: &Rc, - from: &Rc, - to: &Rc, -) -> Rc { - if Rc::ptr_eq(node, from) { - return Rc::clone(to); - } - Rc::new( - node.with_new_children(|child| replace(child, from, to)) - .unwrap() - .with_guarantee(node.guarantee.clone()), - ) -} - -/// `coarse`, a build by `job`, as the same build by `(region, job)` -/// regrouped by `job`: each output state merges its job's per-region -/// states. -fn rolled_up(coarse: &OperatorNode) -> Rc { - let Some(ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction: Reduction::Reduce(by), - filter, - }) = coarse.asap() - else { - panic!("a grouped build: {:?}", coarse.operator) - }; - let region = child.schema.column_id("region").unwrap(); - assert_eq!(by.keys().len(), 1); - let finer = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - summary_type: family.clone(), - input: input.clone(), - reduction: Reduction::by(vec![region, by.keys()[0]]), - filter: filter.clone(), - })) - .unwrap(); - OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![finer], - group_by: Some(vec![1]), - })) - .unwrap() -} - -/// Every local sketch rule for `sql` (one build by `job`), each with its -/// rollup from `(region, job)`: the family, the direct root and the -/// rolled-up root. Explicit rule fixtures keep this execution test independent -/// of the search archive and its tighter, potentially much larger capacities. -async fn direct_and_rolled_up( - sql: &str, -) -> Vec<(SketchAlgorithm, Rc, Rc)> { - let root = lower_sql(sql, &catalog()).await.unwrap(); - let mut out = Vec::new(); - for candidate in explicit_local_candidates(root) { - let Some(build) = build(&candidate) else { - continue; - }; - let Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(kind), - .. - }) = build.asap() - else { - continue; - }; - let merge = rolled_up(&build); - let rolled = replace(&candidate, &build, &merge); - rolled.validate_structure().unwrap(); - out.push((*kind, candidate, rolled)); - } - out -} - -/// Distinct users per job: KMV, HLL and UnivMon states of `(region, job)` -/// merged per job equal each job's state built directly (KMV keeps the k -/// smallest hashes of the union, HLL each register's maximum, UnivMon's -/// counters add), so every answer is the direct build's. -#[tokio::test] -async fn distinct_counts_rolled_up_equal_the_direct_build() { - let pairs = - direct_and_rolled_up("SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job") - .await; - let families: Vec<_> = pairs.iter().map(|(family, ..)| *family).collect(); - assert!(families.contains(&SketchAlgorithm::Kmv), "{families:?}"); - for (family, direct, rolled) in pairs { - let (direct, rolled) = (execute(&direct), execute(&rolled)); - assert_eq!(direct.len(), JOBS.len()); - assert_eq!(rolled, direct, "{family:?}"); - } -} - -/// p99 latency per job. DDSketch's bucket counts add, so the merged state -/// is the direct build's and so is its answer. KLL's compactions are -/// randomized, so its rolled-up answer differs, but stays within its rank -/// bound of the exact p99, as the direct build's does. -#[tokio::test] -async fn quantiles_rolled_up_keep_the_sketch_bound() { - const EPSILON: f64 = 0.05; - { - let pairs = direct_and_rolled_up( - "SELECT job, approx_percentile_cont(latency, 0.99) AS value FROM http GROUP BY job", - ) - .await; - for (family, direct, rolled) in pairs { - let (direct, rolled) = (execute(&direct), execute(&rolled)); - assert_eq!(direct.len(), JOBS.len()); - match family { - SketchAlgorithm::DDSketch => assert_eq!(rolled, direct), - SketchAlgorithm::Kll => { - for (job, answer) in rolled { - let mut latencies: Vec = rows() - .into_iter() - .filter(|row| row.1 == job) - .map(|row| row.2) - .collect(); - latencies.sort_by(f64::total_cmp); - let rank = latencies.partition_point(|&v| v <= answer) as f64 - / latencies.len() as f64; - assert!( - (rank - 0.99).abs() <= EPSILON, - "{job}: rank {rank} of {answer}" - ); - } - } - other => panic!("unexpected quantile family {other:?}"), - } - } - } -} - -/// A regrouping merge keys only by group keys of its input states (or by -/// none, a global aggregate), and never merges counter states of different -/// groups. -#[tokio::test] -async fn invalid_rollups_are_rejected() { - let root = lower_sql( - "SELECT job, COUNT(DISTINCT user_id) AS value FROM http GROUP BY job", - &catalog(), - ) - .await - .unwrap(); - let candidate = explicit_local_candidates(root) - .into_iter() - .find(|candidate| { - build(candidate).is_some_and(|build| { - matches!( - build.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - .. - }) - ) - }) - }) - .unwrap(); - let merge = rolled_up(&build(&candidate).unwrap()); - let finer = Rc::clone(merge.children()[0]); - let regroup = |keys: Vec| { - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![Rc::clone(&finer)], - group_by: Some(keys), - })) - .map_err(|error| error.to_string()) - }; - // The state column is no key; nor is a column twice. - for keys in [vec![2], vec![1, 1]] { - assert!(regroup(keys.clone()).is_err(), "{keys:?}"); - } - // No keys: the global build over the same rows. - assert!(regroup(vec![]).is_ok()); - // Both keys, swapped, are still group keys of the input. - assert!(regroup(vec![1, 0]).is_ok()); - // Rate states merge consecutive windows of one series, not groups. - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - .. - }) = finer.asap() - else { - unreachable!() - }; - let rates = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - summary_type: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - input: input.clone(), - reduction: reduction.clone(), - filter: None, - })) - .unwrap(); - let merged_rates = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![rates], - group_by: Some(vec![1]), - })); - assert!(merged_rates - .unwrap_err() - .to_string() - .contains(&CoverageError::SummaryTypeCannotRegroup.to_string())); -} diff --git a/crates/integration-tests/tests/sql_to_post_asap.rs b/crates/integration-tests/tests/sql_to_post_asap.rs index f8484222f..c41aa4062 100644 --- a/crates/integration-tests/tests/sql_to_post_asap.rs +++ b/crates/integration-tests/tests/sql_to_post_asap.rs @@ -19,28 +19,13 @@ use std::rc::Rc; -use asap_frontend_sql::{lower_sql, lower_sql_dialect, SqlCatalog}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_integration_tests::physical_export::physical_asap_dag; -use asap_integration_tests::planning::{ - logical_candidates, logical_candidates_ignoring_guarantees, -}; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::physical_export::{EdgeRole, PhysicalASAPNodeId, PhysicalASAPOperatorPayload}; +use asap_integration_tests::planning::logical_candidates_ignoring_guarantees; +use asap_types::ir::physical_export::{PhysicalASAPNodeId, PhysicalASAPOperatorPayload}; use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ - ExactKind, ExactParams, FieldDataType, SketchAlgorithm, SketchStatistic, SummaryUpdate, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr}; -use asap_types::workload::SqlDialect; - -/// The first logical candidate (Stage 1, in enumeration order) of `root` as -/// a query at `accuracy` that uses an ASAP operator, or the plain exact one -/// when none does: the summary-bound form the pins below inspect. -fn realize(root: &Rc, accuracy: AccuracyTarget) -> Rc { - first_summary_bound(logical_candidates(Rc::clone(root), accuracy)) -} +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, Predicate}; /// [`realize`] with every output accepted whatever its guarantee: a value /// filter over an estimate has no guarantee, so Pass 1 prunes the summary @@ -68,30 +53,6 @@ fn first_summary_bound(candidates: Vec>) -> Rc { node } -/// `node` with every `Project` above it skipped: Stage 1 restores a -/// rewritten aggregate's output names with a `Project`, and SQL wraps every -/// query in one. -fn below_projects(mut node: &Rc) -> &Rc { - while let Some(NonASAPOp::Project { child, .. }) = node.non_asap() { - node = child; - } - node -} - -/// The first `SummaryAgg` reached from `node` through unary nodes, estimates -/// and finalized accumulators. -fn first_summary_agg(node: &Rc) -> &Rc { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { .. }) => node, - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - first_summary_agg(summary_input) - } - _ => first_summary_agg( - unary_child(node).unwrap_or_else(|| panic!("no SummaryAgg below {:?}", node.operator)), - ), - } -} - /// The single input of a unary non-ASAP node (Project, Filter, Sort, ...) or /// of a `FinalizeExactAccumulator`; `None` for anything else. fn unary_child(node: &OperatorNode) -> Option<&Rc> { @@ -145,263 +106,6 @@ async fn lower(sql: &str) -> Rc { .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")) } -#[tokio::test] -async fn clickhouse_temporal_sql_reuses_rate_and_increase_physical_summaries() { - for (function, expected) in [ - ( - "asap_rate", - FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - ), - ( - "asap_increase", - FieldDataType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - ), - ] { - let sql = format!( - "SELECT service, {function}(latency, ts, 300000) AS v \ - FROM metrics WHERE bytes > 0 GROUP BY service" - ); - let pre_asap = lower_sql_dialect(&sql, &catalog(), SqlDialect::ClickhouseSQL) - .await - .expect("explicit temporal SQL must lower"); - let planned = realize(&pre_asap, AccuracyTarget::exact()); - let physical = first_summary_agg(&planned); - let Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - reduction, - child, - .. - }) = &physical.operator - else { - panic!("expected a shared SummaryAgg, got {:?}", physical.operator); - }; - assert_eq!(family, &expected); - assert_eq!(reduction, &Reduction::PerEntity); - assert!( - is_kept_non_asap(child), - "expected a retained temporal SQL input, got {:?}", - child.operator - ); - assert!( - matches!(child.non_asap(), Some(NonASAPOp::TimeRange { range, child, .. }) - if *range == std::time::Duration::from_secs(300) - && matches!(child.non_asap(), Some(NonASAPOp::Project { .. }))) - ); - } -} - -#[tokio::test] -async fn clickhouse_outer_sum_recursively_binds_inner_temporal_aggregate() { - for (function, window_ms) in [ - ("asap_rate", 300_000), - ("asap_rate", 3_600_000), - ("asap_increase", 300_000), - ] { - let sql = format!( - "SELECT sum(v) AS value FROM (\ - SELECT service, {function}(latency, ts, {window_ms}) AS v \ - FROM metrics GROUP BY service)" - ); - let pre_asap = lower_sql_dialect(&sql, &catalog(), SqlDialect::ClickhouseSQL) - .await - .expect("nested temporal SQL must lower"); - let root = realize(&pre_asap, AccuracyTarget::exact()); - - fn has_temporal_summary(node: &OperatorNode) -> bool { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: - FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), - .. - }) => true, - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - has_temporal_summary(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => has_temporal_summary(child), - _ => unary_child(node).is_some_and(|child| has_temporal_summary(child)), - } - } - assert!( - has_temporal_summary(&root), - "inner {function} was hidden: {root:?}" - ); - // The outer sum is an exact Sum state over the finalized per-series - // values, so the export keeps both summaries. - let Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - .. - }) = &first_summary_agg(&root).operator - else { - unreachable!() - }; - assert_eq!( - family, - &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - let dag = physical_asap_dag(&root); - assert_eq!( - dag.nodes - .iter() - .filter(|node| matches!( - node.payload, - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { .. }) - )) - .count(), - 2 - ); - } -} - -#[tokio::test] -async fn sql_full_query_retains_project_and_binds_inner_aggregate() { - let pre_asap = lower("SELECT approx_percentile_cont(latency, 0.99) AS p99 FROM metrics").await; - let Some(NonASAPOp::Project { - cols: expected_cols, - qualifier: expected_qualifier, - .. - }) = pre_asap.non_asap() - else { - panic!("sanity: a SQL root is a Project, unlike lower_promql's bare Aggregate"); - }; - let root = realize(&pre_asap, AccuracyTarget::max_error(0.01)); - let Some(NonASAPOp::Project { - child, - cols, - qualifier, - }) = root.non_asap() - else { - panic!("expected retained Project root, got {:?}", root.operator); - }; - assert_eq!(cols, expected_cols, "projection expressions and aliases"); - assert_eq!(qualifier, expected_qualifier, "projection qualifier"); - assert_eq!(root.schema.fields[0].name, "p99", "project output schema"); - assert_eq!( - root.schema.fields[0].dtype, - FieldDataType::Plain(DataType::Float64) - ); - assert!( - matches!( - below_projects(child).operator, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - ), - "the Aggregate under Project must be summary-bound" - ); -} - -/// A relational join remains a read-time node while both derived-table -/// aggregates are independently selected as physical summaries. -#[tokio::test] -async fn sql_join_recursively_binds_both_temporal_aggregate_children() { - let pre_asap = lower_sql_dialect( - "SELECT a.service, a.v / b.v AS ratio FROM \ - (SELECT service, asap_rate(latency, ts, 300000) AS v FROM metrics WHERE service='errors' GROUP BY service) a \ - INNER JOIN \ - (SELECT service, asap_rate(latency, ts, 300000) AS v FROM metrics WHERE service='requests' GROUP BY service) b \ - ON b.service=a.service", - &catalog(), - SqlDialect::ClickhouseSQL, - ) - .await - .expect("two-subquery rate ratio must lower"); - let root = realize(&pre_asap, AccuracyTarget::exact()); - let Some(NonASAPOp::Project { - child: join, cols, .. - }) = root.non_asap() - else { - panic!( - "expected Project above relational join, got {:?}", - root.operator - ); - }; - assert!(matches!( - &cols[1].expr, - ScalarExpr::Arithmetic { - op: asap_types::ir::scalar::ArithmeticOpKind::Div, - .. - } - )); - let Some(NonASAPOp::Join { - left, - right, - kind, - pred, - }) = join.non_asap() - else { - panic!( - "expected read-time relational join, got {:?}", - join.operator - ); - }; - assert_eq!(kind, &asap_types::ir::operator::JoinKind::Inner); - assert!(matches!( - &pred.0, - ScalarExpr::Compare { - left, - op: asap_types::ir::scalar::CompareOpKind::Eq, - right, - .. - } if matches!( - (left.as_ref(), right.as_ref()), - (ScalarExpr::Column(0), ScalarExpr::Column(2)) - | (ScalarExpr::Column(2), ScalarExpr::Column(0)) - ) - )); - assert_eq!( - join.schema - .fields - .iter() - .map(|field| field.name.as_str()) - .collect::>(), - vec!["service", "v", "service", "v"] - ); - for child in [left, right] { - let Some(NonASAPOp::Project { - child: aggregate, .. - }) = child.non_asap() - else { - panic!( - "derived table Project was not retained: {:?}", - child.operator - ); - }; - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: aggregate }) = - &aggregate.operator - else { - panic!("derived table Project must consume finalized exact values"); - }; - assert!(matches!( - aggregate.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - .. - }) - )); - } - assert!(join - .guarantee - .as_ref() - .is_some_and(|value| value.is_exact())); - let dag = physical_asap_dag(&root); - let join_id = dag - .nodes - .iter() - .find(|node| { - matches!( - node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Join { .. }) - ) - }) - .expect("relational join node") - .id; - let roles = dag - .edges - .iter() - .filter(|edge| edge.consumer == join_id) - .map(|edge| edge.role) - .collect::>(); - assert_eq!(roles, vec![EdgeRole::Left, EdgeRole::Right]); -} - #[tokio::test] async fn sql_relational_parents_retain_summary_bound_aggregate() { let pre_asap = lower( @@ -534,158 +238,3 @@ async fn sql_filter_keeps_read_predicate_and_summary_population_selection() { PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Filter { pred, .. }) if *pred == expected_wire ))); } - -#[tokio::test] -async fn sql_quantile_binds_kll_sketch_over_named_column() { - let pre_asap = lower("SELECT approx_percentile_cont(latency, 0.99) FROM metrics").await; - let planned = asap_integration_tests::planning::explicit_summary_candidate( - pre_asap, - SketchAlgorithm::Kll, - ); - let root = below_projects(&planned); - - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!("expected SummaryEstimate root, got {:?}", root.operator); - }; - assert!(matches!(query, SketchStatistic::Quantile { q } if *q == 0.99)); - assert_eq!( - root.schema.fields.len(), - 1, - "no GROUP BY — a single output column" - ); - assert_eq!( - root.schema.fields[0].dtype, - FieldDataType::Plain(DataType::Float64), - "the summary-state type must not propagate past the estimate" - ); - - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - summary_type: family, - input, - reduction, - .. - }) = &summary_input.operator - else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!(family, &FieldDataType::Sketch(SketchAlgorithm::Kll)); - assert_eq!( - input, - &SummaryUpdate::column(ColumnRef::Qualified { - table: "metrics".into(), - name: "latency".into(), - }), - "SQL binds the intent's own named input column, not a synthetic sample value" - ); - assert_eq!( - reduction, - &Reduction::by(vec![]), - "global quantile — no GROUP BY, full reduction" - ); - assert_eq!( - summary_input.schema.fields[0].dtype, - FieldDataType::Sketch(SketchAlgorithm::Kll) - ); - - assert!( - is_kept_non_asap(child), - "expected a kept non-ASAP leaf, got {:?}", - child.operator - ); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Scan { .. }))); - assert!( - child - .schema - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_))), - "logical edges carry only plain columns" - ); -} - -/// `SELECT COUNT(DISTINCT service) FROM metrics` at ε = 0.01 lowers to -/// `AggIntent::Cardinality` (`sql_lowering.rs::count_distinct_is_cardinality`) -/// — this fixture explicitly selects the HLL rule, independently of the -/// optimizer's cost preference. HLL is sized from ε and the default δ = 0.01 (#694): `m ≥ -/// 1/(2ε²δ)` reaches precision 18, the largest. -#[tokio::test] -async fn sql_count_distinct_with_epsilon_binds_hll_over_named_column() { - let pre_asap = lower("SELECT COUNT(DISTINCT service) FROM metrics").await; - let planned = asap_integration_tests::planning::explicit_summary_candidate( - pre_asap, - SketchAlgorithm::Hll, - ); - let root = below_projects(&planned); - - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!("expected SummaryEstimate root, got {:?}", root.operator); - }; - assert!(matches!(query, SketchStatistic::Cardinality)); - assert_eq!( - root.schema.fields[0].dtype, - FieldDataType::Plain(DataType::Int64), - "COUNT(DISTINCT …) reads back out as an integer count" - ); - - let Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - input, - reduction, - .. - }) = &summary_input.operator - else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!(family, &FieldDataType::Sketch(SketchAlgorithm::Hll)); - assert_eq!( - input, - &SummaryUpdate::column(ColumnRef::Qualified { - table: "metrics".into(), - name: "service".into(), - }) - ); - assert_eq!(reduction, &Reduction::by(vec![])); -} - -#[tokio::test] -async fn map_projection_export_keeps_the_map_over_an_avg_of_exact_states() { - let pre = lower_sql_dialect( - "SELECT map('job', t.service) AS labels, t.avg_bytes FROM (SELECT service, AVG(bytes) AS avg_bytes FROM metrics GROUP BY service) t WHERE t.avg_bytes > 100", - &catalog(), SqlDialect::ClickhouseSQL, - ).await.unwrap(); - let root = realize(&pre, AccuracyTarget::exact()); - let dag = physical_asap_dag(&root); - assert!(dag.nodes.iter().any(|node| matches!(&node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Project { cols, .. }) - if cols.iter().any(|item| matches!(&item.expr, ScalarExpr::FunctionCall { name, .. } if name == "map")) - ))); - // The AVG under the map projection is built from exact Sum and Count - // states, divided at read time. - let families: Vec<_> = dag - .nodes - .iter() - .filter_map(|node| match &node.payload { - PhysicalASAPOperatorPayload::ASAP(ASAPOp::SummaryAgg { - summary_type: family, - .. - }) => Some(family), - _ => None, - }) - .collect(); - assert_eq!( - families, - vec![ - &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - &FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count), - ] - ); -} diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs deleted file mode 100644 index e624feb75..000000000 --- a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs +++ /dev/null @@ -1,750 +0,0 @@ -//! Frontends → Pass 1 → composed candidate logical ASAP DAGs: every candidate of -//! a query is a valid DAG with the query's output columns (names, types and -//! nullability). - -use std::rc::Rc; -use std::sync::Arc; - -use asap_executor::{ - physical_planner::compile, - runtime::{Limits, RunContext, Scope}, - sources::InMemoryDataStore, - values::{Batch, Value}, -}; -use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::{ - compose_logical_candidate, generate_per_computation_candidates, -}; -use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{ - ExactKind, FieldDataType, SketchAlgorithm, SummaryInputExpr, SummaryUpdate, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode, QueryRoot, ScalarExpr}; -use asap_types::stages::HowToComputeAggregateMeasure; -use asap_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; -use futures::StreamExt; - -fn accuracy() -> AccuracyTarget { - AccuracyTarget::max_error(0.01).failure_probability(0.01) -} - -async fn sql(query: &str) -> QueryRoot { - let catalog = SqlCatalog::new().with_table( - "flows", - Schema::new(vec![ - Field::plain("src", DataType::Utf8, false), - Field::plain("dst", DataType::Utf8, false), - Field::plain("bytes", DataType::Float64, false), - ]), - ); - let root = lower_sql(query, &catalog) - .await - .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")); - QueryRoot::Operator(root) -} - -fn promql(query: &str) -> QueryRoot { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: accuracy(), - ..Default::default() - }, - predictability: Default::default(), - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")) - .remove(0) -} - -fn output_columns(root: &QueryRoot) -> Vec<(String, String, bool)> { - match root { - QueryRoot::Operator(node) => node - .schema - .fields - .iter() - .map(|field| { - ( - field.name.clone(), - format!("{:?}", field.dtype), - field.nullable, - ) - }) - .collect(), - QueryRoot::Scalar(_) => vec![], - } -} - -/// Compose every candidate of `root`; returns how many there were. -fn compose_all(label: &str, root: QueryRoot) -> usize { - let expected = output_columns(&root); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[root], &[None], &Default::default()) - .unwrap(), - ) - .unwrap(); - let radix: Vec<_> = inventory - .targets - .iter() - .map(|target| target.choice_count()) - .collect(); - let total: usize = radix.iter().product(); - let mut choice = vec![0; radix.len()]; - for _ in 0..total { - let composed = compose_logical_candidate(&inventory, &choice) - .unwrap_or_else(|e| panic!("{label} {choice:?}: {e}")); - let root = &composed[0]; - root.validate_structure() - .unwrap_or_else(|e| panic!("{label} {choice:?}: {e}")); - let columns = output_columns(root); - // A heap top-k answers the declared `[keys.., topk_k]` with one row - // per ranked item: `topk_k` its Float64 estimate (the value the - // exact top-k's physical plan answers too), then the item labels. - let heap_topk = expected - .last() - .is_some_and(|(name, _, _)| name.starts_with("topk_")) - && columns.get(expected.len() - 1).map(|c| c.1.as_str()) == Some("Plain(Float64)"); - if heap_topk { - let (ranked, keys) = expected.split_last().unwrap(); - assert_eq!(&columns[..keys.len()], keys, "{label} {choice:?}"); - assert_eq!(columns[keys.len()].0, ranked.0, "{label} {choice:?}"); - } else { - assert_eq!(columns, expected, "{label} {choice:?}"); - } - for (digit, radix) in choice.iter_mut().zip(&radix).rev() { - *digit += 1; - if *digit < *radix { - break; - } - *digit = 0; - } - } - total -} - -#[tokio::test] -async fn sql_candidates_compose() { - for query in [ - "SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'", - "SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10", - "SELECT src, avg(bytes), count(*), approx_distinct(dst) FROM flows GROUP BY src", - "SELECT count(*) FILTER (WHERE bytes > 10), approx_percentile_cont(bytes, 0.99) FROM flows", - "SELECT avg(bytes) FROM flows", - "SELECT sum(flows.bytes) FROM flows", - "SELECT src, approx_percentile_cont(bytes, 0.99) FILTER (WHERE bytes > 10) FROM flows GROUP BY src", - "SELECT approx_percentile_cont(bytes, 0.99) FILTER (WHERE bytes > 10) FROM flows", - ] { - assert!(compose_all(query, sql(query).await) >= 1, "{query}"); - } -} - -#[test] -fn promql_candidates_compose() { - for query in [ - "avg by (job) (m)", - "avg_over_time(m[5m])", - "quantile(0.99, m) - quantile(0.5, m)", - "scalar(quantile(0.99, m)) - scalar(quantile(0.5, m))", - "topk by (job) (10, sum_over_time(m[1m]))", - "topk(3, sum by (job) (rate(m[5m])))", - "count by (job) (m)", - "count_over_time(m[5m])", - "quantile_over_time(0.99, m[5m])", - ] { - assert!(compose_all(query, promql(query)) >= 1, "{query}"); - } -} - -/// The summary estimates of `root`'s candidates that vary one target's -/// choice, every other target taking its first. -fn composed_estimates(root: QueryRoot) -> Vec> { - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries(&[root], &[None], &Default::default()) - .unwrap(), - ) - .unwrap(); - let targets = inventory.targets.len(); - let choices: Vec<_> = inventory - .targets - .iter() - .enumerate() - .flat_map(|(target, computation)| { - (1..computation.choice_count()).map(move |index| { - let mut choice = vec![0; targets]; - choice[target] = index; - choice - }) - }) - .collect(); - choices - .into_iter() - .flat_map(|choice| { - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(node) = &composed[0] else { - panic!("operator root expected") - }; - OperatorNode::reachable(node) - }) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - .collect() -} - -fn update_of(estimate: &OperatorNode) -> (SketchAlgorithm, SummaryUpdate) { - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(kind), - input, - .. - }) = summary_input.asap() - else { - panic!("sketch state expected") - }; - (*kind, input.clone()) -} - -/// A heap sketch's bound is per item frequency, not Top-K membership, and -/// its weights are not proven non-negative: a Top-K estimate gets no -/// guarantee. A unit-count CMS estimate keeps its bound. -#[test] -fn topk_estimates_carry_no_frequency_bound() { - let topk = composed_estimates(promql("topk by (job) (10, sum_over_time(m[1m]))")); - let algorithms: Vec<_> = topk.iter().map(|node| update_of(node).0).collect(); - assert_eq!( - algorithms, - [ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap - ] - ); - assert!(topk.iter().all(|node| node.guarantee.is_none())); - - let count = composed_estimates(promql("count by (job) (m)")); - let cms = count - .iter() - .find(|node| update_of(node).0 == SketchAlgorithm::Cms) - .expect("CMS candidate"); - assert!(cms.guarantee.as_ref().is_some_and(|g| !g.is_exact())); -} - -/// Top-K over a grouped sum is a whole-expression heap: it absorbs the -/// sum and ranks its group keys, read from the rows the sum reads. -#[test] -fn topk_over_an_aggregate_ranks_its_labels() { - let estimates = composed_estimates(promql("topk(3, sum by (job) (rate(m[5m])))")); - assert_eq!(estimates.len(), 2); - for estimate in &estimates { - assert_eq!( - update_of(estimate).1.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) - ); - } -} - -/// A subquery inside a measure filter is substituted too: each choice for -/// the inner quantile gives its own DAG, and an approximate inner result clears -/// the outer measure's guarantee. -#[tokio::test] -async fn subquery_in_a_measure_filter_is_substituted() { - let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ - FROM flows GROUP BY src"; - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[sql(query).await], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 2); - let inner = inventory - .targets - .iter() - .position(|target| { - inventory - .aggregate(target) - .non_asap() - .and_then(|op| match op { - NonASAPOp::Aggregate { reduction, .. } => reduction.group_keys(), - _ => None, - }) - .is_some_and(|by| by.keys().is_empty()) - }) - .expect("inner quantile"); - let candidates = &inventory.targets[inner].ways_to_compute; - assert!(candidates.len() > 1); - let (mut seen, mut approximate_inner) = (Vec::new(), false); - for (index, candidate) in candidates.iter().enumerate() { - let mut choice = vec![0; 2]; - choice[inner] = index; - let composed = compose_logical_candidate(&inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - let shown = format!("{root:?}"); - assert!(!seen.contains(&shown), "{candidate:?} composes as another"); - seen.push(shown); - let finalize = OperatorNode::reachable(root) - .into_iter() - .find(|node| { - matches!( - node.asap(), - Some(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(child.asap(), Some(ASAPOp::SummaryAgg { filter: Some(_), .. })) - ) - }) - .expect("filtered outer count"); - let Some(ASAPOp::FinalizeExactAccumulator { child }) = finalize.asap() else { - unreachable!() - }; - let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); - // The state's children: its input rows, then the subquery. - let inputs_exact = child.children().into_iter().all(|node| exact(node)); - assert_eq!(exact(&finalize), inputs_exact, "{candidate:?}"); - approximate_inner |= !inputs_exact; - } - assert!(approximate_inner); -} - -/// Run the logical DAG `root` over `rows` of its one scanned table, each row -/// given by column name. Rows come back shown and sorted. -fn execute(root: &Rc, rows: &[Vec<(&str, Value)>]) -> Vec> { - let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); - let scan = dag - .nodes - .iter() - .find(|node| { - matches!( - &node.payload, - PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) - ) - }) - .expect("one scan"); - let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) = &scan.payload else { - unreachable!() - }; - let schema = Arc::new(scan.output_schema.clone()); - let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); - let rows = rows - .iter() - .map(|row| { - schema - .fields - .iter() - .map(|field| { - row.iter() - .find(|(name, _)| *name == field.name) - .map(|(_, value)| value.clone()) - .expect("value for every column") - }) - .collect() - }) - .collect(); - let store = InMemoryDataStore::default() - .with_raw_data( - source.clone(), - schema.clone(), - vec![Batch::try_new(schema.clone(), rows).unwrap()], - ) - .unwrap(); - let bound = plan.instantiate(Arc::new(store)).unwrap(); - let context = RunContext::new( - Scope::Query { - evaluation_time_ms: 0, - }, - Limits::default(), - ) - .unwrap(); - let show = |value: &Value| match value { - Value::Null => "NULL".to_string(), - Value::Bool(value) => value.to_string(), - Value::Float64(value) => value.to_string(), - Value::Utf8(value) => format!("{value:?}"), - other => format!("{other:?}"), - }; - let mut out: Vec> = futures::executor::block_on(async { - let mut stream = bound.execute(plan.roots(), context).unwrap().remove(0); - let mut rows = Vec::new(); - while let Some(batch) = stream.next().await { - rows.extend( - batch - .unwrap() - .to_rows() - .iter() - .map(|row| row.iter().map(show).collect()), - ); - } - rows - }); - out.sort(); - out -} - -/// SQL `avg` from its `sum` and `count` states, composed for the query. -async fn avg_as_sum_and_count( - catalog: &SqlCatalog, - query: &str, -) -> (Rc, Rc) { - let exact = lower_sql(query, catalog).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(exact.clone())], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let choice: Vec<_> = inventory - .targets - .iter() - .map(|target| { - target - .ways_to_compute - .iter() - .position(|c| { - *c == HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count, - ]) - }) - .expect("avg as sum and count") - }) - .collect(); - let composed = - compose_logical_candidate(&inventory, &choice).unwrap_or_else(|e| panic!("{query}: {e}")); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - assert!( - OperatorNode::reachable(root) - .iter() - .any(|node| matches!(node.non_asap(), Some(NonASAPOp::Join { .. }))), - "{query}: sum joined with count expected" - ); - assert_eq!( - output_columns(&QueryRoot::Operator(root.clone())), - output_columns(&QueryRoot::Operator(exact.clone())), - "{query}" - ); - (exact, root.clone()) -} - -fn avg_table() -> SqlCatalog { - SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("k", DataType::Utf8, true), - Field::plain("ok", DataType::Bool, false), - Field::plain("value", DataType::Float64, false), - ]), - ) -} - -/// SQL `avg` as `sum` and `count` states executes as the exact aggregate: -/// NULL and '' groups stay apart, a non-Utf8 key works, and an empty global -/// input averages to NULL. -#[tokio::test] -async fn sql_avg_from_sum_and_count_executes_as_avg() { - let catalog = avg_table(); - let row = |k: Option<&str>, ok: bool, value: f64| { - vec![ - ("k", k.map_or(Value::Null, |k| Value::Utf8(k.into()))), - ("ok", Value::Bool(ok)), - ("value", Value::Float64(value)), - ] - }; - let rows = vec![ - row(None, true, 2.), - row(None, true, 4.), - row(Some(""), false, 10.), - row(Some("a"), false, 200.), - row(Some("a"), true, 1.), - ]; - let rows_of = |rows: &[&[&str]]| -> Vec> { - rows.iter() - .map(|row| row.iter().map(|cell| cell.to_string()).collect()) - .collect() - }; - for (query, input, want) in [ - ( - "SELECT k, avg(value) FROM t GROUP BY k", - rows.clone(), - rows_of(&[&["\"\"", "10"], &["\"a\"", "100.5"], &["NULL", "3"]]), - ), - ( - "SELECT ok, avg(value) FROM t GROUP BY ok", - rows.clone(), - rows_of(&[&["false", "105"], &["true", "2.3333333333333335"]]), - ), - ( - "SELECT avg(value) FROM t", - rows.clone(), - rows_of(&[&["43.4"]]), - ), - ("SELECT avg(value) FROM t", vec![], rows_of(&[&["NULL"]])), - ] { - let (exact, root) = avg_as_sum_and_count(&catalog, query).await; - assert_eq!(execute(&exact, &input), want, "{query}"); - assert_eq!(execute(&root, &input), want, "{query}"); - } -} - -/// A filtered SQL `avg` is NULL where no row passes the filter: its quotient -/// is NULL when the count is 0. The runtime executes no filtered state yet, -/// so the shape is checked. -#[tokio::test] -async fn filtered_sql_avg_is_null_without_matching_rows() { - let catalog = avg_table(); - for query in [ - "SELECT k, avg(value) FILTER (WHERE value > 100) FROM t GROUP BY k", - "SELECT avg(value) FILTER (WHERE value > 100) FROM t", - ] { - let (_, root) = avg_as_sum_and_count(&catalog, query).await; - assert!(root.schema.fields.last().unwrap().nullable, "{query}"); - let null_when_no_count = OperatorNode::reachable(&root).iter().any(|node| { - let Some(NonASAPOp::Project { cols, child, .. }) = node.non_asap() else { - return false; - }; - let count = child.schema.fields.len() - 1; - matches!( - &cols.last().unwrap().expr, - ScalarExpr::Case { branches, .. } if matches!( - &branches[..], - [(ScalarExpr::Compare { left, op: CompareOpKind::Eq, right, .. }, - ScalarExpr::Literal(ScalarValue::Null))] - if **left == ScalarExpr::Column(count) - && **right == ScalarExpr::Literal(ScalarValue::Int64(0)) - ) - ) - }); - assert!(null_when_no_count, "{query}: {root:#?}"); - } -} - -/// PromQL `avg` divides its `sum` and `count` vectors with a `BinaryOp`. -#[test] -fn promql_avg_divides_sum_and_count_vectors() { - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[promql("avg by (job) (m)")], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - assert_eq!( - inventory.targets[0].ways_to_compute[0], - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum, ExactKind::Count]) - ); - let composed = compose_logical_candidate(&inventory, &[0]).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - let kinds: Vec<_> = OperatorNode::reachable(root) - .iter() - .map(|node| node.operator.kind_name()) - .collect(); - assert!( - kinds.contains(&"BinaryOp") && !kinds.contains(&"Join"), - "{kinds:?}" - ); -} - -/// `t(k Utf8 NULL, )` as a catalog. -fn table(columns: &[(&str, bool)]) -> SqlCatalog { - let mut fields = vec![Field::plain("k", DataType::Utf8, true)]; - fields.extend( - columns - .iter() - .map(|(name, nullable)| Field::plain(*name, DataType::Float64, *nullable)), - ); - SqlCatalog::new().with_table("t", Schema::new(fields)) -} - -/// Every candidate of `query` over `catalog`, composed, by candidate. -async fn every_candidate( - catalog: &SqlCatalog, - query: &str, -) -> Vec<( - HowToComputeAggregateMeasure, - Result, asap_logical_asap_optimization::PerComputationCandidateError>, -)> { - let root = lower_sql(query, catalog).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 1, "{query}"); - inventory.targets[0] - .ways_to_compute - .iter() - .enumerate() - .map(|(index, candidate)| { - ( - candidate.clone(), - compose_logical_candidate(&inventory, &[index]), - ) - }) - .collect() -} - -/// The update of every `count` state (exact or sketch) below `root`. -fn count_updates(root: &QueryRoot) -> Vec { - let QueryRoot::Operator(root) = root else { - panic!("operator root expected") - }; - OperatorNode::reachable(root) - .iter() - .filter_map(|node| match node.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: family, - input, - .. - }) if !matches!( - family, - FieldDataType::ExactAggregate(asap_types::ir::schema::ExactKind::Sum, _) - ) => - { - Some(input.clone()) - } - _ => None, - }) - .collect() -} - -/// Whether `update` reads only the column `name` of `t`. -fn reads_only(update: &SummaryUpdate, name: &str) -> bool { - let column = SummaryInputExpr::Column(ColumnRef::Qualified { - table: "t".into(), - name: name.into(), - }); - match &update.item { - Some(item) => *item == column && update.weight == SummaryInputExpr::Constant(1.0), - None => update.weight == column, - } -} - -/// `avg(c)`'s count counts `c`, not a nullable `value` column: the result is -/// the exact `avg`. -#[tokio::test] -async fn sql_avg_counts_its_own_input() { - let catalog = table(&[("value", true), ("c", false)]); - let query = "SELECT k, avg(c) FROM t GROUP BY k"; - let (exact, root) = avg_as_sum_and_count(&catalog, query).await; - let row = |k: &str, value: Option, c: f64| { - vec![ - ("k", Value::Utf8(k.into())), - ("value", value.map_or(Value::Null, Value::Float64)), - ("c", Value::Float64(c)), - ] - }; - let rows = vec![ - row("a", None, 1.), - row("a", Some(5.), 3.), - row("b", None, 4.), - ]; - let want = vec![ - vec!["\"a\"".to_string(), "2".to_string()], - vec!["\"b\"".to_string(), "4".to_string()], - ]; - assert_eq!(execute(&exact, &rows), want); - assert_eq!(execute(&root, &rows), want); -} - -/// SQL `avg` as `sum` and `count` keeps an explicit output alias. -#[tokio::test] -async fn sql_avg_keeps_its_explicit_output_name() { - let (_, root) = - avg_as_sum_and_count(&avg_table(), "SELECT avg(value) AS avg_latency FROM t").await; - assert_eq!(root.schema.fields.last().unwrap().name, "avg_latency"); -} - -/// SQL `avg` of a non-null Int64 column as `sum` and `count` keeps the exact plan's Float64 columns. -#[tokio::test] -async fn sql_avg_of_an_int64_column_keeps_the_exact_columns() { - let catalog = SqlCatalog::new().with_table( - "t", - Schema::new(vec![ - Field::plain("k", DataType::Utf8, true), - Field::plain("n", DataType::Int64, false), - ]), - ); - let query = "SELECT k, avg(n) FROM t GROUP BY k"; - let (exact, root) = avg_as_sum_and_count(&catalog, query).await; - assert_eq!( - root.schema.fields.last().unwrap().plain_dtype(), - Some(&DataType::Float64) - ); - let row = |k: &str, n: i64| vec![("k", Value::Utf8(k.into())), ("n", Value::Int64(n))]; - let rows = vec![row("a", 1), row("a", 2), row("b", 4)]; - let want = vec![ - vec!["\"a\"".to_string(), "1.5".to_string()], - vec!["\"b\"".to_string(), "4".to_string()], - ]; - assert_eq!(execute(&exact, &rows), want); -} - -/// `count(*)` with no column known to be non-null (one nullable numeric -/// column, or several and none named `value`) is unsupported for every -/// candidate, not a count of a nullable column or an invalid DAG. -#[tokio::test] -async fn sql_count_star_without_a_non_null_column_is_unsupported() { - for columns in [&[("v", true)][..], &[("a", true), ("b", true)][..]] { - let catalog = table(columns); - for (candidate, composed) in - every_candidate(&catalog, "SELECT k, count(*) FROM t GROUP BY k").await - { - match composed { - Err(asap_logical_asap_optimization::PerComputationCandidateError::Unsupported( - _, - )) => {} - other => panic!("{columns:?} {candidate:?}: {other:?}"), - } - } - } -} - -/// `count(v)` of a nullable `v` updates on `v` under its `v IS NOT NULL` -/// filter: it counts the non-null values. The runtime executes no filtered -/// state yet, so the shape is checked. -#[tokio::test] -async fn sql_count_of_a_nullable_column_counts_its_values() { - let catalog = table(&[("v", true), ("n", false)]); - for (candidate, composed) in - every_candidate(&catalog, "SELECT k, count(v) FROM t GROUP BY k").await - { - let composed = composed.unwrap_or_else(|e| panic!("{candidate:?}: {e}")); - composed[0].validate_structure().unwrap(); - let updates = count_updates(&composed[0]); - assert_eq!(updates.len(), 1, "{candidate:?}"); - assert!(reads_only(&updates[0], "v"), "{candidate:?}: {updates:?}"); - } -} diff --git a/crates/logical-asap-optimization/src/pass1.rs b/crates/logical-asap-optimization/src/pass1.rs index 6f56465ae..8aac868d0 100644 --- a/crates/logical-asap-optimization/src/pass1.rs +++ b/crates/logical-asap-optimization/src/pass1.rs @@ -1,14 +1,12 @@ -//! Stage 1, Pass 1: per-computation candidate generation and composition, -//! and the summary realizations it offers. Its one entry is +//! Stage 1, Pass 1: per-computation candidate generation and the summary +//! realizations it offers. Its one entry is //! [`pass1_memo`], whose output //! memo Pass 2 extends. #[cfg(any(test, feature = "test-support"))] -pub(crate) mod composition; /// Stage 1, Pass 1's entry: the initial candidates of the memo groups. pub(crate) mod logical_memo_candidates; -/// Stage 1, Pass 1's computation targets, their candidates and their -/// composition. No execution timing is assigned: that is a Stage 2 +/// Stage 1, Pass 1's computation targets and their candidates. No execution timing is assigned: that is a Stage 2 /// materialization decision. Its target list /// (`PerComputationCandidates`) is Pass 1's working form, public only to /// tests (`crate::test_support`, feature `test-support`). diff --git a/crates/logical-asap-optimization/src/pass1/composition.rs b/crates/logical-asap-optimization/src/pass1/composition.rs deleted file mode 100644 index 4e4bdfd2a..000000000 --- a/crates/logical-asap-optimization/src/pass1/composition.rs +++ /dev/null @@ -1,1054 +0,0 @@ -//! Composing one whole-workload candidate DAG from one per-computation -//! candidate per target (Algorithm 1's `SUBSTITUTE`): test support only. -//! Planning adds each way to compute to the memo instead (#1207 PR 3 -//! deletes this). -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::column_resolution::ResolveError; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::topk::ranked_column; -use asap_types::ir::schema::{ColumnId, Field}; -use asap_types::ir::schema::{DataType, Schema}; -use asap_types::ir::schema::{ - EntityIdentity, ExactKind, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, - SummaryInputExpr, SummaryUpdate, WeightDomain, -}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, OperatorResultKind, - Predicate, ProjectItem, QueryRoot, ScalarExpr, -}; - -use asap_accuracy::propagation::{raw_data_samples_of_metric, with_guarantees}; -use asap_accuracy::sizing::sketch_statistic; - -use asap_types::stages::{ComputationTarget, HavingPlacement, HowToComputeAggregateMeasure}; - -use super::per_computation_candidates::*; -/// `update` with the open requirement that its weights are samples of -/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap -/// update whose weights are not otherwise proven. -fn with_raw_data_samples_requirement( - mut update: SummaryUpdate, - metric: Option<&str>, -) -> SummaryUpdate { - if let Some(metric) = metric { - if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { - update.weight_domain = WeightDomain::RawDataSamplesOfMetric { - metric: metric.to_string(), - }; - } - } - update -} - -/// Build one whole-workload candidate (end of Pass 1): `choice[i]` picks -/// `inventory.targets[i]`'s candidate, or past its own candidates its fused -/// computation's ([`ComputationTarget::computation_for`]). An aggregate with -/// a chosen candidate other than `Exact` is replaced measure by measure: an -/// exact accumulator by `SummaryAgg` → `FinalizeExactAccumulator`, a summary -/// by `SummaryAgg` → `SummaryEstimate`, `avg` by its `sum` and `count` -/// accumulators divided, a fused computation by one state over the absorbed -/// target's input, and any other measure by its own single-measure -/// aggregate; several measures are joined back on the group keys. A summary -/// is sized here for its measure's accuracy requirement. A measure filter -/// becomes `SummaryAgg.filter`; -/// `HAVING` is applied as its target records. A sub-DAG shared between roots -/// stays shared. Every node carries the guarantee `asap_accuracy::propagation::composed_guarantee` -/// gives it. -pub fn compose_logical_candidate( - inventory: &PerComputationCandidates, - choice: &[usize], -) -> Result, PerComputationCandidateError> { - compose(inventory, &inventory.roots, choice) -} - -/// [`compose_logical_candidate`] of `roots` (some of `inventory.roots`), -/// each summary sized for its measure's whole budget. -fn compose( - inventory: &PerComputationCandidates, - roots: &[QueryRoot], - choice: &[usize], -) -> Result, PerComputationCandidateError> { - if choice.len() != inventory.targets.len() { - return Err(PerComputationCandidateError::InvalidChoice); - } - let mut chosen: Chosen = HashMap::new(); - for (target, &index) in inventory.targets.iter().zip(choice) { - let (candidate, absorbs) = target - .computation_for(index) - .ok_or(PerComputationCandidateError::InvalidChoice)?; - chosen - .entry(Rc::as_ptr(inventory.aggregate(target))) - .or_default() - .push(ChosenComputation { - target, - candidate, - fused: absorbs.is_some(), - }); - } - // Consumer `Filter`s whose key-only `HAVING` moves below a substituted - // aggregate: each is replaced by its input. - let pushed_below: HashSet<_> = chosen - .values() - .filter(|targets| substituted(targets)) - .flat_map(|targets| &targets[0].target.having_filters) - .filter(|entry| entry.placement == HavingPlacement::BelowAggregate) - .filter_map(|entry| entry.filter.as_ref().map(Rc::as_ptr)) - .collect(); - compose_roots(roots, &chosen, &pushed_below, &Sizing::default()) -} - -/// One measure of an aggregate: the aggregate and the measure's index. -type MeasureKey = (*const OperatorNode, usize); - -/// How [`compose_logical_candidate`] sizes summaries in one composition. -#[derive(Default)] -struct Sizing { - /// Every sketch summary state built, with the measure it sizes. - built: std::cell::RefCell, MeasureKey)>>, -} - -/// Every root of `roots` with the chosen candidates substituted, each -/// summary sized as `sizing` says, and every node with its guarantee. -fn compose_roots( - roots: &[QueryRoot], - chosen: &Chosen<'_>, - pushed_below: &HashSet<*const OperatorNode>, - sizing: &Sizing, -) -> Result, PerComputationCandidateError> { - let (mut cache, mut guaranteed) = (HashMap::new(), HashMap::new()); - let mut rewrite = |node: &Rc| { - let node = rewrite(node, chosen, pushed_below, sizing, &mut cache)?; - with_guarantees(&node, &mut guaranteed).map_err(PerComputationCandidateError::from) - }; - let roots = roots - .iter() - .map(|root| { - let root = match root { - QueryRoot::Operator(node) => QueryRoot::Operator(rewrite(node)?), - QueryRoot::Scalar(expr) => { - let mut rewritten = HashMap::new(); - for node in expr.operator_refs() { - rewritten.insert(Rc::as_ptr(node), rewrite(node)?); - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| rewritten[&Rc::as_ptr(node)].clone()), - ) - } - }; - // A substitution this module cannot express is an error, never an - // invalid DAG. - root.validate_structure()?; - Ok(root) - }) - .collect::, PerComputationCandidateError>>()?; - // The built states, as the roots hold them: with their guarantees. - for (state, _) in sizing.built.borrow_mut().iter_mut() { - if let Some(with_guarantee) = guaranteed.get(&Rc::as_ptr(state)) { - *state = with_guarantee.clone(); - } - } - Ok(roots) -} - -/// One target's chosen computation in a whole-workload choice. -struct ChosenComputation<'a> { - /// The target whose measure it computes. - target: &'a ComputationTarget, - /// How it is computed. - candidate: &'a HowToComputeAggregateMeasure, - /// Whether the candidate is the target's fused computation's, which - /// absorbs the targets beneath; `false` for one of its own candidates. - fused: bool, -} - -/// Each aggregate's targets with their chosen computations. -type Chosen<'a> = HashMap<*const OperatorNode, Vec>>; -type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; - -/// Whether any measure of the aggregate is substituted by something other than -/// its plain exact computation. -fn substituted(targets: &[ChosenComputation<'_>]) -> bool { - targets - .iter() - .any(|chosen| *chosen.candidate != HowToComputeAggregateMeasure::Exact) -} - -fn rewrite( - node: &Rc, - chosen: &Chosen<'_>, - pushed_below: &HashSet<*const OperatorNode>, - sizing: &Sizing, - cache: &mut NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - rewrite(child, chosen, pushed_below, sizing, cache)?; - } - let changed = node - .children() - .iter() - .any(|child| !Rc::ptr_eq(child, &cache[&Rc::as_ptr(child)])); - let rebuilt = match (chosen.get(&Rc::as_ptr(node)), node.non_asap()) { - (Some(targets), _) if substituted(targets) => substitute(node, targets, sizing, cache)?, - (_, Some(NonASAPOp::Filter { child, .. })) if pushed_below.contains(&Rc::as_ptr(node)) => { - cache[&Rc::as_ptr(child)].clone() - } - _ if changed => Rc::new(node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?), - _ => node.clone(), - }; - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) -} - -/// Replace an aggregate whose measures are `targets`' (or exact, for a -/// measure that is no target) by its substitution, with the same output -/// columns. -fn substitute( - target: &Rc, - targets: &[ChosenComputation<'_>], - sizing: &Sizing, - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - child, - reduction, - measures, - filters, - .. - }) = target.non_asap() - else { - return Err(PerComputationCandidateError::Unsupported( - "target is not an aggregate", - )); - }; - // A key-only `HAVING` is a row predicate over the input: rows of a - // dropped group never reach the state. - let having = &targets[0].target.having_filters; - // Operator refs in a predicate (a subquery) are this aggregate's - // children, already rewritten. - let rewritten = |pred: &Predicate| { - Predicate( - pred.0 - .map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), - ) - }; - // The metric whose raw data samples, or plain sums of them, are every - // value the aggregate reads: a heap-sketch weight read from them carries - // the open requirement `WeightDomain::RawDataSamplesOfMetric`, which - // plan selection confirms. - let raw_data_samples = raw_data_samples_of_metric(child); - let mut input = cache[&Rc::as_ptr(child)].clone(); - for entry in having { - if entry.placement == HavingPlacement::BelowAggregate { - let keys = reduction.group_keys().map_or(&[][..], |by| by.keys()); - let pred = Predicate(entry.predicate.0.map_columns(&mut |column| keys[column])); - input = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred, - child: input, - }))?; - } - } - let parts = measures - .iter() - .enumerate() - .map(|(measure, intent)| { - // `fused`: the candidate is the target's fused computation's, - // which absorbs the target beneath. - let (candidate, raw_data_samples_of, fused) = targets - .iter() - .find(|chosen| chosen.target.measure_index == measure) - .map_or( - (&HowToComputeAggregateMeasure::Exact, None, false), - |chosen| (chosen.candidate, raw_data_samples.as_deref(), chosen.fused), - ); - let filter = filters.get(measure).and_then(Option::as_ref).map(rewritten); - let key = (Rc::as_ptr(target), measure); - if fused { - return match candidate { - HowToComputeAggregateMeasure::ExactAccumulator(_) => nested_sum(target, cache), - HowToComputeAggregateMeasure::Summary(algorithm) => whole_expression( - target, - intent, - algorithm, - raw_data_samples_of, - (sizing, key), - cache, - ), - HowToComputeAggregateMeasure::Exact => { - Err(PerComputationCandidateError::Unsupported( - "plain exact execution of a fused computation", - )) - } - }; - } - substitute_measure( - target, - intent, - candidate, - filter, - &input, - raw_data_samples_of, - (sizing, key), - ) - }) - .collect::, _>>()?; - let mut output = match <[_; 1]>::try_from(parts) { - Ok([part]) => with_names_of(target, part)?, - Err(parts) => join_measures(target, parts)?, - }; - for entry in having { - if entry.placement == HavingPlacement::AboveOutput && entry.filter.is_none() { - output = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { - pred: rewritten(&entry.predicate), - child: output, - }))?; - } - } - Ok(output) -} - -/// `output` with `target`'s column names: an estimate or accumulator names -/// its value after the operation (`quantile`, `sum`), not the aggregate's -/// output column. -pub(crate) fn with_names_of( - target: &OperatorNode, - output: Rc, -) -> Result, PerComputationCandidateError> { - let names = |node: &OperatorNode| -> Vec { - node.schema - .fields - .iter() - .map(|field| field.name.clone()) - .collect() - }; - // A top-k's ranked rows may carry their item columns after the - // aggregate's ([`ranked_as_topk`]). - if names(&output).starts_with(&names(target)) { - return Ok(with_aggregate_qualifiers(target, output)); - } - if output.schema.fields.len() != target.schema.fields.len() { - return Err(PerComputationCandidateError::Unsupported( - "substituted output with other columns than the aggregate", - )); - } - let cols = names(target) - .into_iter() - .enumerate() - .map(|(column, name)| ProjectItem { - alias: Some(name), - expr: ScalarExpr::Column(column), - }) - .collect(); - Ok(with_aggregate_qualifiers( - target, - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: output, - }))?, - )) -} - -/// An equivalent aggregate replacement must retain SQL field identity. These -/// explicit qualifiers survive child rebinding; types and keys remain derived -/// from executable operators rather than copied from the requested output. -fn with_aggregate_qualifiers(target: &OperatorNode, output: Rc) -> Rc { - if output - .schema - .fields - .iter() - .zip(&target.schema.fields) - .all(|(field, expected)| field.table == expected.table) - { - return output; - } - let mut output = output.as_ref().clone(); - for (field, expected) in output.schema.fields.iter_mut().zip(&target.schema.fields) { - field.table = expected.table.clone(); - } - Rc::new(output) -} - -/// A whole-expression top-k for `target` ([`whole_expression_input`]): one -/// heap summary of `algorithm` over the absorbed aggregate's rewritten -/// input, partitioned as `target`. -fn whole_expression( - target: &OperatorNode, - intent: &AggIntent, - algorithm: &SketchAlgorithm, - raw_data_samples_of: Option<&str>, - (sizing, key): (&Sizing, MeasureKey), - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let (input, update, partitions) = whole_expression_input(target).ok_or( - PerComputationCandidateError::Unsupported("whole-expression top-k input"), - )?; - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: cache[&Rc::as_ptr(&input)].clone(), - input: with_raw_data_samples_requirement(update, raw_data_samples_of), - summary_type: FieldDataType::Sketch(*algorithm), - reduction: partitions, - filter: None, - }))?; - sizing.built.borrow_mut().push((state.clone(), key)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: state, - query: sketch_statistic(intent).ok_or(PerComputationCandidateError::Unsupported( - "sketch evaluation", - ))?, - }))?; - ranked_as_topk(target, estimate) -} - -/// A heap's top-k `estimate` (`[partitions.., items.., topk]`, one row per -/// ranked item) in `target`'s columns: its `[keys.., topk_k]`, with -/// `topk_k` the estimated Float64 weight, then the item columns. This is -/// the exact top-k's physical shape (its ranked rows' value, then the -/// columns naming each ranked item: PromQL labels, SQL group keys), so -/// consumers of `[keys.., topk_k]` find those columns where they were. -fn ranked_as_topk( - target: &OperatorNode, - estimate: Rc, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { reduction, .. }) = target.non_asap() else { - return Err(PerComputationCandidateError::Unsupported( - "top-k target is not an aggregate", - )); - }; - let ranked_index = reduction.group_keys().map_or(0, |keys| keys.len()); - let fields = &estimate.schema.fields; - let estimated = fields.len() - 1; - let cols = target - .schema - .fields - .iter() - .enumerate() - .map(|(index, field)| { - let source = if index == ranked_index { - estimated - } else { - fields[..estimated] - .iter() - .position(|input| input.name == field.name) - .ok_or(PerComputationCandidateError::Unsupported( - "top-k estimate without a ranked-row identity column", - ))? - }; - Ok(ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(source), - }) - }) - .collect::, PerComputationCandidateError>>()?; - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: estimate, - }, - ))?) -} - -/// A nested-sum merge for `target` ([`nested_sum_input`]): one exact `Sum` -/// accumulator over the absorbed sum's rewritten input, grouped as -/// `target`, finalized. -fn nested_sum( - target: &OperatorNode, - cache: &NodeToRewrittenNodeCache, -) -> Result, PerComputationCandidateError> { - let (rows, intent, reduction) = nested_sum_input(target).ok_or( - PerComputationCandidateError::Unsupported("nested-sum merge input"), - )?; - let state = exact_accumulator( - ExactKind::Sum, - &intent, - &cache[&Rc::as_ptr(&rows)], - reduction, - )?; - Ok(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child: state }, - ))?) -} - -/// An unfiltered exact accumulator of `kind` for `intent` over `input`, -/// grouped by `reduction`. -pub(crate) fn exact_accumulator( - kind: ExactKind, - intent: &AggIntent, - input: &Rc, - reduction: Reduction, -) -> Result, PerComputationCandidateError> { - let family = FieldDataType::exact(kind); - let read = measure_input(intent, input, None)?; - Ok(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::SummaryAgg { - input: summary_update(intent, &family, &reduction, &input.schema, read)?, - child: Rc::clone(input), - summary_type: family, - reduction, - filter: None, - }, - ))?) -} - -/// One measure of `target` over `input`: its group keys, then its value. -fn substitute_measure( - target: &OperatorNode, - intent: &AggIntent, - candidate: &HowToComputeAggregateMeasure, - filter: Option, - input: &Rc, - raw_data_samples_of: Option<&str>, - (sizing, key): (&Sizing, MeasureKey), -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - reduction, - output_names, - .. - }) = target.non_asap() - else { - unreachable!("substitute checked the aggregate") - }; - let state = |family: FieldDataType, intent: &AggIntent, read: SummaryInputExpr| { - Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::SummaryAgg { - child: input.clone(), - input: with_raw_data_samples_requirement( - summary_update(intent, &family, reduction, &input.schema, read)?, - raw_data_samples_of, - ), - summary_type: family, - reduction: reduction.clone(), - filter: filter.clone(), - }, - ))?) - }; - let finalized = |kind: &ExactKind, intent: &AggIntent, read| { - let state = state(FieldDataType::exact(*kind), intent, read)?; - Ok::<_, PerComputationCandidateError>(OperatorNode::new_shared(Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child: state }, - ))?) - }; - Ok(match candidate { - HowToComputeAggregateMeasure::Exact => { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: input.clone(), - reduction: reduction.clone(), - measures: vec![intent.clone()], - // One name per measure, as the target names this one (an - // empty name is the intent's synthetic one). - output_names: vec![output_names.get(key.1).cloned().unwrap_or_default()], - filters: vec![filter.clone()], - having: None, - }))? - } - HowToComputeAggregateMeasure::ExactAccumulator(kinds) if kinds.len() == 1 => { - let read = measure_input(intent, input, filter.as_ref())?; - finalized(&kinds[0], intent, read)? - } - HowToComputeAggregateMeasure::Summary(algorithm) => { - let family = FieldDataType::Sketch(*algorithm); - let summary_input = state( - family, - intent, - measure_input(intent, input, filter.as_ref())?, - )?; - sizing.built.borrow_mut().push((summary_input.clone(), key)); - let estimate = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: sketch_statistic(intent).ok_or( - PerComputationCandidateError::Unsupported("sketch evaluation"), - )?, - }))?; - match intent { - AggIntent::TopK { .. } => ranked_as_topk(target, estimate)?, - _ => estimate, - } - } - HowToComputeAggregateMeasure::ExactAccumulator(kinds) => { - let (AggIntent::Avg { col }, [ExactKind::Sum, ExactKind::Count]) = - (intent, kinds.as_slice()) - else { - return Err(PerComputationCandidateError::Unsupported( - "several exact states other than avg's sum then count", - )); - }; - let sum_intent = AggIntent::Sum { col: *col }; - // The count counts the averaged column, as `avg` does: it is - // non-null (a nullable input prunes this candidate), so it counts - // the rows the sum reads. - let read = measure_input(&sum_intent, input, filter.as_ref())?; - let sum = finalized(&ExactKind::Sum, &sum_intent, read.clone())?; - let count = finalized(&ExactKind::Count, &AggIntent::Count, read)?; - if input.result_kind == OperatorResultKind::Relation { - return sql_avg(reduction, filter.is_some(), sum, count); - } - // `avg` is Float64 whatever the summed type, as `Avg`'s output is, - // and a PromQL binary operator reads Float64 sample values. A - // grouped sum or count ends with its value, cast when not Float64 - // (a count is Int64). A per-series sum keeps the input's labels, - // so its value is the sample-value column; a per-series count is - // Float64. - let (sum, count) = match reduction { - Reduction::Reduce(by) => ( - last_as_float64(by.len(), sum)?, - last_as_float64(by.len(), count)?, - ), - Reduction::PerEntity => { - let value = resolve_column_ref(&ColumnRef::SampleValue, &sum.schema) - .ok() - .and_then(|column| sum.schema.fields.get(column)); - if value.map(|field| &field.dtype) - != Some(&FieldDataType::Plain(DataType::Float64)) - { - return Err(PerComputationCandidateError::Unsupported( - "non-Float64 per-series avg", - )); - } - (sum, count) - } - }; - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - // A per-series window may hold no finite sum to divide. - checked_finite_division: matches!(reduction, Reduction::PerEntity), - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), - vector_match: None, - }, - return_bool: false, - lhs: sum, - rhs: count, - }))? - } - }) -} - -/// `node`, a grouped aggregate's `keys` group keys and then its value, with -/// the value cast to Float64 unless it already is one. -fn last_as_float64( - keys: usize, - node: Rc, -) -> Result, PerComputationCandidateError> { - if node.schema.fields.last().map(|field| &field.dtype) - == Some(&FieldDataType::Plain(DataType::Float64)) - { - return Ok(node); - } - let mut cols = key_items(keys); - cols.push(ProjectItem { - alias: None, - expr: ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(keys)), - to: DataType::Float64, - try_cast: false, - }, - }); - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: node, - }, - ))?) -} - -/// SQL `avg` from its finalized `sum` and `count`: joined on the group keys -/// (NULL keys match), then `sum / count` as Float64. A group with no row -/// passing the measure filter, or an empty global input, counts 0 and its -/// `avg` is NULL. -fn sql_avg( - reduction: &Reduction, - filtered: bool, - sum: Rc, - count: Rc, -) -> Result, PerComputationCandidateError> { - let Reduction::Reduce(by) = reduction else { - return Err(PerComputationCandidateError::Unsupported( - "SQL avg without `by` keys", - )); - }; - if by.is_without() { - return Err(PerComputationCandidateError::Unsupported( - "SQL avg without `by` keys", - )); - } - let keys = by.len(); - let float = |column| { - Box::new(ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(column)), - to: DataType::Float64, - try_cast: false, - }) - }; - let count_column = keys + 1 + keys; - let mut avg = ScalarExpr::Arithmetic { - op: ArithmeticOpKind::Div, - left: float(keys), - right: float(count_column), - semantics: ExprSemantics::Sql, - }; - if filtered || keys == 0 { - avg = ScalarExpr::Case { - operand: None, - branches: vec![( - ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(count_column)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), - semantics: ExprSemantics::Sql, - }, - ScalarExpr::Literal(ScalarValue::Null), - )], - else_expr: Some(Box::new(avg)), - }; - } - let mut cols = key_items(keys); - cols.push(ProjectItem { - alias: None, - expr: avg, - }); - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: join_on_keys(keys, sum, count)?, - }, - ))?) -} - -/// `left` joined with `right` on their leading `keys` columns; NULL keys -/// match each other, as grouping treats them. -fn join_on_keys( - keys: usize, - left: Rc, - right: Rc, -) -> Result, PerComputationCandidateError> { - let width = left.schema.fields.len(); - let column = |id| Box::new(ScalarExpr::Column(id)); - let matches = (0..keys) - .map(|key| { - let equal = ScalarExpr::Compare { - left: column(key), - op: CompareOpKind::Eq, - right: column(width + key), - semantics: ExprSemantics::Sql, - }; - if right.schema.fields[key].nullable { - ScalarExpr::BoolOr(vec![ - equal, - ScalarExpr::BoolAnd(vec![ - ScalarExpr::IsNull(column(key)), - ScalarExpr::IsNull(column(width + key)), - ]), - ]) - } else { - equal - } - }) - .collect::>(); - let (kind, pred) = match matches.len() { - 0 => ( - JoinKind::Cross, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ), - _ => (JoinKind::Inner, ScalarExpr::BoolAnd(matches)), - }; - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Join { - kind, - pred: Predicate(pred), - left, - right, - }, - ))?) -} - -/// Join single-measure `parts` (each: group keys, then one value) on their -/// group keys and project `target`'s output columns. Every part has the same -/// groups, so an inner join keeps them all; NULL keys match each other, as -/// grouping treats them. -fn join_measures( - target: &OperatorNode, - parts: Vec>, -) -> Result, PerComputationCandidateError> { - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) = target.non_asap() - else { - return Err(PerComputationCandidateError::Unsupported( - "multi-measure aggregate without `by` keys", - )); - }; - if by.is_without() { - return Err(PerComputationCandidateError::Unsupported( - "multi-measure aggregate without `by` keys", - )); - } - let keys = by.len(); - let width = keys + 1; - let mut parts = parts.into_iter(); - let mut joined = parts.next().expect("an aggregate has measures"); - for part in parts { - joined = join_on_keys(keys, joined, part)?; - } - let mut cols = key_items(keys); - cols.extend( - (0..target.schema.fields.len() - keys).map(|measure| ProjectItem { - alias: None, - expr: ScalarExpr::Column(measure * width + keys), - }), - ); - for (item, field) in cols.iter_mut().zip(&target.schema.fields) { - item.alias = Some(field.name.clone()); - } - Ok(OperatorNode::new_shared(Operator::NonASAP( - NonASAPOp::Project { - cols, - qualifier: None, - child: joined, - }, - ))?) -} - -fn key_items(keys: usize) -> Vec { - (0..keys) - .map(|key| ProjectItem { - alias: None, - expr: ScalarExpr::Column(key), - }) - .collect() -} - -/// What a measure's state reads from each row of `input`, resolvable there. -/// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count -/// included, reads its input column (the sample value when it names none). -fn measure_input( - intent: &AggIntent, - input: &Rc, - filter: Option<&Predicate>, -) -> Result { - let schema = &input.schema; - let read = match intent { - AggIntent::Count if input.result_kind == OperatorResultKind::Relation => { - SummaryInputExpr::Column(counted_column(schema, filter)?) - } - // A top-k ranks its child's aggregate output, not a column of its own. - AggIntent::TopK { .. } => SummaryInputExpr::Column( - ranked_value(input) - .map_err(|_| { - PerComputationCandidateError::Unsupported("Top-K has no ranked column") - })? - .1, - ), - _ => summarised_input(intent, schema).ok_or(PerComputationCandidateError::Unsupported( - "input column outside child schema", - ))?, - }; - if let SummaryInputExpr::Column(column) = &read { - resolve_column_ref(column, schema).map_err(|_| { - PerComputationCandidateError::Unsupported("summary input column does not resolve") - })?; - } - Ok(read) -} - -/// The column a SQL `count` state updates on. The runtime skips NULL updates, -/// so it is a column non-null on every counted row: one the measure filter -/// proves non-null (`count(col)` lowers to the filter `col IS NOT NULL`), else -/// a non-nullable one, Float64 first (the runtime's numeric update). -fn counted_column( - schema: &Schema, - filter: Option<&Predicate>, -) -> Result { - fn not_null(expr: &ScalarExpr) -> Vec { - match expr { - ScalarExpr::IsNotNull(inner) => match **inner { - ScalarExpr::Column(column) => vec![column], - _ => vec![], - }, - ScalarExpr::BoolAnd(conjuncts) => conjuncts.iter().flat_map(not_null).collect(), - _ => vec![], - } - } - let non_nullable = |float: bool| { - schema - .fields - .iter() - .enumerate() - .filter_map(move |(id, field)| { - (!field.nullable - && (!float || field.dtype == FieldDataType::Plain(DataType::Float64))) - .then_some(id) - }) - }; - let reference = |id: usize| { - schema - .fields - .get(id) - .map(asap_types::ir::schema::Field::column_ref) - }; - filter - .map_or_else(Vec::new, |filter| not_null(&filter.0)) - .into_iter() - .chain(non_nullable(true)) - .chain(non_nullable(false)) - .find_map(|id| { - reference(id).filter(|column| resolve_column_ref(column, schema).ok() == Some(id)) - }) - .ok_or(PerComputationCandidateError::Unsupported( - "SQL count over rows with no column known to be non-null", - )) -} - -/// What each input row contributes, given what the measure reads -/// ([`measure_input`]), following the legacy realization rules: heap sketches -/// rank series identities, frequency sketches count values, and every other -/// family reads the measure's input. -fn summary_update( - intent: &AggIntent, - family: &FieldDataType, - reduction: &Reduction, - child: &Schema, - weight: SummaryInputExpr, -) -> Result { - let algorithm = match family { - FieldDataType::Sketch(kind) => Some(kind), - _ => None, - }; - Ok(match (intent, algorithm) { - (AggIntent::TopK { .. }, Some(_)) => { - if reduction.group_keys().is_some_and(|keys| keys.is_without()) { - return Err(PerComputationCandidateError::Unsupported( - "Top-K partitions given by `without`", - )); - } - let groups: Vec = reduction - .group_keys() - .into_iter() - .flat_map(|keys| keys.iter().copied()) - .collect(); - let item = if child.has_promql_series_identity() { - asap_types::ir::schema::topk::series_identity_item(child, &groups) - } else if child.closed { - // A closed child (an aggregate's output, say) lists every - // column: the item is its columns outside the partition and - // the ranked value, as the legacy current-series and counter - // rules rank them. - let SummaryInputExpr::Column(ranked) = &weight else { - return Err(PerComputationCandidateError::Unsupported( - "Top-K ranked value is not a column", - )); - }; - let value = resolve_column_ref(ranked, child).map_err(|_| { - PerComputationCandidateError::Unsupported("Top-K ranked value does not resolve") - })?; - let items: Vec<_> = child - .fields - .iter() - .enumerate() - .filter(|(index, _)| *index != value && !groups.contains(index)) - .map(|(_, field)| SummaryInputExpr::Column(field.column_ref())) - .collect(); - if items.is_empty() { - return Err(PerComputationCandidateError::Unsupported( - "Top-K item has no label outside its partition", - )); - } - SummaryInputExpr::Tuple(items) - } else { - let excluding = groups - .iter() - .filter_map(|&index| child.fields.get(index)) - .map(asap_types::ir::schema::Field::column_ref) - .collect(); - SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { excluding }) - }; - SummaryUpdate { - item: Some(item), - // The ranked value ([`measure_input`]). - weight, - // Not proven non-negative; selection decides whether CMS is admissible. - weight_domain: WeightDomain::UnknownOrSigned, - } - } - (_, Some(SketchAlgorithm::UnivMon)) | (AggIntent::Count, Some(SketchAlgorithm::Cms)) => { - SummaryUpdate { - item: Some(weight), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - } - } - _ => SummaryUpdate { - item: None, - weight, - weight_domain: WeightDomain::UnknownOrSigned, - }, - }) -} - -/// The column fed into a *single-column* summary: the intent's leading -/// positional input resolved to a name against the child schema, or the PromQL -/// sample value when it reads none. Callers are responsible for only reaching -/// here with a one-column intent — [`summarised_input`] is the general form. -pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { - match intent - .input_cols() - .first() - .and_then(|id| child_schema.fields.get(*id)) - { - Some(c) => c.column_ref(), - None => ColumnRef::SampleValue, - } -} - -/// [`ranked_column`] as a reference: the sample value when it is that -/// column, else the column by name. -pub(crate) fn ranked_value( - child: &Rc, -) -> Result<(ColumnId, ColumnRef), ResolveError> { - let ranked = ranked_column(child)?; - let column = match resolve_column_ref(&ColumnRef::SampleValue, &child.schema) { - Ok(value) if value == ranked => ColumnRef::SampleValue, - _ => child.schema.fields[ranked].column_ref(), - }; - Ok((ranked, column)) -} - -/// What the summary consumes per input row. An intent that reads one column (or -/// none) feeds that column; `COUNT(DISTINCT a, b)` feeds the whole tuple as one -/// item, so the distinct-count sketch hashes `(a, b)` rather than `a` — the -/// difference between tuple cardinality and single-column cardinality. -/// -/// A tuple leg outside the child schema is an error rather than -/// [`summarised_column`]'s sample-value fallback: a leg has no sample-value -/// reading, and silently dropping one would under-count: `None` then. -pub(crate) fn summarised_input( - intent: &AggIntent, - child_schema: &Schema, -) -> Option { - let cols = intent.input_cols(); - if cols.len() < 2 { - return Some(SummaryInputExpr::Column(summarised_column( - intent, - child_schema, - ))); - } - let legs = cols - .iter() - .map(|id| child_schema.fields.get(*id).map(Field::column_ref)) - .collect::>>()?; - Some(SummaryInputExpr::Tuple( - legs.into_iter().map(SummaryInputExpr::Column).collect(), - )) -} diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 0a8af72f6..fe145026d 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -39,64 +39,6 @@ //! The input must be a logical DAG: ASAP operators are rejected. A logical //! DAG has no physical properties (#1200). No cost or accuracy model is //! used; ranking and discarding valid candidates belong to Stage 3. -//! -//! # Whole-workload candidates (logical ASAP-aware optimization, end of Pass 1) -//! -//! [`compose_logical_candidate`] is Algorithm 1's -//! `SUBSTITUTE(d, s_1 ↦ c_1, …)`: it picks one per-computation candidate per -//! target (`choice[i]` indexes `targets[i]`'s candidates, then its fused -//! computation's, [`ComputationTarget::computation_for`]) and builds one whole-workload -//! candidate DAG. An aggregate whose chosen candidates are all `Exact` is -//! kept. Otherwise each of its measures is substituted on its own over the -//! aggregate's input: -//! -//! - `ExactAccumulator` with one state: `SummaryAgg` with the exact family → -//! `FinalizeExactAccumulator`. -//! - `ExactAccumulator([Sum, Count])` of an `avg`: the finalized `sum` and -//! `count` accumulators divided: a PromQL `BinaryOp`, or for SQL rows a -//! join on the group keys (NULL keys match) and a SQL division, NULL where -//! the count is 0. -//! - `Summary`: `SummaryAgg` → `SummaryEstimate`. -//! - `Exact` (or a measure that is no target): a single-measure `Aggregate`. -//! -//! A fused computation's candidate reads the absorbed targets' input instead: -//! the whole-expression top-k is one heap `SummaryAgg` keyed by the ranked -//! item → `SummaryEstimate` (over a grouped sum of per-series sums or counts, -//! it reads the per-series target's input, the raw data samples); the nested sum is `SummaryAgg` with the exact -//! `Sum` family grouped by the outer keys → `FinalizeExactAccumulator`. -//! -//! The substituted output keeps the aggregate's column names. The measures -//! of a multi-measure aggregate are joined back on the group keys (NULL keys -//! match, as grouping treats them). A measure filter becomes -//! `SummaryAgg.filter`, so a group with no matching row stays. A -//! `BelowAggregate` `HAVING` becomes a `Filter` below the state, with its key -//! columns renumbered to the input's, and the consumer `Filter` it came from -//! is dropped; it is not a `SummaryAgg.filter`, which would keep the dropped -//! groups. An `AboveOutput` `HAVING` stays a `Filter` above the output. -//! -//! Summaries are sized here, each for its own measure's accuracy requirement. -//! When a query's value composes several approximate inputs (every sketch -//! summary below its root, when its guarantee is not exact), each is sized -//! for an equal share of that requirement: `ε/n` (a relative error -//! `1 − (1 + ε)^{−1/n}`) and `δ/n` for `n` inputs (#669b). Only the inputs -//! approximate in this candidate count. This composition sizes for ε and δ -//! per output value; attached to the memo groups, each candidate's -//! approximate inputs are sized for the values its consumers' targets cover -//! (`δ / (n · V)`, -//! [`pass1_memo`](crate::pass1::logical_memo_candidates::pass1_memo)), -//! their final sizes. Pass 2 inherits them or takes the strictest shared -//! consumer's, and Stage 3 only checks the composed accuracy. -//! -//! Every node of the candidate gets a theoretical guarantee, composed with -//! the built-in accuracy model through operators (#669): exact over exact -//! inputs; a `SummaryEstimate` the sized family's bound where that bound -//! answers the estimate (not for Top-K, where a heap sketch bounds item -//! frequencies, not membership; not for a Count-Min family over weights not -//! proven non-negative); aggregation and arithmetic over approximate values -//! by their composition rules, with statistics left symbolic for plan -//! selection to bind; and `None` where no rule applies (comparisons and -//! value predicates, rank errors). Plan selection's accuracy model decides -//! a candidate without one. use std::collections::{HashMap, HashSet}; use std::rc::Rc; @@ -871,54 +813,22 @@ pub(super) fn whole_expression_input( #[cfg(test)] mod tests { - /// Aggregate replacement qualifiers survive both renaming and later input rebinding. - #[test] - fn aggregate_replacement_preserves_sql_field_identity() { - let original = - crate::test_support::lower_promql("sum by (job) (m)", AccuracyTarget::exact()); - let mut target = original.as_ref().clone(); - target.schema.fields[0].table = Some("metrics".into()); - let projected = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: target - .schema - .fields - .iter() - .enumerate() - .map(|(column, field)| ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(column), - }) - .collect(), - qualifier: None, - child: original, - })) - .unwrap(); - assert_eq!(projected.schema.fields[0].table, None); - let replaced = with_names_of(&target, projected).unwrap(); - assert_eq!(replaced.schema, target.schema); - let rebound = replaced.with_new_children(Rc::clone).unwrap(); - assert_eq!(rebound.schema, target.schema); - } use super::*; - use crate::pass1::composition::*; use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind}; use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::schema::{EntityIdentity, NonNegativeWeightProof, WeightDomain}; - use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, Operator, ProjectItem, ScalarExpr, - }; + use asap_types::ir::schema::{NonNegativeWeightProof, WeightDomain}; + use asap_types::ir::{BinaryOperator, ExprSemantics, Operator, ProjectItem, ScalarExpr}; use asap_accuracy::propagation::raw_data_samples_of_metric; - use asap_accuracy::composition::{StreamNorm, PRODUCT_OPERAND_DOMAINS}; - use asap_accuracy::propagation::operand_guarantee; + use asap_accuracy::composition::PRODUCT_OPERAND_DOMAINS; + use asap_accuracy::test_support::{ - composed_guarantee, estimate_has_bound, estimates_raw_data_samples_of, fold_stats, - raw_data_samples_sign_statistic, scalar_guarantee, sign_statistic, - source_distinct_guarantee, stream_norm, QUOTIENT_OPERAND_DOMAINS, + composed_guarantee, estimate_has_bound, scalar_guarantee, source_distinct_guarantee, + QUOTIENT_OPERAND_DOMAINS, }; use asap_types::ir::properties::guarantee::{AccuracyMetric, BoundExpr, ProbabilityExpr}; @@ -1072,51 +982,6 @@ mod tests { } } - /// Count-Min and Count Sketch errors are additive: through - /// arithmetic they compose as absolute errors, each normalized bound - /// times the (symbolic) stream norm of its summary's source, here - /// over each source's raw value column, for plan selection to bind. - #[test] - fn normalized_additive_errors_compose_as_absolute_errors() { - let cms = with_bound(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01, 0.001); - let count_sketch = with_bound(AccuracyMetric::ABSOLUTE_STREAM_L2, 0.02, 0.002); - // Summaries of each source's raw value column. - let raw = |query: &str, guarantee: AccuracyGuarantee| { - ScalarExpr::PromqlScalarFromVector(crate::test_support::estimate_over( - crate::test_support::weighted( - crate::test_support::cms_summary(query), - SummaryInputExpr::Column(ColumnRef::SampleValue), - ), - guarantee, - )) - }; - let g = of(&arith( - ArithmeticOpKind::Sub, - raw("count by (job) (m)", cms.clone()), - raw("count by (job) (n)", count_sketch), - )) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert!(close(delta(&g), 0.003)); - let norms = |statistic: &str| match statistic { - "stream_l1_norm[m]" => Some(100.0), - "stream_l2_norm[n]" => Some(10.0), - _ => None, - }; - assert!(g.spec.error.has_unknown()); - assert!(close(g.bind(&norms).spec.error.evaluate().unwrap(), 1.2)); - let doubled = of(&arith( - ArithmeticOpKind::Mul, - lit(2.0), - raw("count by (job) (m)", cms), - )) - .unwrap(); - assert!(close( - doubled.bind(&norms).spec.error.evaluate().unwrap(), - 2.0 - )); - } - /// A product of two estimates has an absolute bound /// `|x|·e_y + |y|·e_x + e_x·e_y`, and a quotient with an approximate /// denominator one that needs the denominator bounded away from @@ -1532,89 +1397,6 @@ mod tests { } } - /// Which stream norm bounds a Count-Min or Count Sketch estimate (#669). - mod stream_norms { - use super::*; - use crate::test_support::{cms_summary, weighted}; - use asap_accuracy::composition::STREAM_L1_NORM; - use asap_accuracy::test_support::{source_norm_statistic, STREAM_ROW_COUNT}; - - /// A count (constant weight) is bounded by its weight times the - /// source's row count; a sum of the source's raw value column by - /// the source's norm; a sum of a computed value (`m * 1000`, - /// `rate(…)`) by nothing that can be bound. - #[test] - fn a_norm_follows_the_update_weight() { - let count = Rc::new(cms_summary("count by (job) (m)")); - assert_eq!( - stream_norm(&count), - Some(StreamNorm::ConstantWeight { - weight: 1.0, - source: "m".into() - }) - ); - let raw = weighted( - cms_summary("count by (job) (m)"), - SummaryInputExpr::Column(ColumnRef::SampleValue), - ); - assert_eq!(stream_norm(&raw), Some(StreamNorm::Source("m".into()))); - for computed in ["count by (job) (m * 1000)", "count by (job) (rate(m[1m]))"] { - let summary = weighted( - cms_summary(computed), - SummaryInputExpr::Column(ColumnRef::SampleValue), - ); - assert_eq!(stream_norm(&summary), None, "{computed}"); - } - } - - /// The absolute form of a Count-Min bound names the statistic its - /// update weight allows: the source's norm for its raw value - /// column, the weight times the source's row count for a count. - #[test] - fn the_absolute_bound_names_the_weighted_norm() { - let frequency = AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE_STREAM_L1, - BoundExpr::Constant { value: 0.01 }, - ProbabilityExpr::Constant { value: 0.001 }, - vec![], - ); - let estimate = |summary: Rc| { - operand_guarantee(&crate::test_support::estimate_over( - summary, - frequency.clone(), - )) - .unwrap() - }; - let raw = estimate(weighted( - cms_summary("count by (job) (m)"), - SummaryInputExpr::Column(ColumnRef::SampleValue), - )); - let l1 = source_norm_statistic(STREAM_L1_NORM, "m"); - assert_eq!( - raw.bind(&|s| (s == l1).then_some(100.0)) - .spec - .error - .evaluate(), - Some(1.0) - ); - let count = estimate(Rc::new(cms_summary("count by (job) (m)"))); - assert!(count - .bind(&|s| (s == l1).then_some(100.0)) - .spec - .error - .has_unknown()); - let rows = source_norm_statistic(STREAM_ROW_COUNT, "m"); - assert_eq!( - count - .bind(&|s| (s == rows).then_some(100.0)) - .spec - .error - .evaluate(), - Some(1.0) - ); - } - } - fn inventory(query: &str) -> PerComputationCandidates { let root = crate::test_support::lower_promql( query, @@ -1635,37 +1417,6 @@ mod tests { .unwrap() } - /// Every candidate of target `t`, then of its fused computation, composed - /// with every other target at its first candidate, with the candidate. - fn composed_with( - inventory: &PerComputationCandidates, - t: usize, - ) -> Vec<(HowToComputeAggregateMeasure, Rc)> { - (0..inventory.targets[t].choice_count()) - .map(|c| { - let mut choice = vec![0; inventory.targets.len()]; - choice[t] = c; - let composed = compose_logical_candidate(inventory, &choice).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - let (candidate, _) = inventory.targets[t].computation_for(c).unwrap(); - (candidate.clone(), root.clone()) - }) - .collect() - } - - fn heap_states(root: &Rc) -> Vec> { - OperatorNode::reachable(root) - .into_iter() - .filter(|node| { - matches!(node.asap(), Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(kind), .. - }) if kind.capability().answers(StatisticKind::TopK)) - }) - .collect() - } - fn topk_target(inventory: &PerComputationCandidates) -> usize { inventory .targets @@ -1680,112 +1431,6 @@ mod tests { .expect("a top-k target") } - /// Example 1's Q2: the top-k over a per-series `sum_over_time` keeps - /// only its exact candidate and gets a fused whole-expression - /// computation with a Count-Min and a CountSketch heap, which replace the - /// heaps over the exact sum. The fused computation absorbs the inner sum: composed, no - /// aggregate is computed and the heap reads the range selector's raw - /// samples. - #[test] - fn whole_expression_topk_replaces_the_heap_over_the_exact_sum() { - let inventory = inventory("topk by (job) (10, sum_over_time(m[1m]))"); - let topk = topk_target(&inventory); - let target = &inventory.targets[topk]; - assert_eq!( - target.ways_to_compute, - [HowToComputeAggregateMeasure::Exact] - ); - let fused = target - .fused_computation - .as_ref() - .expect("a fused computation"); - assert_eq!( - fused.ways_to_compute, - [ - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - ] - ); - let inner = fused.absorbed_targets[0]; - assert_eq!(inventory.targets[inner].ways_to_compute.len(), 1); - assert_eq!(inventory.combinations(), 3); - for (candidate, root) in composed_with(&inventory, topk) { - if candidate == HowToComputeAggregateMeasure::Exact { - continue; - } - assert!( - !OperatorNode::reachable(&root).iter().any(|n| matches!( - n.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) || matches!( - n.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )), - "{candidate:?}: the inner sum is not computed" - ); - let [state] = &heap_states(&root)[..] else { - panic!("one heap") - }; - let Some(ASAPOp::SummaryAgg { child, .. }) = state.asap() else { - unreachable!() - }; - assert!(matches!( - child.non_asap(), - Some(NonASAPOp::TimeRange { .. }) - )); - } - } - - /// The whole-expression heap reads the inner sum's input, so the outer - /// top-k's partition keys (indices into the sum's output) are moved onto - /// that input by column. - #[test] - fn whole_expression_topk_partitions_index_the_heap_input() { - let root = crate::test_support::lower_promql( - "topk by (job) (2, sum by (service, job) (rate(m[1m])))", - AccuracyTarget::max_error(0.1), - ); - let inventory = generate_per_computation_candidates( - &crate::stage0::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let topk = topk_target(&inventory); - let mut heaps = 0; - for (candidate, root) in composed_with(&inventory, topk) { - // The top-k's own candidate is exact; its summaries are the - // fused computation's. - if !matches!(candidate, HowToComputeAggregateMeasure::Summary(_)) { - continue; - } - for state in heap_states(&root) { - let Some(ASAPOp::SummaryAgg { - child, reduction, .. - }) = state.asap() - else { - unreachable!() - }; - let names: Vec<_> = reduction - .group_keys() - .expect("partitioned by job") - .keys() - .iter() - .map(|&index| child.schema.fields[index].name.as_str()) - .collect(); - assert_eq!(names, ["job"]); - heaps += 1; - } - } - assert_eq!( - heaps, 2, - "a whole-expression Count-Min and CountSketch heap" - ); - } - /// How many whole-workload choices enumeration keeps: every digit over /// its target's `choice_count`, less those [`absorbed_choice_varies`] /// skips. @@ -1813,166 +1458,6 @@ mod tests { } } - /// `topk(k, sum by (G) (sum_over_time(m[W])))` and its `count_over_time` - /// form rank the groups `G` by the sum (count) of their raw data samples: the - /// top-k keeps its exact candidate and gets the fused whole-expression - /// Count-Min and CountSketch heaps, which absorb both the grouped sum and - /// the per-series sum (count) beneath it. Composed, no aggregate is - /// computed: the heap reads the range selector's raw data samples, keyed by - /// `G` (less the top-k's own `by` labels, its partitions), weighted by - /// the sample value (a sum, an open requirement on `m`) or by one (a - /// count, proven non-negative). - #[test] - fn whole_expression_topk_over_a_grouped_sum_of_range_sums_reads_the_raw_data_samples() { - let samples = WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }; - let unit = WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }; - let value = SummaryInputExpr::Column(ColumnRef::SampleValue); - let one = SummaryInputExpr::Constant(1.0); - for (query, weight, domain, partitions) in [ - ( - "topk(3, sum by (region) (sum_over_time(m[1m])))", - &value, - &samples, - &[][..], - ), - ( - "topk(3, sum by (region) (count_over_time(m[1m])))", - &one, - &unit, - &[], - ), - ( - "topk by (job) (3, sum by (job, region) (sum_over_time(m[1m])))", - &value, - &samples, - &["job"], - ), - ] { - let inventory = inventory(query); - let topk = topk_target(&inventory); - let target = &inventory.targets[topk]; - assert_eq!( - target.ways_to_compute, - [HowToComputeAggregateMeasure::Exact], - "{query}" - ); - let fused = target - .fused_computation - .as_ref() - .expect("a fused computation"); - assert_eq!( - fused.ways_to_compute, - [ - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CmsWithHeap), - HowToComputeAggregateMeasure::Summary(SketchAlgorithm::CountSketchWithHeap), - ], - "{query}" - ); - assert_eq!( - fused.absorbed_targets.len(), - 2, - "{query}: both aggregates beneath" - ); - assert_eq!( - inventory.combinations(), - enumerated_choices(&inventory), - "{query}" - ); - let mut heaps = 0; - for (candidate, root) in composed_with(&inventory, topk) { - if candidate == HowToComputeAggregateMeasure::Exact { - continue; - } - assert!( - !OperatorNode::reachable(&root).iter().any(|n| matches!( - n.non_asap(), - Some(NonASAPOp::Aggregate { .. }) - ) || matches!( - n.asap(), - Some(ASAPOp::FinalizeExactAccumulator { .. }) - )), - "{query} {candidate:?}: no sum is computed" - ); - let [state] = &heap_states(&root)[..] else { - panic!("{query}: one heap") - }; - let Some(ASAPOp::SummaryAgg { - child, - input, - reduction, - .. - }) = state.asap() - else { - unreachable!() - }; - assert!( - matches!(child.non_asap(), Some(NonASAPOp::TimeRange { .. })), - "{query}: the heap reads the raw data samples" - ); - assert_eq!( - input.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("region".into()))), - "{query}" - ); - assert_eq!(&input.weight, weight, "{query}"); - assert_eq!(&input.weight_domain, domain, "{query}"); - let names: Vec<_> = reduction - .group_keys() - .map_or(&[][..], |keys| keys.keys()) - .iter() - .map(|&index| child.schema.fields[index].name.as_str()) - .collect(); - assert_eq!(names, partitions, "{query}"); - // The answer has the top-k's columns, `region` among them. - assert!( - root.schema.fields.iter().any(|f| f.name == "region"), - "{query}: {:?}", - root.schema - ); - heaps += 1; - } - assert_eq!(heaps, 2, "{query}"); - } - } - - /// The weight domain of every heap-sketch update in every candidate of - /// `query`'s top-k. - fn heap_weight_domains(query: &str) -> Vec { - let inventory = inventory(query); - let topk = topk_target(&inventory); - composed_with(&inventory, topk) - .iter() - .flat_map(|(_, root)| heap_states(root)) - .map(|state| match state.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => input.weight_domain.clone(), - _ => unreachable!(), - }) - .collect() - } - - /// Pass 1 does not read the metric's type: heap weights that are raw - /// samples of `m` carry the open requirement that `m` is a counter, under - /// any name (`m_total` included), for plan selection to confirm. - #[test] - fn heap_weights_over_raw_data_samples_record_an_open_requirement() { - for (query, metric) in [ - ("topk by (job) (10, sum_over_time(m[1m]))", "m"), - ("topk by (job) (10, sum_over_time(m_total[1m]))", "m_total"), - ] { - let domains = heap_weight_domains(query); - assert_eq!(domains.len(), 2, "{query}"); - assert!( - domains.iter().all(|d| *d - == WeightDomain::RawDataSamplesOfMetric { - metric: metric.into() - }), - "{query}: {domains:?}" - ); - } - } - /// Only raw data samples and plain sums of them: arithmetic, `rate` and other /// functions end the requirement. #[test] @@ -1994,28 +1479,6 @@ mod tests { } } - /// An avg's exact states are its sum then its count: any other order - /// is refused rather than divided the wrong way round. - #[test] - fn avg_states_out_of_order_are_refused() { - let mut inventory = inventory("avg by (job) (m)"); - assert_eq!( - inventory.targets[0].ways_to_compute, - [HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum, - ExactKind::Count - ])] - ); - assert!(compose_logical_candidate(&inventory, &[0]).is_ok()); - inventory.targets[0].ways_to_compute = vec![ - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count, ExactKind::Sum]), - ]; - assert!(matches!( - compose_logical_candidate(&inventory, &[0]), - Err(PerComputationCandidateError::Unsupported(_)) - )); - } - /// The target whose fused computation is a nested-sum merge. fn nested_sum_target(inventory: &PerComputationCandidates) -> Option { let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); @@ -2026,78 +1489,6 @@ mod tests { }) } - /// `sum by (job) (sum_over_time(m[5m]))` gets a fused nested sum - /// beside its own accumulator: one exact Sum accumulator over the range's - /// raw data samples, grouped by `job`, which absorbs the inner per-series sum. - /// Composed, no per-series state is built, the value is exact and the - /// output columns are the outer sum's. - #[test] - fn nested_sum_merges_into_one_exact_accumulator() { - let inventory = inventory("sum by (job) (sum_over_time(m[5m]))"); - let outer = nested_sum_target(&inventory).expect("a nested-sum merge"); - let target = &inventory.targets[outer]; - let sum = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - assert_eq!(target.ways_to_compute, std::slice::from_ref(&sum)); - let fused = target - .fused_computation - .as_ref() - .expect("a fused computation"); - assert_eq!(fused.ways_to_compute, [sum]); - let inner = fused.absorbed_targets[0]; - assert_eq!(inventory.targets[inner].ways_to_compute.len(), 1); - assert_eq!(inventory.combinations(), 2); - let composed = composed_with(&inventory, outer); - let (_, separate) = &composed[0]; - let (_, merged) = &composed[1]; - let states: Vec<_> = OperatorNode::reachable(merged) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - let [state] = states.as_slice() else { - panic!("one accumulator: {states:?}") - }; - let Some(ASAPOp::SummaryAgg { - child, - summary_type: family, - reduction, - .. - }) = state.asap() - else { - unreachable!() - }; - assert_eq!( - *family, - FieldDataType::ExactAggregate(ExactKind::Sum, asap_types::ir::schema::ExactParams::Sum) - ); - assert!(matches!( - child.non_asap(), - Some(NonASAPOp::TimeRange { .. }) - )); - let Reduction::Reduce(keys) = reduction else { - panic!("grouped by job: {reduction:?}") - }; - assert_eq!( - keys.keys() - .iter() - .map(|&k| child.schema.fields[k].name.as_str()) - .collect::>(), - ["job"] - ); - assert!(merged - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact)); - let names = |node: &OperatorNode| { - node.schema - .fields - .iter() - .map(|f| f.name.clone()) - .collect::>() - }; - assert_eq!(names(merged), names(separate)); - merged.validate_structure().unwrap(); - } - /// The merge needs a sum over a sum's value: an outer sum over a rate, /// an outer count, or an outer sum over an inner `max` gets none. #[test] @@ -2231,168 +1622,6 @@ mod tests { assert_eq!(inventory.combinations(), enumerated); } - /// A quantile estimate of a metric's samples carries their sign only for - /// φ in [0, 1]: outside it PromQL answers -Inf or +Inf. Pass 1 offers - /// no sketch for such a φ. - #[test] - fn a_quantile_estimate_shares_its_raw_data_samples_sign_only_inside_the_unit_interval() { - let estimate_of = |query: &str| { - let inventory = inventory(query); - (0..inventory.targets.len()) - .flat_map(|t| composed_with(&inventory, t)) - .find_map(|(_, root)| { - OperatorNode::reachable(&root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - }; - let inside = estimate_of("quantile_over_time(0.5, m[1m])").expect("a sketch estimate"); - assert_eq!(estimates_raw_data_samples_of(&inside).as_deref(), Some("m")); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = inside.asap() else { - unreachable!() - }; - for q in [-1.0, 2.0] { - let outside = OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(summary_input), - query: SketchStatistic::Quantile { q }, - }), - inside.schema.clone(), - ); - assert_eq!(estimates_raw_data_samples_of(&outside), None, "q = {q}"); - } - for query in [ - "quantile_over_time(-1, m[1m])", - "quantile_over_time(2, m[1m])", - ] { - assert!(estimate_of(query).is_none(), "{query}: a sketch is offered"); - } - } - - /// A quantile read from merged tumbling windows of a metric's samples is - /// still one of the samples (or between two), so it carries their sign - /// as the same quantile over the raw window does; φ outside [0, 1] - /// still carries none. - #[test] - fn a_quantile_of_merged_tumbling_windows_shares_its_raw_data_samples_sign() { - let inventory = inventory("quantile_over_time(0.99, m[100s])"); - let estimate = (0..inventory.targets.len()) - .flat_map(|t| composed_with(&inventory, t)) - .find_map(|(_, root)| { - OperatorNode::reachable(&root) - .into_iter() - .find(|n| matches!(n.asap(), Some(ASAPOp::SummaryEstimate { .. }))) - }) - .expect("a sketch estimate"); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = estimate.asap() else { - unreachable!() - }; - let Some(ASAPOp::SummaryAgg { - child: range, - summary_type: family, - input, - reduction, - filter, - }) = summary_input.asap() - else { - panic!("{summary_input:?}") - }; - let chain = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { - child: Rc::clone(range.children()[0]), - summary_type: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - filter: filter.clone(), - tumbling_window_width_ms: 10_000, - })) - .unwrap(), - ); - let merged = Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryMerge { - children: vec![chain], - group_by: None, - })) - .unwrap(), - ); - let read = |q: f64| { - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: Rc::clone(&merged), - query: SketchStatistic::Quantile { q }, - }), - estimate.schema.clone(), - ) - }; - assert_eq!( - estimates_raw_data_samples_of(&read(0.99)).as_deref(), - Some("m") - ); - assert_eq!( - sign_statistic(&fold_stats(&Reduction::by(vec![]), &Rc::new(read(0.99))).unwrap()), - Some(raw_data_samples_sign_statistic("m").as_str()) - ); - for q in [-1.0, 2.0] { - assert_eq!(estimates_raw_data_samples_of(&read(q)), None, "q = {q}"); - } - } - - /// The metric whose raw data samples, or plain sums of them, a composed node's - /// values estimate: a whole-expression heap's estimates and an exact - /// sum over the samples are; a rate or any other function is not. A - /// fold over them names its unknown sign by that metric. - #[test] - fn values_estimating_raw_data_samples_name_their_sign_by_the_metric() { - let inventory = inventory("topk by (job) (10, sum_over_time(m[1m]))"); - let topk = topk_target(&inventory); - for (candidate, root) in composed_with(&inventory, topk) { - let estimate = OperatorNode::reachable(&root) - .into_iter() - .find(|n| { - matches!( - n.asap(), - Some( - ASAPOp::SummaryEstimate { .. } - | ASAPOp::FinalizeExactAccumulator { .. } - ) - ) - }) - .unwrap(); - assert_eq!( - estimates_raw_data_samples_of(&estimate).as_deref(), - Some("m"), - "{candidate:?}" - ); - let stats = fold_stats(&Reduction::by(vec![]), &estimate).unwrap(); - assert_eq!( - asap_accuracy::test_support::sign_statistic(&stats), - Some(asap_accuracy::test_support::raw_data_samples_sign_statistic("m").as_str()) - ); - } - let rate = inventory_root("sum by (job) (rate(m[1m]))"); - let finalized = OperatorNode::reachable(&rate) - .into_iter() - .filter(|n| matches!(n.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. }))) - .collect::>(); - assert!(!finalized.is_empty()); - for node in finalized { - assert_eq!(estimates_raw_data_samples_of(&node), None); - } - let stats = fold_stats(&Reduction::by(vec![]), &rate).unwrap(); - assert_eq!(sign_statistic(&stats), None); - } - - /// The first composed candidate of `query`. - fn inventory_root(query: &str) -> Rc { - let inventory = inventory(query); - let composed = - compose_logical_candidate(&inventory, &vec![0; inventory.targets.len()]).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root") - }; - root.clone() - } - /// A Count-Min estimate has a bound only over weights proven /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. #[test] @@ -2452,231 +1681,4 @@ mod tests { let hydra = hll(SketchAlgorithm::HydraHll); assert!(guarantee(&hydra, "a").is_none()); } - - /// `avg_over_time` divides with a finite-division guard; a grouped `avg` does not. - #[test] - fn temporal_average_divides_with_a_finite_division_guard() { - let division_guard = |query: &str| { - let root = inventory_root(query); - let divisions: Vec = OperatorNode::reachable(&root) - .iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::BinaryOp { operator, .. }) - if operator.kind == BinaryOpKind::Arithmetic(ArithmeticOpKind::Div) => - { - Some(operator.checked_finite_division) - } - _ => None, - }) - .collect(); - let [guarded] = divisions[..] else { - panic!("{query}: one sum / count division: {root:#?}") - }; - guarded - }; - assert!(division_guard("avg_over_time(a[5m])")); - assert!(!division_guard("avg by (job) (m)")); - } - - /// The candidates of `query` lowered at `accuracy`, over rows without the - /// series-identity column (open PromQL schemas): over identity-carrying - /// (closed) rows a grouped top-k gets no whole-expression heap. - fn inventory_at(query: &str, accuracy: AccuracyTarget) -> PerComputationCandidates { - let root = crate::test_support::lower_promql(query, accuracy); - generate_per_computation_candidates( - &crate::stage0::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - } - - fn open_inventory(query: &str) -> PerComputationCandidates { - inventory_at( - query, - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ) - } - - /// The estimate and state of the top-k's fused whole-expression heap of - /// `algorithm`. - fn fused_heap( - inventory: &PerComputationCandidates, - algorithm: SketchAlgorithm, - ) -> (SketchStatistic, Rc) { - let topk = topk_target(inventory); - let own = inventory.targets[topk].ways_to_compute.len(); - let (_, root) = composed_with(inventory, topk) - .into_iter() - .skip(own) - .find(|(candidate, _)| *candidate == HowToComputeAggregateMeasure::Summary(algorithm)) - .unwrap_or_else(|| panic!("a fused {algorithm:?} candidate")); - let [state] = &heap_states(&root)[..] else { - panic!("one heap: {root:#?}") - }; - let estimate = OperatorNode::reachable(&root) - .into_iter() - .find_map(|node| match node.asap() { - Some(ASAPOp::SummaryEstimate { - summary_input, - query, - }) if Rc::ptr_eq(summary_input, state) => Some(query.clone()), - _ => None, - }) - .expect("the heap's estimate"); - (estimate, state.clone()) - } - - fn heap_update(state: &OperatorNode) -> &SummaryUpdate { - match state.asap() { - Some(ASAPOp::SummaryAgg { input, .. }) => input, - _ => unreachable!(), - } - } - - /// A top-k over a `count` fuses into one global unit-weight heap over the raw rows, ranking the label. - #[test] - fn count_ranked_topk_fuses_to_one_global_heap_sketch() { - let inventory = open_inventory("topk(10, count by (service) (m))"); - let topk = topk_target(&inventory); - let fused = inventory.targets[topk] - .fused_computation - .as_ref() - .expect("a fused heap"); - assert!(fused - .ways_to_compute - .contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CmsWithHeap - ))); - let (estimate, state) = fused_heap(&inventory, SketchAlgorithm::CmsWithHeap); - assert_eq!(estimate, SketchStatistic::TopK { k: 10 }); - let update = heap_update(&state); - assert_eq!( - update.item, - Some(SummaryInputExpr::Column(ColumnRef::Named("service".into()))) - ); - assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - assert_eq!( - update.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - } - ); - let Some(ASAPOp::SummaryAgg { child, .. }) = state.asap() else { - unreachable!() - }; - assert!(!child.contains_asap()); - } - - /// The ranked item over several group keys is their tuple, in the lowered (name-sorted) key order. - #[test] - fn multidimensional_ranked_item_is_a_structured_tuple() { - let inventory = open_inventory("topk(10, count by (service, region) (m))"); - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CmsWithHeap); - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - assert_eq!( - heap_update(&state).item, - Some(SummaryInputExpr::Tuple(vec![ - column("region"), - column("service") - ])) - ); - } - - /// The top-k's own partition keys partition the heap and are left out of the ranked tuple. - #[test] - fn subpopulation_columns_are_not_part_of_the_ranked_item_tuple() { - let inventory = - open_inventory("topk by (region) (10, count by (service, method, region) (m))"); - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CmsWithHeap); - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - assert_eq!( - heap_update(&state).item, - Some(SummaryInputExpr::Tuple(vec![ - column("method"), - column("service") - ])) - ); - let Some(ASAPOp::SummaryAgg { - child, reduction, .. - }) = state.asap() - else { - unreachable!() - }; - let keys: Vec<_> = reduction - .group_keys() - .expect("partitioned by region") - .keys() - .iter() - .map(|&k| child.schema.fields[k].name.as_str()) - .collect(); - assert_eq!(keys, ["region"]); - } - - /// A per-series top-k ranks series identity (column, else label set) weighted by sample value. - #[test] - fn temporal_per_entity_topk_uses_series_identity_and_sample_value() { - let query = "topk by (service) (5, sum_over_time(m[1m]))"; - let with_identity = inventory(query); - let without_identity = open_inventory(query); - for (inventory, item) in [ - ( - with_identity, - SummaryInputExpr::Column(ColumnRef::Named( - asap_types::ir::schema::PROMQL_SERIES_IDENTITY.into(), - )), - ), - ( - without_identity, - SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { - excluding: vec![ColumnRef::Named("service".into())], - }), - ), - ] { - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CountSketchWithHeap); - let update = heap_update(&state); - assert_eq!(update.item, Some(item)); - assert_eq!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ); - } - } - - // Identity realization must preserve named labels and every top-k candidate's output contract. - #[test] - fn topk_keeps_known_labels_beside_series_identity() { - let inventory = inventory("topk by (service) (5, sum_over_time(m{instance=\"a\"}[1m]))"); - let topk = topk_target(&inventory); - let expected = &inventory.aggregate(&inventory.targets[topk]).schema; - assert!(expected.fields.iter().any(|field| field.name == "instance")); - assert!(expected - .fields - .iter() - .any(|field| field.name == asap_types::ir::schema::PROMQL_SERIES_IDENTITY)); - for (_, root) in composed_with(&inventory, topk) { - assert_eq!(root.schema.fields, expected.fields); - assert_eq!(root.schema.time_index, expected.time_index); - assert_eq!(root.schema.closed, expected.closed); - // A physical heap may prove an additional partition/item key. - // Every required uniqueness fact must still follow from its keys. - for required in &expected.unique_keys { - assert!( - root.schema.unique_keys.iter().any(|supplied| { - supplied.iter().all(|column| required.contains(column)) - }), - "required key {required:?} is not implied by {:?}", - root.schema.unique_keys - ); - } - } - let (_, state) = fused_heap(&inventory, SketchAlgorithm::CountSketchWithHeap); - assert!(matches!( - heap_update(&state).item, - Some(SummaryInputExpr::Tuple(_)) - )); - } } diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs index 3ab7403e6..fda16d839 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs @@ -319,25 +319,6 @@ fn assert_sql_count_choices(choices: &[HowToComputeAggregateMeasure]) { ); } -fn quantile() -> AggIntent { - AggIntent::Quantile { - col: Some(1), - q: 0.5, - } -} - -fn grouped_quantile(having: Option) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![0]), - measures: vec![quantile()], - output_names: vec![], - filters: vec![], - having, - })) - .unwrap() -} - /// `count(*) by (src_ip)`, the form the SQL frontend gives `GROUP BY src_ip`. fn grouped_count(having: Option) -> Rc { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { @@ -744,396 +725,6 @@ fn without_and_per_entity_having_stays_above() { } } -/// A workload candidate replaces only chosen targets, derives the summary's -/// coverage from its subtree, and keeps unchosen plans as they are, now -/// with an exact guarantee. -#[test] -fn composed_candidate_replaces_chosen_target_with_summary_evaluation() { - use crate::pass1::composition::compose_logical_candidate; - let producer = aggregate(AggIntent::Cardinality { cols: vec![0] }); - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(producer.clone())], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let exact = compose_logical_candidate(&inventory, &[0]).unwrap(); - let QueryRoot::Operator(kept) = &exact[0] else { - panic!("operator root expected") - }; - assert_eq!(kept.operator.kind_name(), producer.operator.kind_name()); - assert_eq!(kept.schema, producer.schema); - assert!(kept.guarantee.as_ref().is_some_and(|g| g.is_exact())); - - let hll = compose_logical_candidate(&inventory, &[1]).unwrap(); - let QueryRoot::Operator(estimate) = &hll[0] else { - panic!("operator root expected") - }; - assert!(matches!( - estimate.asap(), - Some(ASAPOp::SummaryEstimate { .. }) - )); - hll[0].validate_structure().unwrap(); - assert!(compose_logical_candidate(&inventory, &[99]).is_err()); -} - -/// The one root of the workload candidate composed from `roots` and `choice`. -fn composed(roots: Vec>, choice: &[usize]) -> Vec> { - use crate::pass1::composition::compose_logical_candidate; - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &roots - .iter() - .cloned() - .map(QueryRoot::Operator) - .collect::>(), - &vec![None; roots.len()], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - compose_logical_candidate(&inventory, choice) - .unwrap() - .into_iter() - .map(|root| { - root.validate_structure().unwrap(); - match root { - QueryRoot::Operator(node) => node, - QueryRoot::Scalar(_) => panic!("operator root expected"), - } - }) - .collect() -} -/// Output columns as (name, type, nullable). -fn columns(node: &OperatorNode) -> Vec<(String, FieldDataType, bool)> { - node.schema - .fields - .iter() - .map(|field| (field.name.clone(), field.dtype.clone(), field.nullable)) - .collect() -} -// Quantile readouts restore the aggregate's output names with an identity projection. -fn without_output_aliases(node: &OperatorNode) -> &OperatorNode { - if let Some(NonASAPOp::Project { cols, child, .. }) = node.non_asap() { - assert!(cols - .iter() - .enumerate() - .all(|(index, item)| item.expr == ScalarExpr::Column(index))); - without_output_aliases(child) - } else { - node - } -} -fn summary_agg(node: &OperatorNode) -> &ASAPOp { - let node = without_output_aliases(node); - match node.asap() { - Some(ASAPOp::SummaryEstimate { summary_input, .. }) => summary_agg(summary_input), - Some(ASAPOp::FinalizeExactAccumulator { child }) => summary_agg(child), - Some(agg @ ASAPOp::SummaryAgg { .. }) => agg, - _ => panic!("summary evaluation expected, got {node:#?}"), - } -} - -/// A measure filter becomes the state's filter; a key-only `HAVING` applied -/// by every consumer moves below the state; a value `HAVING` stays a `Filter` -/// above the estimate. -#[test] -fn composed_filters_and_having() { - let kll = 1; - let filtered = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![0]), - measures: vec![quantile()], - output_names: vec![], - filters: vec![Some(always())], - having: None, - })) - .unwrap(); - let [root] = <[_; 1]>::try_from(composed(vec![filtered.clone()], &[kll])).unwrap(); - assert!( - matches!(summary_agg(&root), ASAPOp::SummaryAgg { filter: Some(f), child, .. } - if *f == always() && child.non_asap().is_some_and(|op| matches!(op, NonASAPOp::Scan { .. }))) - ); - assert_eq!(columns(&root)[0], columns(&filtered)[0]); - - let key = equals(0, ScalarValue::Utf8("a".into())); - let [root] = <[_; 1]>::try_from(composed( - vec![having_filter(grouped_quantile(None), key.clone())], - &[kll], - )) - .unwrap(); - let ASAPOp::SummaryAgg { child, filter, .. } = summary_agg(&root) else { - unreachable!() - }; - assert_eq!(*filter, None); - assert!(matches!(child.non_asap(), Some(NonASAPOp::Filter { pred, .. }) if *pred == key)); - - let value = || equals(1, ScalarValue::Int64(10)); - for root in composed( - vec![ - having_filter(grouped_quantile(None), value()), - grouped_quantile(Some(value())), - ], - &[kll, kll], - ) { - let Some(NonASAPOp::Filter { pred, child }) = root.non_asap() else { - panic!("value HAVING above the estimate expected, got {root:#?}") - }; - assert_eq!(*pred, value()); - assert!(matches!( - without_output_aliases(child).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - )); - } -} - -/// SQL `avg` joins its finalized `sum` and `count` accumulator states on the -/// group keys and divides them. -#[test] -fn composed_avg_divides_sum_and_count_states() { - let avg = aggregate_over( - scan(false), - vec![AggIntent::Avg { col: Some(1) }], - vec![], - None, - ); - let [root] = <[_; 1]>::try_from(composed(vec![avg.clone()], &[0])).unwrap(); - // The quotient is renamed to the aggregate's column. - assert_eq!(columns(&root), columns(&avg)); - let join = OperatorNode::reachable(&root) - .into_iter() - .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Join { .. }))) - .unwrap_or_else(|| panic!("sum joined with count expected, got {root:#?}")); - let Some(NonASAPOp::Join { left, right, .. }) = join.non_asap() else { - unreachable!() - }; - for (side, kind) in [(left, ExactKind::Sum), (right, ExactKind::Count)] { - assert!(matches!( - summary_agg(side), - ASAPOp::SummaryAgg { summary_type: FieldDataType::ExactAggregate(found, _), .. } if *found == kind - )); - } -} - -/// Each measure of a multi-measure aggregate is substituted on its own and the -/// measures are joined back on the group keys, with the aggregate's columns. -#[test] -fn composed_multi_measure_aggregate_joins_its_measures() { - for reduction in [Reduction::by(vec![0]), Reduction::by(vec![])] { - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction, - measures: vec![ - AggIntent::Sum { col: Some(1) }, - quantile(), - AggIntent::StdDev { - col: Some(1), - population: true, - }, - ], - output_names: vec![], - filters: vec![], - having: None, - })) - .unwrap(); - let [root] = <[_; 1]>::try_from(composed(vec![node.clone()], &[0, 1])).unwrap(); - assert!(matches!(root.non_asap(), Some(NonASAPOp::Project { .. }))); - let reached = OperatorNode::reachable(&root); - let joins = reached - .iter() - .filter(|node| matches!(node.non_asap(), Some(NonASAPOp::Join { .. }))) - .count(); - assert_eq!(joins, 2); - assert!(reached - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::SummaryEstimate { .. })))); - let names = |node: &OperatorNode| -> Vec<_> { - columns(node) - .into_iter() - .map(|(name, dtype, _)| (name, dtype)) - .collect() - }; - assert_eq!(names(&root), names(&node)); - } -} - -/// A measure left exact beside a sketched one keeps its output name: the -/// split exact aggregate names its one measure as the target named it, so the -/// executor finds one name per measure. -#[test] -fn composed_mixed_exact_and_sketch_measures_keep_output_names() { - let quantile = |q| AggIntent::Quantile { col: Some(1), q }; - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![]), - measures: vec![quantile(0.5), quantile(0.9)], - output_names: vec!["p50".into(), "p90".into()], - filters: vec![], - having: None, - })) - .unwrap(); - assert_eq!( - candidates_for(quantile(0.5))[0], - HowToComputeAggregateMeasure::Exact - ); - // p50 exact, p90 from the first sketch family. - let [root] = <[_; 1]>::try_from(composed(vec![node.clone()], &[0, 1])).unwrap(); - let exact_parts: Vec<_> = OperatorNode::reachable(&root) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Aggregate { - measures, - output_names, - .. - }) => Some((measures.len(), output_names.clone())), - _ => None, - }) - .collect(); - assert_eq!(exact_parts, vec![(1, vec!["p50".to_string()])]); - assert_eq!(columns(&root), columns(&node)); -} - -/// A summary estimate carries its sized family's bound; an exact accumulator -/// is exact; anything computed from an approximate value, a value `HAVING` -/// included, has no guarantee. -#[test] -fn composed_guarantees() { - let value = equals(1, ScalarValue::Int64(10)); - let roots = composed( - vec![ - aggregate(AggIntent::Cardinality { cols: vec![0] }), - aggregate(AggIntent::Sum { col: Some(1) }), - having_filter(grouped_quantile(None), value), - ], - &[1, 0, 1], - ); - let guarantee = |node: &OperatorNode| node.guarantee.clone(); - let hll = guarantee(&roots[0]).expect("HLL bound"); - assert!(!hll.is_exact()); - assert!(guarantee(&roots[1]).is_some_and(|g| g.is_exact())); - let Some(NonASAPOp::Filter { child, .. }) = roots[2].non_asap() else { - panic!("value HAVING above the estimate expected") - }; - assert!(guarantee(child).is_some()); - assert!(guarantee(&roots[2]).is_none()); -} - -/// The aggregate's own value `having` stays a `Filter` above the output, and -/// a subquery in it reads the subquery's substitution. -#[test] -fn own_having_subquery_is_substituted() { - use crate::pass1::composition::compose_logical_candidate; - let inner = aggregate(quantile()); - let outer = grouped_quantile(Some(above_scalar(1, inner.clone()))); - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(outer.clone())], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - assert_eq!(inventory.targets.len(), 2); - let kll = 1; - let composed = compose_logical_candidate(&inventory, &[kll, kll]).unwrap(); - let QueryRoot::Operator(root) = &composed[0] else { - panic!("operator root expected") - }; - let Some(NonASAPOp::Filter { pred, child }) = root.non_asap() else { - panic!("own HAVING above the estimate expected, got {root:#?}") - }; - assert!(matches!( - without_output_aliases(child).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - )); - let [subquery] = <[_; 1]>::try_from(pred.0.operator_refs()).unwrap(); - assert!( - matches!( - without_output_aliases(subquery).asap(), - Some(ASAPOp::SummaryEstimate { .. }) - ), - "{subquery:#?}" - ); - assert!(OperatorNode::reachable(root) - .iter() - .all(|node| !matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })))); -} - -/// A per-series `avg` over PromQL rows that carry the series identity -/// (`$promql_series_identity` after the sample value) composes from its sum -/// and count states: the summed value is found by role, not as the last -/// column. -#[test] -fn per_series_avg_with_series_identity_composes_from_sum_and_count() { - use asap_types::ir::schema::schema_support::with_promql_series_identity; - let [QueryRoot::Operator(root)] = &exact_promql("avg_over_time(m[5m])")[..] else { - panic!("one operator root"); - }; - let root = with_promql_series_identity(root).unwrap(); - let [composed] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); - assert_eq!(columns(&composed), columns(&root)); -} - -/// Exact per-series `rate` keeps every label and retypes the value column to the Rate state. -#[test] -fn per_series_rate_keeps_labels_and_retypes_its_state() { - use asap_types::ir::schema::ExactParams; - let [QueryRoot::Operator(root)] = &exact_promql("rate(m{job=\"api\"}[5m])")[..] else { - panic!("one operator root"); - }; - let [out] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); - let rate = FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate); - let ASAPOp::SummaryAgg { - summary_type: family, - reduction, - .. - } = summary_agg(&out) - else { - unreachable!() - }; - assert_eq!(*family, rate); - assert_eq!(*reduction, Reduction::PerEntity); - let state = OperatorNode::reachable(&out) - .into_iter() - .find(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .unwrap(); - // The state replaces the sample value in place, named `state`; the - // finalized output has the rate's own columns. - let label = Field::plain("job", DataType::Utf8, true); - assert_eq!( - columns(&state), - [ - ( - "ts".into(), - FieldDataType::Plain(DataType::Timestamp), - false - ), - ("state".into(), rate, false), - (label.name, label.dtype, label.nullable), - ] - ); - assert_eq!(state.schema.time_index, Some(0)); - assert_eq!(out.schema, root.schema); -} - -/// Exact grouped `avg` composes from sum and count with the aggregate's whole schema. -#[test] -fn grouped_avg_composition_preserves_the_whole_schema() { - let [QueryRoot::Operator(root)] = &exact_promql("avg by (job) (m)")[..] else { - panic!("one operator root"); - }; - let [out] = <[_; 1]>::try_from(composed(vec![root.clone()], &[0])).unwrap(); - assert!(OperatorNode::reachable(&out) - .iter() - .any(|node| matches!(node.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))); - assert_eq!(out.schema, root.schema); -} - // Structurally equal aggregates are one memo group with one target, and its // HAVING placement is decided over every consumer of the group: an unfiltered // consumer keeps the key-only HAVING above the output for the filtered ones. diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/logical-asap-optimization/src/pass1/realization.rs index 9e60623ec..b9e0bc393 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/logical-asap-optimization/src/pass1/realization.rs @@ -295,7 +295,6 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc #[cfg(test)] mod tests { use super::*; - use crate::pass1::composition::summarised_input; use crate::test_support::metric_scan; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ExprSemantics, Predicate}; @@ -341,32 +340,6 @@ mod tests { .is_none()); } - /// A summary reads the intent's own input columns: one column by name, - /// none as the sample value, several as one hashed tuple. - #[test] - fn summarised_input_reads_the_intents_columns() { - // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. - let scan = metric_scan(&["job"]); - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - let cases = [ - (vec![2], column("job")), - (vec![1], column("value")), - (vec![], SummaryInputExpr::Column(ColumnRef::SampleValue)), - ( - vec![1, 2], - SummaryInputExpr::Tuple(vec![column("value"), column("job")]), - ), - ]; - for (cols, want) in cases { - let intent = AggIntent::Cardinality { cols: cols.clone() }; - assert_eq!( - summarised_input(&intent, &scan.schema), - Some(want), - "wrong summarised input for cols={cols:?}" - ); - } - } - /// Each approximable intent is offered the families it was offered when /// the list was written by hand, in the same order; a non-sketch intent /// is offered none. diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index d1699ab00..5f1d5673e 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -4,9 +4,7 @@ use std::rc::Rc; -// Stage 1 internals other crates' tests call: Pass 1's working form, -// and single-candidate composition. -pub use crate::pass1::composition::compose_logical_candidate; +// Stage 1 internals other crates' tests call: Pass 1's working form. pub use crate::pass1::per_computation_candidates::{ generate_per_computation_candidates, PerComputationCandidates, }; @@ -174,88 +172,6 @@ pub fn time_range(range: Duration, child: Rc) -> Rc .unwrap() } -/// The Count-Min summary of a Stage 1 candidate of `query`. -pub fn cms_summary(query: &str) -> OperatorNode { - use crate::pass1::composition::compose_logical_candidate; - use crate::pass1::per_computation_candidates::generate_per_computation_candidates; - use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; - use asap_types::ir::{ASAPOp, QueryRoot}; - let target = AccuracyTarget::max_error(0.01).failure_probability(0.01); - let root = lower_promql(query, target); - // Inspect each local computation choice under the built-in deployment. - let per_computation = generate_per_computation_candidates( - &crate::stage0::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &asap_types::workload::DataWorkload::default(), - ) - .unwrap(), - ) - .unwrap() - .without_unsupported_summaries(None); - let summary = per_computation - .targets - .iter() - .enumerate() - .flat_map(|(target, choices)| { - let inventory = &per_computation; - (0..choices.choice_count()).filter_map(move |candidate| { - let mut assignment = vec![0; inventory.targets.len()]; - assignment[target] = candidate; - compose_logical_candidate(inventory, &assignment).ok() - }) - }) - .flat_map(|roots| match &roots[0] { - QueryRoot::Operator(node) => OperatorNode::reachable(node), - QueryRoot::Scalar(_) => vec![], - }) - .find(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(kind), - .. - }) if *kind == SketchAlgorithm::Cms - ) - }) - .map(|node| (*node).clone()) - .unwrap_or_else(|| panic!("{query}: a Count-Min summary")); - summary -} - -/// `summary` with its update weight replaced by `weight`. -pub fn weighted( - mut summary: OperatorNode, - weight: asap_types::ir::schema::SummaryInputExpr, -) -> Rc { - use asap_types::ir::{ASAPOp, Operator}; - if let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = &mut summary.operator { - input.weight = weight; - } - Rc::new(summary) -} - -/// A point-count estimate over `summary` carrying `guarantee`. -pub fn estimate_over( - summary: Rc, - guarantee: asap_types::ir::properties::AccuracyGuarantee, -) -> Rc { - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::SketchStatistic; - use asap_types::ir::{ASAPOp, Operator}; - Rc::new( - OperatorNode::new(Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: summary, - query: SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }, - })) - .unwrap() - .with_guarantee(Some(guarantee)), - ) -} - /// The workload entry of a query evaluated as `recurrence` says, exact and /// with no latency bound. pub fn entry( diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 26ad5b02e..953bf57f0 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -66,7 +66,7 @@ same pipeline behind the `OptimizationPass` trait. | Concern | Where it lives | | --- | --- | -| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) and composing one choice into a logical DAG (`compose_logical_candidate`) | `asap-logical-asap-optimization`, `pass1/per_computation_candidates.rs` | +| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) | `asap-logical-asap-optimization`, `pass1/per_computation_candidates.rs` | | Which summary families serve an intent (`summary_candidates`) | `asap-logical-asap-optimization`, `pass1/realization.rs` | | Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-logical-asap-optimization`, `pass2/memo/rules/` and `pass2/memo/window_candidates.rs`, over the helpers in `pass2/asap_aware_cse.rs` | | Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | `asap-logical-asap-optimization`, `pass2/memo/enumeration.rs` | diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index d46ee4c89..4935b3c26 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -14,7 +14,7 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | After | The DAG holds | Timing | | --- | --- | --- | | Stage 0 (frontends) | `NonASAPOp` nodes only; aggregates carry `AggIntent` measures with each query's `AccuracyTarget` as written | none | -| Stage 1 (`compose_logical_candidate`, Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | +| Stage 1 (Pass 1 and Pass 2 rules) | `ASAPOp` nodes where a candidate chose an exact accumulator or a summary (`SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge`, `SummaryEstimate`, `FinalizeExactAccumulator`); everything else stays `NonASAPOp` | none: Stage 1 never decides when anything runs | | Stage 2 (`generate_physical_asap_memo_candidates`) and Stage 3 (`select_plan`, which builds the plan) | the same operators, with a timing and a `Lifetime` on each summary build | every node of the plan timed from the physical candidate it was built from (`selected_physical_candidate`) | - Structurally equal sub-DAGs are interned before Stage 1 From 61541f748003c74d1c6d72be731bf1d15ffdd12e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:21:11 +0000 Subject: [PATCH 20/30] test(sql): lower main's new ClickHouse tests without the removed accuracy argument (#1207 PR 0) --- crates/frontend-sql/tests/clickbench/clickbench.rs | 11 ++--------- crates/frontend-sql/tests/sql_lowering.rs | 12 +++--------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/crates/frontend-sql/tests/clickbench/clickbench.rs b/crates/frontend-sql/tests/clickbench/clickbench.rs index c57b15017..5dfbbec25 100644 --- a/crates/frontend-sql/tests/clickbench/clickbench.rs +++ b/crates/frontend-sql/tests/clickbench/clickbench.rs @@ -19,7 +19,6 @@ use std::rc::Rc; use asap_frontend_sql::{lower_sql_dialect, SqlCatalog, SqlError as LoweringError}; use asap_types::ir::operator::{AggIntent, GroupKeys}; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; use asap_types::workload::SqlDialect; @@ -166,13 +165,7 @@ fn queries() -> Vec<&'static str> { } async fn lower(q: &str) -> Result, LoweringError> { - lower_sql_dialect( - q, - &catalog(), - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await + lower_sql_dialect(q, &catalog(), SqlDialect::ClickhouseSQL).await } /// Every ClickBench query lowers end to end. A query that stops lowering @@ -251,7 +244,7 @@ async fn top_search_phrases_canonicalize_to_top_k() { let (keys, inner) = first_aggregate(child).expect("q13 expected a per-phrase Aggregate"); assert_eq!(*keys, GroupKeys::by(vec![column("SearchPhrase")])); assert!( - matches!(inner.as_slice(), [AggIntent::Count { .. }]), + matches!(inner.as_slice(), [AggIntent::Count]), "q13 expected a per-phrase Count, got {inner:?}" ); } diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 9197318c3..84829c04b 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -1788,7 +1788,6 @@ async fn clickhouse_identifiers_keep_their_case() { "SELECT RegionID, COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID", &hits, SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), ) .await .unwrap_or_else(|e| panic!("mixed-case ClickHouse columns must resolve: {e}")); @@ -1799,14 +1798,9 @@ async fn clickhouse_identifiers_keep_their_case() { "expected COUNT(DISTINCT UserID) over column 1, got {measures:?}" ); - let err = lower_sql_dialect( - "SELECT userid FROM hits", - &hits, - SqlDialect::ClickhouseSQL, - AccuracyTarget::exact(), - ) - .await - .expect_err("ClickHouse must not match `userid` to `UserID`"); + let err = lower_sql_dialect("SELECT userid FROM hits", &hits, SqlDialect::ClickhouseSQL) + .await + .expect_err("ClickHouse must not match `userid` to `UserID`"); assert!( format!("{err}").contains("userid"), "unexpected error: {err}" From 77ebd23353dddaf372a5f00ef927e0b403ed5ce6 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:07:51 +0000 Subject: [PATCH 21/30] refactor: one crate per stage (#1207 PR 0b) Move-only. Split asap-logical-asap-optimization into asap-stage0-memo, asap-stage1-pass1-per-computation-candidates and asap-stage1-pass2-reuse-candidates; rename asap-physical-asap-optimization to asap-stage2-physical-optimization and asap-plan-selection to asap-stage3-plan-selection. The old test_support splits by caller. The crate-graph test checks the new crates; no stage crate depends on another. --- Cargo.lock | 94 +++--- Cargo.toml | 8 +- crates/cost/Cargo.toml | 5 - crates/cost/src/lib.rs | 2 +- crates/cost/src/usage.rs | 2 +- crates/devtools/Cargo.toml | 12 +- crates/devtools/src/bin/asap_planner.rs | 2 +- crates/devtools/src/bin/sketch_coverage.rs | 4 +- crates/devtools/src/planning_stages.rs | 13 +- crates/devtools/tests/viewer_contract.rs | 2 +- crates/executor/Cargo.toml | 3 +- crates/executor/tests/current_series_heap.rs | 6 +- .../executor/tests/deployment_computation.rs | 5 +- crates/executor/tests/physical_semantics.rs | 2 +- crates/frontend-common/src/canonicalize.rs | 2 +- crates/frontend-metricsql/Cargo.toml | 3 +- crates/frontend-promql/Cargo.toml | 5 +- crates/frontend-sql/Cargo.toml | 5 +- crates/frontend-sql/tests/frequency_l2.rs | 4 +- crates/integration-tests/Cargo.toml | 9 +- crates/integration-tests/tests/crate_graph.rs | 51 ++-- .../tests/per_computation_candidates_sql.rs | 4 +- crates/logical-asap-optimization/src/lib.rs | 82 ------ crates/logical-asap-optimization/src/pass1.rs | 20 -- crates/logical-asap-optimization/src/pass2.rs | 6 - .../src/test_support.rs | 271 ------------------ .../tests/pass_layering.rs | 59 ---- crates/planner/Cargo.toml | 21 +- crates/planner/src/lib.rs | 6 +- crates/planner/src/pass.rs | 2 +- crates/planner/src/run.rs | 12 +- crates/stage0-memo/Cargo.toml | 10 + .../src/stage0.rs => stage0-memo/src/lib.rs} | 13 +- .../Cargo.toml | 20 +- .../src/lib.rs | 49 ++++ .../src}/logical_memo_candidates.rs | 2 +- .../src}/per_computation_candidates.rs | 8 +- .../src}/per_computation_candidates_tests.rs | 8 +- .../src}/realization.rs | 6 +- .../src/test_fixtures.rs | 49 ++++ .../src/test_support.rs | 45 +++ .../stage1-pass2-reuse-candidates/Cargo.toml | 14 + .../stage1-pass2-reuse-candidates/src/lib.rs | 21 ++ .../src}/logical_memo_candidates.rs | 4 +- .../src}/memo.rs | 2 +- .../src}/memo/rules.rs | 0 .../src}/memo/rules/finer_aggregate.rs | 4 +- .../src}/memo/rules/rollup.rs | 2 +- .../src}/memo/rules/scan_labels.rs | 2 +- .../src}/memo/rules/summary_inputs.rs | 2 +- .../src}/memo/rules/topk.rs | 2 +- .../src}/memo/window_candidates.rs | 2 +- .../Cargo.toml | 2 +- .../src/catalog/mod.rs | 0 .../src/lib.rs | 0 .../Cargo.toml | 2 +- .../src/lib.rs | 0 .../src/milp/mod.rs | 0 .../src/milp/model.rs | 0 .../src/report.rs | 0 .../src/select.rs | 0 .../tests/cost_inputs_json.rs | 0 crates/types/src/ir/schema/schema_support.rs | 2 +- crates/types/src/test_support.rs | 75 ++++- crates/types/tests/planner_vocabulary.rs | 2 +- docs/design_docs/architecture/README.md | 7 +- ...d_interface_with_pluggable_optimization.md | 10 +- .../decisions/concat-unique-keys.md | 2 +- docs/design_docs/decisions/cse-cost-model.md | 2 +- .../proposals/asap-primitive-schema.md | 4 +- .../joint-planning-implementation-plan.md | 16 +- .../proposals/joint-planning-search.md | 2 +- .../planner-layering-example2-acceptance.md | 2 +- .../design_docs/proposals/planner-layering.md | 14 +- .../asap-aware-mapping-architecture.md | 53 ++-- .../asap-aware-mapping-contracts.md | 6 +- .../develop_docs/extend-asap-aware-mapping.md | 8 +- docs/develop_docs/library-api.md | 22 +- docs/develop_docs/pre-asap-ir.md | 2 +- .../target-candidate-api-migration.md | 2 +- 80 files changed, 544 insertions(+), 678 deletions(-) delete mode 100644 crates/logical-asap-optimization/src/lib.rs delete mode 100644 crates/logical-asap-optimization/src/pass1.rs delete mode 100644 crates/logical-asap-optimization/src/pass2.rs delete mode 100644 crates/logical-asap-optimization/src/test_support.rs delete mode 100644 crates/logical-asap-optimization/tests/pass_layering.rs create mode 100644 crates/stage0-memo/Cargo.toml rename crates/{logical-asap-optimization/src/stage0.rs => stage0-memo/src/lib.rs} (65%) rename crates/{logical-asap-optimization => stage1-pass1-per-computation-candidates}/Cargo.toml (51%) create mode 100644 crates/stage1-pass1-per-computation-candidates/src/lib.rs rename crates/{logical-asap-optimization/src/pass1 => stage1-pass1-per-computation-candidates/src}/logical_memo_candidates.rs (97%) rename crates/{logical-asap-optimization/src/pass1 => stage1-pass1-per-computation-candidates/src}/per_computation_candidates.rs (99%) rename crates/{logical-asap-optimization/src/pass1 => stage1-pass1-per-computation-candidates/src}/per_computation_candidates_tests.rs (99%) rename crates/{logical-asap-optimization/src/pass1 => stage1-pass1-per-computation-candidates/src}/realization.rs (98%) create mode 100644 crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs create mode 100644 crates/stage1-pass1-per-computation-candidates/src/test_support.rs create mode 100644 crates/stage1-pass2-reuse-candidates/Cargo.toml create mode 100644 crates/stage1-pass2-reuse-candidates/src/lib.rs rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/logical_memo_candidates.rs (95%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo.rs (87%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/rules.rs (100%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/rules/finer_aggregate.rs (76%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/rules/rollup.rs (93%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/rules/scan_labels.rs (90%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/rules/summary_inputs.rs (91%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/rules/topk.rs (90%) rename crates/{logical-asap-optimization/src/pass2 => stage1-pass2-reuse-candidates/src}/memo/window_candidates.rs (94%) rename crates/{physical-asap-optimization => stage2-physical-optimization}/Cargo.toml (90%) rename crates/{physical-asap-optimization => stage2-physical-optimization}/src/catalog/mod.rs (100%) rename crates/{physical-asap-optimization => stage2-physical-optimization}/src/lib.rs (100%) rename crates/{plan-selection => stage3-plan-selection}/Cargo.toml (95%) rename crates/{plan-selection => stage3-plan-selection}/src/lib.rs (100%) rename crates/{plan-selection => stage3-plan-selection}/src/milp/mod.rs (100%) rename crates/{plan-selection => stage3-plan-selection}/src/milp/model.rs (100%) rename crates/{plan-selection => stage3-plan-selection}/src/report.rs (100%) rename crates/{plan-selection => stage3-plan-selection}/src/select.rs (100%) rename crates/{plan-selection => stage3-plan-selection}/tests/cost_inputs_json.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 933b01efb..a931d4735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -294,8 +294,6 @@ name = "asap-cost" version = "0.1.0" dependencies = [ "asap-executor", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", "asap-types", "asap_sketchlib", "serde", @@ -312,10 +310,12 @@ dependencies = [ "asap-executor", "asap-frontend-promql", "asap-frontend-sql", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", - "asap-plan-selection", "asap-planner", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", + "asap-stage1-pass2-reuse-candidates", + "asap-stage2-physical-optimization", + "asap-stage3-plan-selection", "asap-types", "serde", "serde_json", @@ -331,8 +331,9 @@ dependencies = [ "arrow", "asap-executor", "asap-frontend-promql", - "asap-logical-asap-optimization", "asap-planner", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", "asap-types", "asap_sketchlib", "chrono", @@ -360,7 +361,8 @@ dependencies = [ "asap-frontend-common", "asap-frontend-metricsql", "asap-frontend-promql", - "asap-logical-asap-optimization", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", "asap-types", "metricsql_parser", "thiserror 2.0.18", @@ -372,8 +374,9 @@ version = "0.1.0" dependencies = [ "asap-accuracy", "asap-frontend-common", - "asap-logical-asap-optimization", "asap-planner", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", "asap-types", "promql-parser", ] @@ -384,9 +387,10 @@ version = "0.1.0" dependencies = [ "asap-frontend-common", "asap-frontend-sql", - "asap-logical-asap-optimization", "asap-planner", "asap-sql-function-catalog", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", "asap-types", "datafusion", "serde", @@ -405,10 +409,11 @@ dependencies = [ "asap-executor", "asap-frontend-promql", "asap-frontend-sql", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", - "asap-plan-selection", "asap-planner", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", + "asap-stage2-physical-optimization", + "asap-stage3-plan-selection", "asap-types", "asap_sketchlib", "futures", @@ -417,67 +422,82 @@ dependencies = [ ] [[package]] -name = "asap-logical-asap-optimization" +name = "asap-planner" version = "0.1.0" dependencies = [ + "arrow", "asap-accuracy", + "asap-cost", + "asap-executor", + "asap-frontend-metricsql", "asap-frontend-promql", + "asap-frontend-sql", + "asap-stage0-memo", + "asap-stage1-pass1-per-computation-candidates", + "asap-stage1-pass2-reuse-candidates", + "asap-stage2-physical-optimization", + "asap-stage3-plan-selection", "asap-types", - "asap_sketchlib", - "rmp-serde", + "futures", "serde", "serde_json", "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "asap-sql-function-catalog" +version = "0.1.0" + +[[package]] +name = "asap-stage0-memo" +version = "0.1.0" +dependencies = [ + "asap-types", ] [[package]] -name = "asap-physical-asap-optimization" +name = "asap-stage1-pass1-per-computation-candidates" version = "0.1.0" dependencies = [ + "asap-accuracy", + "asap-frontend-promql", + "asap-stage0-memo", "asap-types", "thiserror 2.0.18", ] [[package]] -name = "asap-plan-selection" +name = "asap-stage1-pass2-reuse-candidates" version = "0.1.0" dependencies = [ "asap-accuracy", - "asap-cost", "asap-types", - "good_lp", - "highs", - "serde", - "serde_json", "thiserror 2.0.18", ] [[package]] -name = "asap-planner" +name = "asap-stage2-physical-optimization" +version = "0.1.0" +dependencies = [ + "asap-types", + "thiserror 2.0.18", +] + +[[package]] +name = "asap-stage3-plan-selection" version = "0.1.0" dependencies = [ - "arrow", "asap-accuracy", "asap-cost", - "asap-executor", - "asap-frontend-metricsql", - "asap-frontend-promql", - "asap-frontend-sql", - "asap-logical-asap-optimization", - "asap-physical-asap-optimization", - "asap-plan-selection", "asap-types", - "futures", + "good_lp", + "highs", "serde", "serde_json", "thiserror 2.0.18", - "tokio", ] -[[package]] -name = "asap-sql-function-catalog" -version = "0.1.0" - [[package]] name = "asap-types" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4204bb33c..7b29da192 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,11 +4,13 @@ members = [ "crates/types", "crates/frontend-common", "crates/sql-function-catalog", - "crates/logical-asap-optimization", - "crates/physical-asap-optimization", + "crates/stage0-memo", + "crates/stage1-pass1-per-computation-candidates", + "crates/stage1-pass2-reuse-candidates", + "crates/stage2-physical-optimization", "crates/accuracy", "crates/cost", - "crates/plan-selection", + "crates/stage3-plan-selection", "crates/frontend-promql", "crates/frontend-metricsql", "crates/metricsql-common-parser-support", diff --git a/crates/cost/Cargo.toml b/crates/cost/Cargo.toml index 8a8e35fa9..5bbdc691c 100644 --- a/crates/cost/Cargo.toml +++ b/crates/cost/Cargo.toml @@ -18,8 +18,3 @@ serde_json = "1" # The executor's charge for each summary state, checked against the bytes # the state byte models give for it plus the accumulator's own fields (#947). asap-executor = { path = "../executor" } -# Physical plans of PromQL queries for the plan cost tests (tests/), built -# by the stages above this crate; `tests/held_memory.rs` builds them with -# the reference pricer's `optimize_physical_asap`. -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } diff --git a/crates/cost/src/lib.rs b/crates/cost/src/lib.rs index 77c92aba8..8556c86e8 100644 --- a/crates/cost/src/lib.rs +++ b/crates/cost/src/lib.rs @@ -16,7 +16,7 @@ //! [`NodeUsage`]; [`plan_cost`] the whole plan's [`PlanCost`]. Stage 3 //! computes no cost of its own. //! -//! Plan selection (Stage 3, `asap-plan-selection`) documents how it prices +//! Plan selection (Stage 3, `asap-stage3-plan-selection`) documents how it prices //! a candidate with these. use std::collections::BTreeMap; diff --git a/crates/cost/src/usage.rs b/crates/cost/src/usage.rs index 8bfa7afec..50fb85dae 100644 --- a/crates/cost/src/usage.rs +++ b/crates/cost/src/usage.rs @@ -1,6 +1,6 @@ //! Costing a physical plan DAG node by node: each node's resource usage //! ([`node_usage`]), then the plan's usage, latency, retention, retained tumbling window -//! chains and cost ([`plan_cost`]). Moved from Stage 3 (`asap-plan-selection`, +//! chains and cost ([`plan_cost`]). Moved from Stage 3 (`asap-stage3-plan-selection`, //! whose module doc, "Cost model", describes the model), so Stage 3 computes //! no cost of its own. diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index bb24e585b..1e6c69df1 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -12,10 +12,12 @@ edition = "2021" asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-accuracy = { path = "../accuracy" } -asap-logical-asap-optimization = { path = "../logical-asap-optimization" } -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates" } +asap-stage1-pass2-reuse-candidates = { path = "../stage1-pass2-reuse-candidates" } +asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection" } +asap-stage3-plan-selection = { path = "../stage3-plan-selection" } # The installable plan of the selected candidate (`installable_plan_of`), # whose stored outputs `asap_planner` writes. asap-planner = { path = "../planner" } @@ -34,9 +36,9 @@ tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } # Pass 1's target list (`per_computation_candidates`) for the tests. -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } [[test]] diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 6e3fa25bb..56c88886d 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -102,8 +102,8 @@ use asap_devtools::planning_stages::{ use asap_devtools::planning_stages_text::planning_stages_text; use asap_devtools::SqlCatalog; use asap_frontend_promql::{metric_type_warnings, promql_metric_names}; -use asap_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_planner::{installable_plan_of, PlanningModels, SelectionError}; +use asap_stage3_plan_selection::{large_peak_memory_warning, AcceptedAccuracy, Selection}; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::flat::{flatten, FlatDag, FlatNode}; use asap_types::ir::physical_export::PhysicalASAPDAG; diff --git a/crates/devtools/src/bin/sketch_coverage.rs b/crates/devtools/src/bin/sketch_coverage.rs index 4af160d1e..4438a9b71 100644 --- a/crates/devtools/src/bin/sketch_coverage.rs +++ b/crates/devtools/src/bin/sketch_coverage.rs @@ -15,8 +15,8 @@ use asap_devtools::corpora::{ }; use asap_devtools::{lower_promql_with_sample_interval, SqlCatalog}; use asap_frontend_sql::lower_sql_dialect; -use asap_logical_asap_optimization::memo_of_queries; -use asap_logical_asap_optimization::pass1_memo; +use asap_stage0_memo::memo_of_queries; +use asap_stage1_pass1_per_computation_candidates::pass1_memo; use asap_types::ir::schema::FieldDataType; use asap_types::ir::{ASAPOp, Operator, OperatorNode, QueryRoot}; use asap_types::workload::SqlDialect; diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index dbf051f9a..21fea4c96 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -3,15 +3,16 @@ //! [`run_stages_until`] for `asap_planner`. use asap_frontend_sql::SqlCatalog; -use asap_logical_asap_optimization::{ - memo_of_queries, pass1_memo, pass2_after_local_window_composition, - pass2_local_window_composition, Pass1Error, -}; -use asap_physical_asap_optimization::generate_physical_asap_memo_candidates; -use asap_plan_selection::select_plan; use asap_planner::{ run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError, }; +use asap_stage0_memo::memo_of_queries; +use asap_stage1_pass1_per_computation_candidates::{pass1_memo, Pass1Error}; +use asap_stage1_pass2_reuse_candidates::{ + pass2_after_local_window_composition, pass2_local_window_composition, +}; +use asap_stage2_physical_optimization::generate_physical_asap_memo_candidates; +use asap_stage3_plan_selection::select_plan; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index 1b5c97ffd..ece3eee51 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -322,8 +322,8 @@ fn wire_name(value: T) -> String { /// `RejectionCause::failed_a_check` does. #[test] fn viewer_knows_every_view_refresh_and_rejection_cause() { - use asap_plan_selection::RejectionCause as C; use asap_planner::MaterializedViewRefresh as R; + use asap_stage3_plan_selection::RejectionCause as C; let refreshes = [R::AtIngestion, R::OnFirstRead]; for refresh in refreshes { // A new variant fails to compile here until it is listed above. diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index 231b6ed1d..7661ad112 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -27,7 +27,8 @@ test-support = ["planner-types/test-support"] [dev-dependencies] asap-executor = { path = ".", features = ["test-support"] } rmp-serde = "1" -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } # The planning stages through the facade (a dev-dependency cycle, for # integration tests only). asap-planner = { path = "../planner" } diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index f53900a4b..e6e39014c 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -335,11 +335,7 @@ fn population_fixture_compiles_with_dynamic_identity() { let topk = planner_types::ir::operator::maintained_population::PopulationStatistic::TopK { k: 1 }; let root = Rc::new(with_series_identity(&original).unwrap()); - let selected = asap_logical_asap_optimization::test_support::population_over_selector( - &root, - topk, - &["job"], - ); + let selected = planner_types::test_support::population_over_selector(&root, topk, &["job"]); let logical = planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap(); let (source, raw_schema) = logical .nodes diff --git a/crates/executor/tests/deployment_computation.rs b/crates/executor/tests/deployment_computation.rs index 080454c79..9fe132ffd 100644 --- a/crates/executor/tests/deployment_computation.rs +++ b/crates/executor/tests/deployment_computation.rs @@ -60,9 +60,8 @@ fn population_dag( grouping: &[&str], ) -> PhysicalASAPDAG { let root = promql_rows::with_series_identity(&lower(query)).unwrap(); - let selected = asap_logical_asap_optimization::test_support::population_over_selector( - &root, statistic, grouping, - ); + let selected = + planner_types::test_support::population_over_selector(&root, statistic, grouping); planner_types::test_support::physical_asap_dag_at_query_time(&selected).unwrap() } diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index b3dc5c0c3..3bc2f98c5 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -648,7 +648,7 @@ fn compacted_kll_tumbling_windows_merge_within_the_calibrated_rank_error() { const ROWS_PER_TUMBLING_WINDOW: u64 = 1000; const K: u32 = 200; // The planner's empirical 99% KLL rank error, `2.296 / k^0.9723` - // (`asap_logical_asap_optimization`'s `kll_rank_error_99`). + // (`asap_accuracy`'s `kll_rank_error_99`). let bound = 2.296 / f64::from(K).powf(0.9723); let n = (TUMBLING_WINDOW_COUNT * ROWS_PER_TUMBLING_WINDOW) as f64; let input = schema(&[("v", DataType::Float64, false)]); diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index ffd5b7e0d..cfa2dcd4c 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match +//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_stage0_memo`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/frontend-metricsql/Cargo.toml b/crates/frontend-metricsql/Cargo.toml index 719d49a5e..caa418db2 100644 --- a/crates/frontend-metricsql/Cargo.toml +++ b/crates/frontend-metricsql/Cargo.toml @@ -24,4 +24,5 @@ asap-types = { path = "../types", features = ["test-support"] } asap-frontend-metricsql = { path = ".", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } # `support.rs` (shared with frontend-promql's tests) binds through Pass 1. -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index 541f1f97f..194c97579 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -19,8 +19,9 @@ promql-parser = { git = "https://github.com/ProjectASAP/promql-parser", rev = "1 [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -# Pass 1's target list (`per_computation_candidates`) for the tests. -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +# Stage 0's memo and Pass 1's target list (`per_computation_candidates`) for the tests. +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } # The planning stages through the facade (a dev-dependency cycle, for # integration tests only). asap-planner = { path = "../planner" } diff --git a/crates/frontend-sql/Cargo.toml b/crates/frontend-sql/Cargo.toml index d210f0a0e..df4998a97 100644 --- a/crates/frontend-sql/Cargo.toml +++ b/crates/frontend-sql/Cargo.toml @@ -36,8 +36,9 @@ test-support = [] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-frontend-sql = { path = ".", features = ["test-support"] } -# Pass 1's target list (`per_computation_candidates`) for the tests. -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +# Stage 0's memo and Pass 1's target list (`per_computation_candidates`) for the tests. +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } # The planning stages through the facade (a dev-dependency cycle, for # integration tests only). asap-planner = { path = "../planner" } diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index bd8261de3..4111c4770 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -107,11 +107,11 @@ async fn frequency_empty_input_guard_uses_an_exact_count() { /// recognized L2 intent. #[tokio::test] async fn stage1_offers_exact_and_summary_l2_candidates() { - use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; + use asap_stage1_pass1_per_computation_candidates::test_support::generate_per_computation_candidates; use asap_types::ir::QueryRoot; let root = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); let inventory = generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( + &asap_stage0_memo::memo_of_queries( &[QueryRoot::Operator(root)], &[None], &Default::default(), diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index c8ee40b02..dbe28ea89 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -9,11 +9,12 @@ autotests = false asap-types = { path = "../types", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } +asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection" } +asap-stage3-plan-selection = { path = "../stage3-plan-selection" } # The fixtures plan through the facade and the developer tools' reference # enumeration. asap-planner = { path = "../planner" } @@ -22,7 +23,7 @@ asap-devtools = { path = "../devtools" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-devtools = { path = "../devtools" } # The tests plan PromQL and SQL workloads through the facade. asap-planner = { path = "../planner" } diff --git a/crates/integration-tests/tests/crate_graph.rs b/crates/integration-tests/tests/crate_graph.rs index 2eca8fe81..1efd73395 100644 --- a/crates/integration-tests/tests/crate_graph.rs +++ b/crates/integration-tests/tests/crate_graph.rs @@ -5,7 +5,8 @@ //! build-dependencies are not checked. Each crate has a role ([`role_of`]), and //! [`rule_violated`] says which role may depend on which: //! -//! - **stage crates** (the frontends, logical, physical, plan-selection) never +//! - **stage crates** (the frontends, Stage 0's memo, Stage 1's Pass 1 and +//! Pass 2, Stage 2's physical optimization, Stage 3's plan selection) never //! depend on each other; they may depend on the shared crates; //! - **asap-accuracy** and **asap-cost** depend only on asap-types; //! - **asap-types** depends on no workspace crate; @@ -31,8 +32,8 @@ enum Role { SharedModel, /// Helper crates of the frontends that are no stage themselves. FrontendHelper, - /// A stage crate: a frontend (Stage 0), logical (Stage 1), physical - /// (Stage 2) or plan-selection (Stage 3). + /// A stage crate: a frontend, Stage 0 (memo), Stage 1 (Pass 1 or Pass 2), + /// Stage 2 (physical optimization) or Stage 3 (plan selection). Stage, /// `asap-planner`: orchestrates the stages and owns the public API. Facade, @@ -53,9 +54,11 @@ fn role_of(name: &str) -> Option { "asap-frontend-promql" | "asap-frontend-sql" | "asap-frontend-metricsql" - | "asap-logical-asap-optimization" - | "asap-physical-asap-optimization" - | "asap-plan-selection" => Role::Stage, + | "asap-stage0-memo" + | "asap-stage1-pass1-per-computation-candidates" + | "asap-stage1-pass2-reuse-candidates" + | "asap-stage2-physical-optimization" + | "asap-stage3-plan-selection" => Role::Stage, "asap-planner" => Role::Facade, "asap-executor" => Role::Runtime, "asap-devtools" | "asap-integration-tests" => Role::Unrestricted, @@ -97,12 +100,12 @@ struct Exception { /// The forbidden edges the workspace keeps today. const EXCEPTIONS: &[Exception] = &[ - // Logical's `test-support` feature lowers PromQL test queries for other + // Pass 1's `test-support` feature lowers PromQL test queries for other // crates' tests. The dependency is optional and enabled only by that - // feature, so no production build links Stage 0 into Stage 1. Made + // feature, so no production build links a frontend into Stage 1. Made // non-optional, the edge fails the test. Exception { - from: "asap-logical-asap-optimization", + from: "asap-stage1-pass1-per-computation-candidates", to: "asap-frontend-promql", only_if_optional: true, }, @@ -209,19 +212,21 @@ fn checker_rejects_an_edge_between_stage_crates() { let metadata = serde_json::json!({"packages": [ package("asap-types", &[], &[]), package("asap-cost", &[("asap-types", false)], &[]), - package("asap-physical-asap-optimization", &[("asap-types", false)], &[]), + package("asap-stage2-physical-optimization", &[("asap-types", false)], &[]), // Forbidden: Stage 3 on Stage 2. package( - "asap-plan-selection", - &[("asap-cost", false), ("asap-physical-asap-optimization", false)], + "asap-stage3-plan-selection", + &[("asap-cost", false), ("asap-stage2-physical-optimization", false)], &[], ), ]}); assert_eq!( crate_graph_violations(&metadata), - vec!["forbidden [dependencies] edge asap-plan-selection -> \ - asap-physical-asap-optimization: stage crates never depend on each other (#1054)" - .to_string()] + vec![ + "forbidden [dependencies] edge asap-stage3-plan-selection -> \ + asap-stage2-physical-optimization: stage crates never depend on each other (#1054)" + .to_string() + ] ); } @@ -229,11 +234,11 @@ fn checker_rejects_an_edge_between_stage_crates() { fn checker_ignores_dev_dependencies_and_external_crates() { let metadata = serde_json::json!({"packages": [ package("asap-types", &[("serde", false)], &[]), - package("asap-logical-asap-optimization", &[("asap-types", false)], &[]), + package("asap-stage1-pass1-per-computation-candidates", &[("asap-types", false)], &[]), package( - "asap-plan-selection", + "asap-stage3-plan-selection", &[("asap-types", false)], - &["asap-logical-asap-optimization"], + &["asap-stage1-pass1-per-computation-candidates"], ), ]}); assert!(crate_graph_violations(&metadata).is_empty()); @@ -246,7 +251,7 @@ fn optional_only_exception_rejects_the_edge_made_non_optional() { package("asap-types", &[], &[]), package("asap-frontend-promql", &[("asap-types", false)], &[]), package( - "asap-logical-asap-optimization", + "asap-stage1-pass1-per-computation-candidates", &[("asap-types", false), ("asap-frontend-promql", optional)], &[], ), @@ -256,7 +261,7 @@ fn optional_only_exception_rejects_the_edge_made_non_optional() { assert_eq!( crate_graph_violations(&metadata(false)), vec![ - "forbidden [dependencies] edge asap-logical-asap-optimization -> \ + "forbidden [dependencies] edge asap-stage1-pass1-per-computation-candidates -> \ asap-frontend-promql: stage crates never depend on each other (#1054)" .to_string() ] @@ -267,14 +272,14 @@ fn optional_only_exception_rejects_the_edge_made_non_optional() { fn checker_rejects_an_unclassified_crate_and_shared_model_edges() { let metadata = serde_json::json!({"packages": [ package("asap-types", &[], &[]), - package("asap-logical-asap-optimization", &[("asap-types", false)], &[]), - package("asap-accuracy", &[("asap-logical-asap-optimization", false)], &[]), + package("asap-stage1-pass1-per-computation-candidates", &[("asap-types", false)], &[]), + package("asap-accuracy", &[("asap-stage1-pass1-per-computation-candidates", false)], &[]), package("asap-new-crate", &[], &[]), ]}); assert_eq!( crate_graph_violations(&metadata), vec![ - "forbidden [dependencies] edge asap-accuracy -> asap-logical-asap-optimization: \ + "forbidden [dependencies] edge asap-accuracy -> asap-stage1-pass1-per-computation-candidates: \ asap-accuracy and asap-cost depend only on asap-types (#1054)" .to_string(), "workspace crate `asap-new-crate` has no role: add it to `role_of` in \ diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index d6192608f..4a5a3c6b3 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -3,7 +3,7 @@ //! ranks. use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_logical_asap_optimization::test_support::generate_per_computation_candidates; +use asap_stage1_pass1_per_computation_candidates::test_support::generate_per_computation_candidates; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; @@ -21,7 +21,7 @@ async fn targets(sql: &str) -> Vec { .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); generate_per_computation_candidates( - &asap_logical_asap_optimization::memo_of_queries( + &asap_stage0_memo::memo_of_queries( &[QueryRoot::Operator(root)], &[None], &Default::default(), diff --git a/crates/logical-asap-optimization/src/lib.rs b/crates/logical-asap-optimization/src/lib.rs deleted file mode 100644 index 38baf4895..000000000 --- a/crates/logical-asap-optimization/src/lib.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! `asap-logical-asap-optimization` — #509 Stage 0 hash-consing and -//! Stage 1, logical ASAP-aware optimization: from the query-language -//! frontends' logical DAGs to the memo of logical candidates -//! (`asap_types::stages::LogicalASAPMemoCandidates`) that Stage 2 implements. -//! -//! Identical-sub-DAG sharing over the logical DAG (Stage 0, the `stage0` -//! module, behind [`memo_of_queries`]) runs before the -//! stages; Stage 1 adds the ASAP-aware sharing that is invisible at the -//! logical level (Pass 2, the `asap_aware_cse` module), e.g. -//! `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading one built sketch. -//! -//! It depends only on `asap-types` and the shared accuracy model -//! (`asap-accuracy`). It depends on no other stage crate, and no other -//! stage crate depends on it except in their tests (CI checks the crate -//! graph): the stage-boundary types (its input -//! `Memo`, Pass 1's `Memo` and Pass 2's `LogicalASAPMemoCandidates`) are in `asap_types::stages`, and -//! the planner facade calls each stage in turn. It depends on no -//! query-language frontend either, except with the `test-support` feature, -//! whose test helpers (`test_support`) lower PromQL test queries through -//! `asap-frontend-promql`; other crates enable that feature only for their -//! tests. -//! -//! Post-lowering **canonicalization** is *not* here: it lives in -//! `asap_frontend_common::canonicalize`, run inside that crate's shared -//! `resolve_root` so every front end normalizes before the IR leaves resolution -//! (issue #34, closed). -//! -//! ## Components -//! -//! The modules are crate-private; the crate root re-exports the items other -//! crates use, one path each, and `test_support` (feature `test-support`) -//! re-exports the internals other crates' tests call. -//! -//! - `stage0` — Stage 0: hash-consing the workload's logical DAG into memo -//! groups through `Memo::find_or_add_group`, behind [`memo_of_queries`]. -//! - `pass1::per_computation_candidates` — Pass 1: per-computation candidate -//! generation and composition, behind -//! [`pass1_memo`]. -//! - `pass2::memo` — Pass 2: the rules over groups, behind -//! [`generate_logical_asap_memo_candidates`]. -//! - `pass1::realization` — the summary families that can serve an aggregate -//! intent, the intent's accuracy target and `(ε, δ)` budget, and default -//! sketch sizing, shared by Pass 1 and Pass 2. -//! -//! Accuracy guarantees come from the built-in analytical accuracy rules in -//! `asap-accuracy` (per-family estimators, composition and budget -//! allocation). They derive a machine-readable -//! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) -//! for every finalized value and propagate it through -//! approximate-over-approximate compositions under conservative rules (no -//! independence assumptions, unknown statistics stay unknown). A -//! composition with no rule leaves the guarantee `None` and keeps the -//! candidate. Pass 1 prunes nothing for accuracy; Pass 2 gives each readout -//! its required accuracy and eliminates the candidates no query can use, -//! with the deployment's measured accuracy (`AccuracyEvidence`) read before -//! each summary's theoretical guarantee. - -mod pass1; -mod pass2; -/// Stage 0: the memo groups of the workload's query roots. -mod stage0; -/// Test helpers shared by the stage crates' tests, and the Stage 1 -/// internals those tests call (feature `test-support`). -#[cfg(any(test, feature = "test-support"))] -pub mod test_support; - -// Stage 1's public API: one path per item, for the planner facade and the -// developer tools. Every other item is crate-private; the ones other crates' -// tests call are re-exported by `test_support`. - -// Stage 0: the memo groups of the workload's query roots, without -// candidates (`asap_types::stages::Memo`). -pub use stage0::memo_of_queries; -// Pass 1: its entry and its errors. Its output and the types the output -// holds are in `asap_types::stages`. -pub use pass1::{pass1_memo, Pass1Error, PerComputationCandidateError}; -// Pass 2: its entry and its error. -pub use pass2::logical_memo_candidates::{ - generate_logical_asap_memo_candidates, pass2_after_local_window_composition, - pass2_local_window_composition, -}; -pub use pass2::memo::rules::Pass2RuleError; diff --git a/crates/logical-asap-optimization/src/pass1.rs b/crates/logical-asap-optimization/src/pass1.rs deleted file mode 100644 index 8aac868d0..000000000 --- a/crates/logical-asap-optimization/src/pass1.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Stage 1, Pass 1: per-computation candidate generation and the summary -//! realizations it offers. Its one entry is -//! [`pass1_memo`], whose output -//! memo Pass 2 extends. - -#[cfg(any(test, feature = "test-support"))] -/// Stage 1, Pass 1's entry: the initial candidates of the memo groups. -pub(crate) mod logical_memo_candidates; -/// Stage 1, Pass 1's computation targets and their candidates. No execution timing is assigned: that is a Stage 2 -/// materialization decision. Its target list -/// (`PerComputationCandidates`) is Pass 1's working form, public only to -/// tests (`crate::test_support`, feature `test-support`). -pub(crate) mod per_computation_candidates; -/// Frontend-to-Pass-1 acceptance tests over Pass 1's working form. -#[cfg(test)] -mod per_computation_candidates_tests; -pub(crate) mod realization; - -pub use logical_memo_candidates::{pass1_memo, Pass1Error}; -pub use per_computation_candidates::PerComputationCandidateError; diff --git a/crates/logical-asap-optimization/src/pass2.rs b/crates/logical-asap-optimization/src/pass2.rs deleted file mode 100644 index dbdc6f273..000000000 --- a/crates/logical-asap-optimization/src/pass2.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Stage 1, Pass 2: the rules over groups and the logical edges. - -/// Stage 1, Pass 2's entry, producing [`asap_types::stages::LogicalASAPMemoCandidates`]. -pub(crate) mod logical_memo_candidates; -/// The rules over groups. -pub(crate) mod memo; diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs deleted file mode 100644 index 5f1d5673e..000000000 --- a/crates/logical-asap-optimization/src/test_support.rs +++ /dev/null @@ -1,271 +0,0 @@ -// Shared fixture helpers for the stage crates' tests (feature -// `test-support`); not every test module uses every helper. -#![allow(dead_code)] - -use std::rc::Rc; - -// Stage 1 internals other crates' tests call: Pass 1's working form. -pub use crate::pass1::per_computation_candidates::{ - generate_per_computation_candidates, PerComputationCandidates, -}; - -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::OperatorNode; -use asap_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; - -/// The statistic classic HLL's failure probability at `precision` and -/// `relative_error` over `source` is left as, for Stage 3 tests that bind -/// it. The estimator keeps the format crate-private. -pub fn hll_classic_failure_statistic(precision: u8, relative_error: f64, source: &str) -> String { - asap_accuracy::estimators::hll::classic_failure_statistic(precision, relative_error, source) -} - -pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .remove(0) - .as_operator() - .cloned() - .expect("operator root") -} - -// ── Shared pre-ASAP fixture builders ───────────────────────────────────── -// -// Every builder returns an `Rc` whose schema is derived by -// `OperatorNode::new_shared`, so a fixture is exactly what a front end -// would hand the planner. Added by the test migration; only add here, never -// rename or remove (several test modules share these). - -use std::time::Duration; - -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; - -/// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, -/// no unique key. -pub fn metric_scan(labels: &[&str]) -> Rc { - metric_scan_with_keys(labels, vec![]) -} - -/// [`metric_scan`] with explicit `unique_keys` (a `[[0]]` key makes CSE -/// willing to hoist the scan). -pub fn metric_scan_with_keys(labels: &[&str], unique_keys: Vec>) -> Rc { - let mut columns = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ]; - columns.extend( - labels - .iter() - .map(|n| Field::plain(*n, DataType::Utf8, true)), - ); - scan("m", Schema::with_time_index(columns, 0, unique_keys)) -} - -/// A predicate-free `TimeSeries(metric)` scan with the given schema. -pub fn scan(metric: &str, schema: Schema) -> Rc { - scan_from( - Source::TimeSeries { - metric: metric.into(), - }, - schema, - ) -} - -pub fn scan_from(source: Source, schema: Schema) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source, - predicates: vec![], - schema, - })) - .unwrap() -} - -/// A general aggregate node. -pub fn aggregate( - reduction: Reduction, - measures: Vec, - output_names: Vec, - having: Option, - child: Rc, -) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction, - measures, - output_names, - filters: vec![], - having, - child, - })) - .unwrap() -} - -/// `intent by (by)` — a single-measure, `HAVING`-free grouped aggregate. -pub fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { - aggregate(Reduction::by(by), vec![intent], vec![], None, child) -} - -/// `intent without (excluded)`. -pub fn without_agg( - excluded: Vec, - intent: AggIntent, - child: Rc, -) -> Rc { - aggregate( - Reduction::Reduce(GroupKeys::without(excluded)), - vec![intent], - vec![], - None, - child, - ) -} - -pub fn filter(pred: ScalarExpr, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(pred), - child, - })) - .unwrap() -} - -pub fn dedup(cols: Vec, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Dedup { - cols, - child, - })) - .unwrap() -} - -/// An explicit range selector `child[range]`. -pub fn time_range(range: Duration, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { - range, - kind: TimeRangeKind::Range, - child, - })) - .unwrap() -} - -/// The workload entry of a query evaluated as `recurrence` says, exact and -/// with no latency bound. -pub fn entry( - recurrence: asap_types::workload::QueryRecurrence, -) -> asap_types::workload::QueryWorkloadEntry { - entry_with(AccuracyTarget::exact(), recurrence) -} - -/// The workload entry of a query with accuracy target `accuracy`, evaluated -/// as `recurrence` says, with no latency bound. -pub fn entry_with( - accuracy: AccuracyTarget, - recurrence: asap_types::workload::QueryRecurrence, -) -> asap_types::workload::QueryWorkloadEntry { - asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: QueryRequirements { - accuracy, - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence, - range_evaluation: None, - } -} - -/// The maintained-population sub-DAG for executor tests: `root` (below its -/// projections, which are kept) replaced by `EvaluatePopulation(statistic)` -/// over a current-series population of the single selector under it, -/// grouped by `grouping`, with no matchers. The caller states the statistic -/// and grouping; nothing is recognized. No planner pass offers this sub-DAG -/// yet (#681). -pub fn population_over_selector( - root: &Rc, - statistic: asap_types::ir::operator::maintained_population::PopulationStatistic, - grouping: &[&str], -) -> Rc { - use asap_types::ir::operator::maintained_population::*; - use asap_types::ir::{ASAPOp, Operator}; - if let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - { - let child = population_over_selector(child, statistic.clone(), grouping); - let project = Operator::NonASAP(NonASAPOp::Project { - cols: cols.clone(), - qualifier: qualifier.clone(), - child, - }); - return Rc::new(OperatorNode::with_schema(project, root.schema.clone())); - } - let selectors: Vec<_> = OperatorNode::reachable(root) - .into_iter() - .filter(|node| matches!(node.non_asap(), Some(NonASAPOp::TimeRange { .. }))) - .collect(); - let [selector] = selectors.as_slice() else { - panic!("one selector under {root:?}"); - }; - let Some(NonASAPOp::TimeRange { range, child, .. }) = selector.non_asap() else { - unreachable!() - }; - let Some(NonASAPOp::Scan { - source: Source::TimeSeries { metric }, - .. - }) = child.non_asap() - else { - panic!("a time-series selector, got {child:?}"); - }; - let population = MaintainedPopulation { - input: PopulationInput::CurrentSeries(CurrentSeriesInput { - metric: metric.clone(), - matchers: Vec::new(), - grouping: grouping.iter().map(|label| label.to_string()).collect(), - without: false, - lookback_ms: range.as_millis() as u64, - }), - max_k: match statistic { - PopulationStatistic::TopK { k } => k, - _ => 0, - }, - quantiles: matches!(statistic, PopulationStatistic::Quantile { .. }), - }; - let maintained = OperatorNode::with_schema( - Operator::ASAP(ASAPOp::MaintainPopulation { - child: selector.clone(), - population, - }), - selector.schema.clone(), - ); - let evaluation = Operator::ASAP(ASAPOp::EvaluatePopulation { - child: Rc::new(maintained), - evaluation: statistic, - }); - Rc::new(OperatorNode::with_schema(evaluation, root.schema.clone())) -} diff --git a/crates/logical-asap-optimization/tests/pass_layering.rs b/crates/logical-asap-optimization/tests/pass_layering.rs deleted file mode 100644 index daf0595a7..000000000 --- a/crates/logical-asap-optimization/tests/pass_layering.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! Stage 1's passes are independent modules (#1096, #1191): neither pass's -//! production code imports the other. Test modules may use the other pass to -//! build fixtures. IR helpers both passes need live in `asap-types`. - -use std::path::{Path, PathBuf}; - -/// Production lines under `src/.rs` and `src//` that name a path -/// through `needle`'s module. -fn production_lines_mentioning(pass: &str, needle: &str) -> Vec { - fn walk(dir: &Path, files: &mut Vec) { - for entry in std::fs::read_dir(dir).unwrap() { - let path = entry.unwrap().path(); - if path.is_dir() { - walk(&path, files); - // `*_tests.rs` files are `#[cfg(test)]` modules. - } else if !path.to_string_lossy().ends_with("_tests.rs") { - files.push(path); - } - } - } - let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); - let mut files = vec![src.join(format!("{pass}.rs"))]; - walk(&src.join(pass), &mut files); - let mut offending = Vec::new(); - for file in files { - let text = std::fs::read_to_string(&file).unwrap(); - // ponytail: a file's production code ends at its `#[cfg(test)] mod tests`; - // parse with `syn` if inline `#[cfg(test)]` items ever need excluding. - let production = text.split("#[cfg(test)]\nmod tests").next().unwrap(); - for (n, line) in production.lines().enumerate() { - if line.contains(needle) { - offending.push(format!("{}:{}: {}", file.display(), n + 1, line.trim())); - } - } - } - offending -} - -/// Pass 1's production source does not reference `pass2`. -#[test] -fn pass1_does_not_import_pass2() { - let offending = production_lines_mentioning("pass1", "pass2::"); - assert!( - offending.is_empty(), - "Pass 1 imports Pass 2:\n{}", - offending.join("\n") - ); -} - -/// Pass 2's production source does not reference `pass1`. -#[test] -fn pass2_does_not_import_pass1() { - let offending = production_lines_mentioning("pass2", "pass1::"); - assert!( - offending.is_empty(), - "Pass 2 imports Pass 1:\n{}", - offending.join("\n") - ); -} diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 6cf31bc74..8a79dbcb9 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -8,21 +8,24 @@ edition = "2021" # returns the selected DAG. It is the only crate that depends on every # frontend — before it, the sole facade re-exporting more than one was `asap-devtools`, # a developer-tools crate. Everything below the frontend boundary lives in the -# stage crates (`asap-logical-asap-optimization`, `asap-physical-asap-optimization`, -# `asap-plan-selection`), so a third party writing an optimization pass (issue -# #430) depends on `asap-plan-selection` (plus `asap-accuracy` to supply its own +# stage crates (`asap-stage0-memo`, `asap-stage1-pass1-per-computation-candidates`, +# `asap-stage1-pass2-reuse-candidates`, `asap-stage2-physical-optimization`, +# `asap-stage3-plan-selection`), so a third party writing an optimization pass (issue +# #430) depends on `asap-stage3-plan-selection` (plus `asap-accuracy` to supply its own # accuracy model or evidence) and never pulls in DataFusion. [dependencies] asap-types = { path = "../types" } asap-cost = { path = "../cost" } -asap-plan-selection = { path = "../plan-selection" } +asap-stage3-plan-selection = { path = "../stage3-plan-selection" } asap-frontend-sql = { path = "../frontend-sql" } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-metricsql = { path = "../frontend-metricsql" } -asap-logical-asap-optimization = { path = "../logical-asap-optimization" } +asap-stage0-memo = { path = "../stage0-memo" } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates" } +asap-stage1-pass2-reuse-candidates = { path = "../stage1-pass2-reuse-candidates" } # The facade runs every stage: Stage 2's catalog and realizer, Stage 3's # `select_plan`, with the deployment's accuracy model. -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } +asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-accuracy = { path = "../accuracy" } # The Planner's own executor: `plan_installable` offers only the summary # families it runs when the caller leaves them unrestricted. @@ -32,10 +35,8 @@ serde_json = "1" thiserror = "2" [dev-dependencies] -asap-plan-selection = { path = "../plan-selection" } -asap-physical-asap-optimization = { path = "../physical-asap-optimization" } -# Pass 1's target list (`per_computation_candidates`) for the tests. -asap-logical-asap-optimization = { path = "../logical-asap-optimization", features = ["test-support"] } +asap-stage3-plan-selection = { path = "../stage3-plan-selection" } +asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-accuracy = { path = "../accuracy", features = ["test-support"] } # Tests read a plan's roots as IR (`PhysicalASAPDAG::query_roots`). asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index da85ad88e..cdd114fb9 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -8,7 +8,7 @@ //! The default pass, [`Stage1ToStage3Pass`], orchestrates the stages //! ([`run_planning`]): Stage 0, Pass 1, Pass 2, Stage 2 //! (`generate_physical_asap_memo_candidates`) and Stage 3 -//! (`asap_plan_selection::select_plan`), which builds the selected plan. +//! (`asap_stage3_plan_selection::select_plan`), which builds the selected plan. //! The stages never call each other. //! //! [`e2e_plan`] runs both steps. [`plan_installable`] runs them for a @@ -51,7 +51,9 @@ pub use asap_cost::{ ResourceSpeed, ResourceUnitCost, SummaryOperation, }; pub use asap_frontend_sql::SqlCatalog; -pub use asap_plan_selection::{AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection}; +pub use asap_stage3_plan_selection::{ + AcceptedAccuracy, ErrorSelectingPlan, RejectionCause, Selection, +}; pub use asap_types::deployment_capabilities::{DeploymentCapabilities, SupportedSummary}; pub use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 70608a2cf..0a7ae008f 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -21,7 +21,7 @@ use asap_types::workload::WorkloadError; use crate::run::{SelectionError, Stage1ToStage3Pass}; use asap_accuracy::{AccuracyEvidence, NoMeasuredGuarantees}; use asap_cost::CostInputs; -use asap_logical_asap_optimization::PerComputationCandidateError; +use asap_stage1_pass1_per_computation_candidates::PerComputationCandidateError; use asap_types::deployment_capabilities::DeploymentCapabilities; use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPDAGValidationError}; diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index c181bed92..337cb032c 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -3,15 +3,17 @@ //! builds the selected plan. No stage calls another, and none has a //! deadline: each runs to completion. -use asap_logical_asap_optimization::memo_of_queries; +use asap_stage0_memo::memo_of_queries; use std::rc::Rc; use std::time::Instant; -use asap_logical_asap_optimization::generate_logical_asap_memo_candidates; -use asap_logical_asap_optimization::{pass1_memo, Pass1Error, PerComputationCandidateError}; -use asap_physical_asap_optimization::generate_physical_asap_memo_candidates; -use asap_plan_selection::{select_plan, ErrorSelectingPlan, Selection}; +use asap_stage1_pass1_per_computation_candidates::{ + pass1_memo, Pass1Error, PerComputationCandidateError, +}; +use asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates; +use asap_stage2_physical_optimization::generate_physical_asap_memo_candidates; +use asap_stage3_plan_selection::{select_plan, ErrorSelectingPlan, Selection}; use asap_types::deployment_capabilities::DeploymentCapabilitiesError; use asap_types::ir::physical_export::PhysicalASAPDAG; use asap_types::ir::schema::schema_support::with_promql_series_identity; diff --git a/crates/stage0-memo/Cargo.toml b/crates/stage0-memo/Cargo.toml new file mode 100644 index 000000000..40804bc41 --- /dev/null +++ b/crates/stage0-memo/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "asap-stage0-memo" +version = "0.1.0" +edition = "2021" + +# #509 Stage 0: hash-consing the workload's logical DAGs into the memo groups +# (`asap_types::stages::Memo`, without candidates) that Stage 1 fills. Depends +# only on asap-types; no other stage crate depends on it except in their tests. +[dependencies] +asap-types = { path = "../types" } diff --git a/crates/logical-asap-optimization/src/stage0.rs b/crates/stage0-memo/src/lib.rs similarity index 65% rename from crates/logical-asap-optimization/src/stage0.rs rename to crates/stage0-memo/src/lib.rs index 4ab97f678..79e269fc9 100644 --- a/crates/logical-asap-optimization/src/stage0.rs +++ b/crates/stage0-memo/src/lib.rs @@ -1,7 +1,12 @@ -//! Stage 0: the memo groups of the workload's query roots (a [`Memo`] -//! without candidates). Each node of the queries' DAGs becomes the group of -//! its expression, found or added by [`Memo::find_or_add_group`], so -//! identical sub-expressions are one group. Stage 1 starts from these groups. +//! `asap-stage0-memo` — #509 Stage 0: the memo groups of the workload's +//! query roots (a [`Memo`] without candidates). Each node of the queries' +//! DAGs becomes the group of its expression, found or added by +//! [`Memo::find_or_add_group`], so identical sub-expressions are one group. +//! Stage 1 starts from these groups. +//! +//! It depends only on `asap-types`, where its output (`Memo`) lives; no +//! other stage crate depends on it except in their tests, and the planner +//! facade calls it first. use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{QueryRoot, SchemaDerivationError}; diff --git a/crates/logical-asap-optimization/Cargo.toml b/crates/stage1-pass1-per-computation-candidates/Cargo.toml similarity index 51% rename from crates/logical-asap-optimization/Cargo.toml rename to crates/stage1-pass1-per-computation-candidates/Cargo.toml index 3154d7233..9cd188ba4 100644 --- a/crates/logical-asap-optimization/Cargo.toml +++ b/crates/stage1-pass1-per-computation-candidates/Cargo.toml @@ -1,29 +1,27 @@ [package] -name = "asap-logical-asap-optimization" +name = "asap-stage1-pass1-per-computation-candidates" version = "0.1.0" edition = "2021" -# #509 Stage 1, logical ASAP-aware optimization: Pass 1 per-computation -# candidates and Pass 2 ASAP-aware CSE over asap-types' operator DAG, with -# the shared analytical accuracy model (`asap-accuracy`). Depends on no -# later stage, and on no query-language frontend except through the -# `test-support` feature. +# #509 Stage 1, Pass 1: the initial candidates of each memo group (exact, +# exact accumulator, summaries, fused builds), with the shared analytical +# accuracy model (`asap-accuracy`). Reads Stage 0's memo through +# `asap_types::stages`; depends on no other stage crate, and on no +# query-language frontend except through the `test-support` feature. [dependencies] -asap_sketchlib = { workspace = true } asap-types = { path = "../types" } asap-accuracy = { path = "../accuracy" } thiserror = "2" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -rmp-serde = "1" # Only for `test-support`: lowering PromQL test queries. asap-frontend-promql = { path = "../frontend-promql", optional = true } [features] -# The test helpers (`test_support`) the later stage crates' tests share. +# The test helpers (`test_support`) other crates' tests share. test-support = ["dep:asap-frontend-promql", "asap-types/test-support"] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } +# Stage 0's memo of the test queries. +asap-stage0-memo = { path = "../stage0-memo" } diff --git a/crates/stage1-pass1-per-computation-candidates/src/lib.rs b/crates/stage1-pass1-per-computation-candidates/src/lib.rs new file mode 100644 index 000000000..ec4fb281c --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/lib.rs @@ -0,0 +1,49 @@ +//! `asap-stage1-pass1-per-computation-candidates` — #509 Stage 1, Pass 1: +//! per-computation candidate generation and the summary realizations it +//! offers. Its one entry is [`pass1_memo`], which fills the +//! memo groups of Stage 0's [`Memo`](asap_types::stages::Memo) with their +//! initial candidates; Pass 2 extends that memo. +//! +//! It depends only on `asap-types` (its input and output memo are in +//! `asap_types::stages`) and the shared accuracy model (`asap-accuracy`). +//! It depends on no other stage crate, and the planner facade calls it +//! after Stage 0. It depends on no query-language frontend either, except +//! with the `test-support` feature, whose test helpers (`test_support`) +//! lower PromQL test queries through `asap-frontend-promql`; other crates +//! enable that feature only for their tests. +//! +//! Accuracy guarantees come from the built-in analytical accuracy rules in +//! `asap-accuracy` (per-family estimators, composition and budget +//! allocation). They derive a machine-readable +//! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) +//! for every finalized value and propagate it through +//! approximate-over-approximate compositions under conservative rules (no +//! independence assumptions, unknown statistics stay unknown). A +//! composition with no rule leaves the guarantee `None` and keeps the +//! candidate. Pass 1 prunes nothing for accuracy; Pass 2 gives each readout +//! its required accuracy and eliminates the candidates no query can use, +//! with the deployment's measured accuracy (`AccuracyEvidence`) read before +//! each summary's theoretical guarantee. + +/// Stage 1, Pass 1's entry: the initial candidates of the memo groups. +mod logical_memo_candidates; +/// Stage 1, Pass 1's computation targets and their candidates. No execution timing is assigned: that is a Stage 2 +/// materialization decision. Its target list +/// (`PerComputationCandidates`) is Pass 1's working form, public only to +/// tests (`crate::test_support`, feature `test-support`). +mod per_computation_candidates; +/// Frontend-to-Pass-1 acceptance tests over Pass 1's working form. +#[cfg(test)] +mod per_computation_candidates_tests; +/// The summary families that can serve an aggregate intent, the input a +/// summary consumes and the column a top-k ranks. +mod realization; +/// Fixtures this crate's own tests share. +#[cfg(test)] +mod test_fixtures; +/// Test helpers other crates' tests call (feature `test-support`). +#[cfg(any(test, feature = "test-support"))] +pub mod test_support; + +pub use logical_memo_candidates::{pass1_memo, Pass1Error}; +pub use per_computation_candidates::PerComputationCandidateError; diff --git a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs similarity index 97% rename from crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs rename to crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs index 92a18e5b8..c1c024dfd 100644 --- a/crates/logical-asap-optimization/src/pass1/logical_memo_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs @@ -3,7 +3,7 @@ //! configuration, fused builds), with every intermediate result its own //! group (a [`Memo`]), which Pass 2 extends. -use crate::pass1::per_computation_candidates::{ +use crate::per_computation_candidates::{ generate_per_computation_candidates, PerComputationCandidateError, }; use asap_types::deployment_capabilities::SupportedSummary; diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs similarity index 99% rename from crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs rename to crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs index fe145026d..79f48793c 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs @@ -55,7 +55,7 @@ use thiserror::Error; use asap_accuracy::sizing::sketch_statistic; -use crate::pass1::realization::summary_candidates; +use crate::realization::summary_candidates; use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; use asap_types::stages::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, @@ -101,7 +101,7 @@ fn prune_own( /// Compact Pass 1 inventory over the memo groups it walked: one target per /// measure of each aggregate group, however many queries read the group. /// Pass 1's working form, not its output: -/// [`pass1_memo`](crate::pass1::pass1_memo) +/// [`pass1_memo`](crate::pass1_memo) /// builds, prunes and attaches it to the memo groups. Public only to tests /// (feature `test-support`). #[derive(Debug, Clone)] @@ -761,7 +761,7 @@ fn reads_only_time_series(node: &OperatorNode) -> bool { pub(super) fn whole_expression_input( target: &OperatorNode, ) -> Option<(Rc, SummaryUpdate, Reduction)> { - use crate::pass1::realization::realize_keyed_additive_summary_input; + use crate::realization::realize_keyed_additive_summary_input; let Some(NonASAPOp::Aggregate { child, reduction, @@ -1407,7 +1407,7 @@ mod tests { let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) .unwrap_or(root); generate_per_computation_candidates( - &crate::stage0::memo_of_queries( + &asap_stage0_memo::memo_of_queries( &[QueryRoot::Operator(root)], &[None], &Default::default(), diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs similarity index 99% rename from crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs rename to crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs index fda16d839..5985738d0 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates_tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs @@ -1,9 +1,9 @@ //! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. -use crate::pass1::per_computation_candidates::{ +use crate::per_computation_candidates::{ computation_candidates_for_intent, generate_per_computation_candidates, PerComputationCandidateError, }; -use crate::stage0::memo_of_queries; +use asap_stage0_memo::memo_of_queries; use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; @@ -304,7 +304,7 @@ fn filtered_measure_keeps_candidates_and_records_its_filter() { /// Target `target`'s own ways to compute. fn choices_of( - inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, + inventory: &crate::per_computation_candidates::PerComputationCandidates, target: usize, ) -> Vec { inventory.targets[target].ways_to_compute.clone() @@ -556,7 +556,7 @@ fn global_aggregate_having_stays_above() { /// The `HAVING` entries of the target whose measure satisfies `is`. fn having_of( - inventory: &crate::pass1::per_computation_candidates::PerComputationCandidates, + inventory: &crate::per_computation_candidates::PerComputationCandidates, is: impl Fn(&AggIntent) -> bool, ) -> Vec { let target = inventory diff --git a/crates/logical-asap-optimization/src/pass1/realization.rs b/crates/stage1-pass1-per-computation-candidates/src/realization.rs similarity index 98% rename from crates/logical-asap-optimization/src/pass1/realization.rs rename to crates/stage1-pass1-per-computation-candidates/src/realization.rs index b9e0bc393..4885bd334 100644 --- a/crates/logical-asap-optimization/src/pass1/realization.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/realization.rs @@ -1,5 +1,5 @@ -//! Summary realization helpers shared by Pass 1 (`per_computation_candidates`) -//! and Pass 2 (`asap_aware_cse`): which sketch kinds can serve an intent, +//! Summary realization helpers of Pass 1 (`per_computation_candidates`): +//! which sketch kinds can serve an intent, //! the column/tuple a summary consumes, the column a top-k ranks, and the //! keyed-additive rule for a heap sketch that reads a larger sub-DAG. @@ -295,7 +295,7 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc #[cfg(test)] mod tests { use super::*; - use crate::test_support::metric_scan; + use crate::test_fixtures::metric_scan; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; use asap_types::ir::{ExprSemantics, Predicate}; diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs new file mode 100644 index 000000000..a69c984be --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -0,0 +1,49 @@ +//! Fixtures this crate's own tests share. Every builder returns an +//! `Rc` whose schema is derived by `OperatorNode::new_shared`, +//! so a fixture is exactly what a front end would hand the planner. + +use std::rc::Rc; + +use asap_types::ir::operator::operator_properties::Source; +use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; +use asap_types::ir::{NonASAPOp, OperatorNode}; + +/// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, +/// no unique key. +pub fn metric_scan(labels: &[&str]) -> Rc { + metric_scan_with_keys(labels, vec![]) +} + +/// [`metric_scan`] with explicit `unique_keys` (a `[[0]]` key makes CSE +/// willing to hoist the scan). +pub fn metric_scan_with_keys(labels: &[&str], unique_keys: Vec>) -> Rc { + let mut columns = vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]; + columns.extend( + labels + .iter() + .map(|n| Field::plain(*n, DataType::Utf8, true)), + ); + scan("m", Schema::with_time_index(columns, 0, unique_keys)) +} + +/// A predicate-free `TimeSeries(metric)` scan with the given schema. +pub fn scan(metric: &str, schema: Schema) -> Rc { + scan_from( + Source::TimeSeries { + metric: metric.into(), + }, + schema, + ) +} + +pub fn scan_from(source: Source, schema: Schema) -> Rc { + OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { + source, + predicates: vec![], + schema, + })) + .unwrap() +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_support.rs b/crates/stage1-pass1-per-computation-candidates/src/test_support.rs new file mode 100644 index 000000000..66c497a1a --- /dev/null +++ b/crates/stage1-pass1-per-computation-candidates/src/test_support.rs @@ -0,0 +1,45 @@ +//! Test helpers: the Pass 1 internals and the PromQL lowering other crates' +//! tests call (feature `test-support`), and the fixtures this crate's own +//! tests share (`cfg(test)`). + +use std::rc::Rc; + +// Pass 1 internals other crates' tests call: Pass 1's working form. +pub use crate::per_computation_candidates::generate_per_computation_candidates; + +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::OperatorNode; +use asap_types::workload::{ + BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, +}; + +pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy, + ..Default::default() + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + range_evaluation: None, + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload::sampled_every( + asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), + DurationMs(1_000), + )), + }; + asap_frontend_promql::lower_promql_query_workload(&workload) + .unwrap() + .remove(0) + .as_operator() + .cloned() + .expect("operator root") +} diff --git a/crates/stage1-pass2-reuse-candidates/Cargo.toml b/crates/stage1-pass2-reuse-candidates/Cargo.toml new file mode 100644 index 000000000..430c48f0f --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "asap-stage1-pass2-reuse-candidates" +version = "0.1.0" +edition = "2021" + +# #509 Stage 1, Pass 2: the reuse rules over memo groups and the logical +# edges (`asap_types::stages::LogicalASAPMemoCandidates`) Stage 2 expands. +# Reads Pass 1's memo through `asap_types::stages`; depends on no other +# stage crate. +[dependencies] +asap-types = { path = "../types" } +# The deployment's measured accuracy (`AccuracyEvidence`) Pass 2 reads. +asap-accuracy = { path = "../accuracy" } +thiserror = "2" diff --git a/crates/stage1-pass2-reuse-candidates/src/lib.rs b/crates/stage1-pass2-reuse-candidates/src/lib.rs new file mode 100644 index 000000000..970b77f40 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/lib.rs @@ -0,0 +1,21 @@ +//! `asap-stage1-pass2-reuse-candidates` — #509 Stage 1, Pass 2: the reuse +//! rules over memo groups and the logical edges. Its entry, +//! [`generate_logical_asap_memo_candidates`], extends Pass 1's memo into +//! the [`LogicalASAPMemoCandidates`](asap_types::stages::LogicalASAPMemoCandidates) +//! Stage 2 expands, e.g. `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading +//! one built sketch. +//! +//! It depends only on `asap-types`, where its input and output are. It +//! depends on no other stage crate, and the planner facade calls it after +//! Pass 1. + +/// Stage 1, Pass 2's entry, producing [`asap_types::stages::LogicalASAPMemoCandidates`]. +mod logical_memo_candidates; +/// The rules over groups. +mod memo; + +pub use logical_memo_candidates::{ + generate_logical_asap_memo_candidates, pass2_after_local_window_composition, + pass2_local_window_composition, +}; +pub use memo::rules::Pass2RuleError; diff --git a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs similarity index 95% rename from crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs rename to crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs index 4da070c98..4865c0a61 100644 --- a/crates/logical-asap-optimization/src/pass2/logical_memo_candidates.rs +++ b/crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs @@ -2,10 +2,10 @@ //! rules over groups, and the logical edges //! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 expands. -use crate::pass2::memo::rules::{ +use crate::memo::rules::{ finer_aggregate, rollup, scan_labels, summary_inputs, topk, Pass2RuleError, }; -use crate::pass2::memo::window_candidates::add_local_window_composition; +use crate::memo::window_candidates::add_local_window_composition; use asap_accuracy::AccuracyEvidence; use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; use asap_types::workload::QueryWorkloadEntry; diff --git a/crates/logical-asap-optimization/src/pass2/memo.rs b/crates/stage1-pass2-reuse-candidates/src/memo.rs similarity index 87% rename from crates/logical-asap-optimization/src/pass2/memo.rs rename to crates/stage1-pass2-reuse-candidates/src/memo.rs index 754a06a5c..2e507211e 100644 --- a/crates/logical-asap-optimization/src/pass2/memo.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo.rs @@ -1,5 +1,5 @@ //! The logical memo's rules over the memo groups Stage 0 builds -//! (`crate::stage0`). +//! (`asap_stage0_memo`). pub(crate) mod rules; /// Rule 1: tumbling-window layouts of a summary state group. diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules.rs similarity index 100% rename from crates/logical-asap-optimization/src/pass2/memo/rules.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/rules.rs diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs similarity index 76% rename from crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs index 95c8c7c0a..fee9f0bfd 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/finer_aggregate.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs @@ -1,8 +1,8 @@ //! Read a SQL aggregate from a finer aggregate's result -//! ([`match_finer`](crate::pass2::finer_aggregate::match_finer)): filter the finer +//! ([`match_finer`](crate::memo::rules::finer_aggregate::match_finer)): filter the finer //! rows, then merge them into the query's groups with each measure's rollup. -use crate::pass2::memo::rules::Pass2RuleError; +use crate::memo::rules::Pass2RuleError; use asap_types::stages::Memo; /// Rule 5, finer aggregate: a candidate of a coarse group that filters and rolls up a finer group. diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs similarity index 93% rename from crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs index de56a825b..2f5b8a77f 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/rollup.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs @@ -1,6 +1,6 @@ //! Producer-level exact and sketch rollups, using the existing capability rules. -use crate::pass2::memo::rules::Pass2RuleError; +use crate::memo::rules::Pass2RuleError; use asap_types::stages::Memo; /// Rule 6, rollup: a candidate of a coarse aggregate or summary group reading diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs similarity index 90% rename from crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs index a808eb48f..345e7b5ec 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/scan_labels.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs @@ -1,7 +1,7 @@ //! Scan-label widening for explicit memo state sharing. Reuses the existing //! Pass 2 widening rule; only its source-chain boundaries are interned here. -use crate::pass2::memo::rules::Pass2RuleError; +use crate::memo::rules::Pass2RuleError; use asap_types::stages::Memo; /// Rule 2, scan widening: a group for the widened scan, and in each original scan group a candidate `Project(widened)`. diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs similarity index 91% rename from crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs index 956141fbf..2bea6eaf2 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/summary_inputs.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs @@ -2,7 +2,7 @@ //! This preserves their full expressions and schema, including opaque series //! identity, rather than assuming that a projection is an identity operation. -use crate::pass2::memo::rules::Pass2RuleError; +use crate::memo::rules::Pass2RuleError; use asap_types::stages::Memo; /// Rule 3, summary inputs: a value projection or filter below a summary build as its own input group. diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs similarity index 90% rename from crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs index e3b1b22dc..0a82547ba 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/topk.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs @@ -1,7 +1,7 @@ //! Exact ranking reuse with explicit exact-input groups. No global selection //! flags are required: only exact implementations can supply these inputs. -use crate::pass2::memo::rules::Pass2RuleError; +use crate::memo::rules::Pass2RuleError; use asap_types::stages::Memo; /// Rule 4, top-k reuse: a candidate of each top-k group reading a larger-`k` group, with an exact-input requirement. diff --git a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs similarity index 94% rename from crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs rename to crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs index a3b660450..804bcdd65 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs +++ b/crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs @@ -3,7 +3,7 @@ //! over the `TimeRange` (and `TimeShift`s) over it as a candidate of the //! summary state group. -use crate::pass2::memo::rules::Pass2RuleError; +use crate::memo::rules::Pass2RuleError; use asap_types::stages::Memo; use asap_types::workload::QueryWorkloadEntry; diff --git a/crates/physical-asap-optimization/Cargo.toml b/crates/stage2-physical-optimization/Cargo.toml similarity index 90% rename from crates/physical-asap-optimization/Cargo.toml rename to crates/stage2-physical-optimization/Cargo.toml index dc4220f5e..db1f94888 100644 --- a/crates/physical-asap-optimization/Cargo.toml +++ b/crates/stage2-physical-optimization/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "asap-physical-asap-optimization" +name = "asap-stage2-physical-optimization" version = "0.1.0" edition = "2021" diff --git a/crates/physical-asap-optimization/src/catalog/mod.rs b/crates/stage2-physical-optimization/src/catalog/mod.rs similarity index 100% rename from crates/physical-asap-optimization/src/catalog/mod.rs rename to crates/stage2-physical-optimization/src/catalog/mod.rs diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/stage2-physical-optimization/src/lib.rs similarity index 100% rename from crates/physical-asap-optimization/src/lib.rs rename to crates/stage2-physical-optimization/src/lib.rs diff --git a/crates/plan-selection/Cargo.toml b/crates/stage3-plan-selection/Cargo.toml similarity index 95% rename from crates/plan-selection/Cargo.toml rename to crates/stage3-plan-selection/Cargo.toml index ab85b889e..22f140ef0 100644 --- a/crates/plan-selection/Cargo.toml +++ b/crates/stage3-plan-selection/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "asap-plan-selection" +name = "asap-stage3-plan-selection" version = "0.1.0" edition = "2021" diff --git a/crates/plan-selection/src/lib.rs b/crates/stage3-plan-selection/src/lib.rs similarity index 100% rename from crates/plan-selection/src/lib.rs rename to crates/stage3-plan-selection/src/lib.rs diff --git a/crates/plan-selection/src/milp/mod.rs b/crates/stage3-plan-selection/src/milp/mod.rs similarity index 100% rename from crates/plan-selection/src/milp/mod.rs rename to crates/stage3-plan-selection/src/milp/mod.rs diff --git a/crates/plan-selection/src/milp/model.rs b/crates/stage3-plan-selection/src/milp/model.rs similarity index 100% rename from crates/plan-selection/src/milp/model.rs rename to crates/stage3-plan-selection/src/milp/model.rs diff --git a/crates/plan-selection/src/report.rs b/crates/stage3-plan-selection/src/report.rs similarity index 100% rename from crates/plan-selection/src/report.rs rename to crates/stage3-plan-selection/src/report.rs diff --git a/crates/plan-selection/src/select.rs b/crates/stage3-plan-selection/src/select.rs similarity index 100% rename from crates/plan-selection/src/select.rs rename to crates/stage3-plan-selection/src/select.rs diff --git a/crates/plan-selection/tests/cost_inputs_json.rs b/crates/stage3-plan-selection/tests/cost_inputs_json.rs similarity index 100% rename from crates/plan-selection/tests/cost_inputs_json.rs rename to crates/stage3-plan-selection/tests/cost_inputs_json.rs diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index 61bc53e69..94e997390 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (hash-consing, `Memo::find_or_add_group`). + // node is legal to share (Stage 0 hash-consing, `cse` in `asap_stage0_memo`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 26a791f86..b6eafaed4 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -1,7 +1,8 @@ //! Helpers that only tests call (this crate's integration tests and other //! crates' tests), exposed under the `test-support` feature so the normal //! public API stays as narrow as its production users need. Each forwards -//! to the crate-private function of the same name. +//! to the crate-private function of the same name, except the fixture +//! [`population_over_selector`], which the executor's tests build. use crate::ir::materialized_view::RawDataWindow; use crate::ir::operator::maintained_population::CurrentSeriesInput; @@ -73,3 +74,75 @@ pub fn physical_asap_dag_at_query_time( .0, ) } + +/// The maintained-population sub-DAG for executor tests: `root` (below its +/// projections, which are kept) replaced by `EvaluatePopulation(statistic)` +/// over a current-series population of the single selector under it, +/// grouped by `grouping`, with no matchers. The caller states the statistic +/// and grouping; nothing is recognized. No planner pass offers this sub-DAG +/// yet (#681). +pub fn population_over_selector( + root: &Rc, + statistic: crate::ir::operator::maintained_population::PopulationStatistic, + grouping: &[&str], +) -> Rc { + use crate::ir::operator::maintained_population::*; + use crate::ir::operator::operator_properties::Source; + if let Some(NonASAPOp::Project { + cols, + qualifier, + child, + }) = root.non_asap() + { + let child = population_over_selector(child, statistic.clone(), grouping); + let project = Operator::NonASAP(NonASAPOp::Project { + cols: cols.clone(), + qualifier: qualifier.clone(), + child, + }); + return Rc::new(OperatorNode::with_schema(project, root.schema.clone())); + } + let selectors: Vec<_> = OperatorNode::reachable(root) + .into_iter() + .filter(|node| matches!(node.non_asap(), Some(NonASAPOp::TimeRange { .. }))) + .collect(); + let [selector] = selectors.as_slice() else { + panic!("one selector under {root:?}"); + }; + let Some(NonASAPOp::TimeRange { range, child, .. }) = selector.non_asap() else { + unreachable!() + }; + let Some(NonASAPOp::Scan { + source: Source::TimeSeries { metric }, + .. + }) = child.non_asap() + else { + panic!("a time-series selector, got {child:?}"); + }; + let population = MaintainedPopulation { + input: PopulationInput::CurrentSeries(CurrentSeriesInput { + metric: metric.clone(), + matchers: Vec::new(), + grouping: grouping.iter().map(|label| label.to_string()).collect(), + without: false, + lookback_ms: range.as_millis() as u64, + }), + max_k: match statistic { + PopulationStatistic::TopK { k } => k, + _ => 0, + }, + quantiles: matches!(statistic, PopulationStatistic::Quantile { .. }), + }; + let maintained = OperatorNode::with_schema( + Operator::ASAP(ASAPOp::MaintainPopulation { + child: selector.clone(), + population, + }), + selector.schema.clone(), + ); + let evaluation = Operator::ASAP(ASAPOp::EvaluatePopulation { + child: Rc::new(maintained), + evaluation: statistic, + }); + Rc::new(OperatorNode::with_schema(evaluation, root.schema.clone())) +} diff --git a/crates/types/tests/planner_vocabulary.rs b/crates/types/tests/planner_vocabulary.rs index 5f5a6ab7a..ebb441646 100644 --- a/crates/types/tests/planner_vocabulary.rs +++ b/crates/types/tests/planner_vocabulary.rs @@ -1,7 +1,7 @@ use asap_types::ir::physical_export::WindowEdgeCompatibility; // The renamed window-edge compatibility still reads and emits the deployed -// wire value. (Tumbling-window coverage names are checked in physical-asap-optimization.) +// wire value. (Tumbling-window coverage names are checked in asap-stage2-physical-optimization.) #[test] fn window_edge_compatibility_names_preserve_wire_values() { let compatibility = diff --git a/docs/design_docs/architecture/README.md b/docs/design_docs/architecture/README.md index f714afc1f..a521d7e3f 100644 --- a/docs/design_docs/architecture/README.md +++ b/docs/design_docs/architecture/README.md @@ -45,9 +45,10 @@ target. | Shared IR | `asap-types` | The unified operator IR (`ir`: one `OperatorNode` before and after ASAP optimization), schemas, workloads, guarantees, and exported plan data | | Front-end common | `frontend-common` | Name-based `UnresolvedOp` tree shared by the front ends, and `resolve_root` into the operator IR | | Query frontends | `frontend-sql`, `frontend-promql`, `frontend-metricsql` | Stage 0: parse source languages and produce canonical Pre-ASAP queries | -| Logical ASAP-aware optimization | `logical-asap-optimization` | Stage 1: per-computation candidates (Pass 1), ASAP-aware CSE (Pass 2), the analytical accuracy model | -| Physical ASAP-aware optimization | `physical-asap-optimization` | Stage 2: materialization and physical operator implementation | -| Plan selection | `plan-selection` | Stage 3: accuracy and latency checks, costing and selection; the planning models and the Stage 1 to Stage 3 driver | +| Stage 0 hash-consing | `stage0-memo` | The memo groups of the workload's query roots | +| Logical ASAP-aware optimization | `stage1-pass1-per-computation-candidates`, `stage1-pass2-reuse-candidates` | Stage 1: per-computation candidates (Pass 1), ASAP-aware CSE (Pass 2) | +| Physical ASAP-aware optimization | `stage2-physical-optimization` | Stage 2: materialization and physical operator implementation | +| Plan selection | `stage3-plan-selection` | Stage 3: accuracy and latency checks, costing and selection; the planning models and the Stage 1 to Stage 3 driver | | Planner facade | `planner` | `e2e_plan`: Stage 0 followed by Stages 1–3 | | Executor | `executor` | Stage 4: runs a `PhysicalASAPDAG` over `asap_sketchlib` summaries; no stage crate depends on it | | Developer inspection | `devtools` | `asap_planner` and the Stage Viewer documents, corpus coverage tools | diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index dfc46531b..f7fb57d5b 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -233,14 +233,16 @@ pass no longer runs these steps; another pass need not run any of them. | Crate | What it holds | |---|---| | `asap-types` | `ParsedWorkload` | -| `asap-plan-selection` | Stage 3: `select_plan`, `Selection` | +| `asap-stage3-plan-selection` | Stage 3: `select_plan`, `Selection` | | `asap-planner` *(new)* | `e2e_plan`, `UserInput`, `FrontendInput`, lowering dispatch; `OptimizationPass`, `OptimizationInput`, `PlanOutput`, `PlanningModels`, `optimize`, `PassRegistry`, `Stage1ToStage3Pass`, `run_planning` | ```text asap-planner ──┬──> asap-frontend-{sql, promql, metricsql} - ├──> asap-logical-asap-optimization ──> asap-types - ├──> asap-physical-asap-optimization ──> asap-types - └──> asap-plan-selection ──> asap-types, asap-cost, asap-accuracy + ├──> asap-stage0-memo ──> asap-types + ├──> asap-stage1-pass1-per-computation-candidates ──> asap-types, asap-accuracy + ├──> asap-stage1-pass2-reuse-candidates ──> asap-types + ├──> asap-stage2-physical-optimization ──> asap-types + └──> asap-stage3-plan-selection ──> asap-types, asap-cost, asap-accuracy ``` `asap-planner` is separate because it is the only crate depending on every diff --git a/docs/design_docs/decisions/concat-unique-keys.md b/docs/design_docs/decisions/concat-unique-keys.md index 4820d2185..4e18cd28f 100644 --- a/docs/design_docs/decisions/concat-unique-keys.md +++ b/docs/design_docs/decisions/concat-unique-keys.md @@ -75,7 +75,7 @@ Both current `Concat`-constructing call sites, and every consumer of test fixtures) — none of them builds a fresh `Concat` with a `Dedup` on top that this feature could remove. - **Every consumer of `Schema::unique_keys`** in the DAG, to check for a - cost beyond "a literal `Dedup` node": `share_common_sub_dags` (crate-private in `asap_logical_asap_optimization`, `stage0/cse.rs`) + cost beyond "a literal `Dedup` node": `share_common_sub_dags` (crate-private in `asap_stage0_memo`, `cse.rs`) (gates CSE producer-sharing on `Schema::has_unique_key()`) and `asap_aware_mapping::rollup::is_legal_rollup_source` (gates rollup-source legality the same way, on an *`Aggregate`'s* own output schema). Neither diff --git a/docs/design_docs/decisions/cse-cost-model.md b/docs/design_docs/decisions/cse-cost-model.md index b7a87edba..45d757f58 100644 --- a/docs/design_docs/decisions/cse-cost-model.md +++ b/docs/design_docs/decisions/cse-cost-model.md @@ -16,7 +16,7 @@ elimination) to identify eligible, structurally identical computations. A shared logical node records an opportunity for reuse; selecting maintained state or independent execution is a separate planning decision. -[`share_common_sub_dags`](../../../crates/logical-asap-optimization/src/stage0/cse.rs) +[`share_common_sub_dags`](../../../crates/stage0-memo/src/cse.rs) (issue #223 stages 1-2, PR #235) already *detects* every structurally-identical, legally-shareable (`Schema::unique_keys`-gated) sub-DAG and shares it **unconditionally** — there is no cost gate on top of legality. This document diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 0eca0b93b..4a0b267e0 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -94,7 +94,7 @@ Based on our requirement, each field should contain the following information. | `FrequencyL2`, `FrequencyEntropy` | `Sketch`: `UnivMon` | `SummaryEstimate(FrequencyL2 \| FrequencyEntropy)` | | `Sum`, `Count`, `Min`, `Max`, `Rate`, `IRate`, `Increase` (exact) | `ExactAggregate(ExactKind, …)` | `FinalizeExactAccumulator` | - The sketch candidates are `summary_candidates(intent)` in `crates/logical-asap-optimization/src/pass1/realization.rs`; exact accumulators come from Pass 1's `ExactAccumulator` candidates in `pass1/per_computation_candidates.rs`; the evaluations are `SketchStatistic` ([§6.5](#65-update-input-and-evaluations-irschemasketchrs-iroperatormaintained_populationrs)). + The sketch candidates are `summary_candidates(intent)` in `crates/stage1-pass1-per-computation-candidates/src/realization.rs`; exact accumulators come from Pass 1's `ExactAccumulator` candidates in `pass1/per_computation_candidates.rs`; the evaluations are `SketchStatistic` ([§6.5](#65-update-input-and-evaluations-irschemasketchrs-iroperatormaintained_populationrs)). - **Time window aggregation intents**: whether states built over smaller windows can answer a larger one. This depends on how the family combines states: - **Merge** (`SummaryMerge`, §5.6): states over disjoint tumbling windows combine into the state of their union, e.g. two 1-minute KLL states answer a 2-minute quantile. Requires a mergeable family. @@ -1212,7 +1212,7 @@ This part of the code answers two questions about summary state: 1. **Update input** (`SummaryUpdate`): when a `SummaryAgg` reads one input row, what does it add to the state? It adds a **weight**, optionally under an **item** key. 2. **Evaluations**: once the state is built, what can be asked of it? `SketchStatistic` is what `SummaryEstimate` asks a sketch (§5.2). `MaintainedPopulation` and `PopulationStatistic` are the exact-population counterpart, used by `MaintainPopulation` and `EvaluatePopulation` (§5.4, §5.5). -**Update input examples** (as the planner builds them in Stage 1, `asap-logical-asap-optimization`): +**Update input examples** (as the planner builds them in Stage 1, `asap-stage1-pass1-per-computation-candidates`): | Query | Sketch | `item` | `weight` | `weight_domain` | |---|---|---|---|---| diff --git a/docs/design_docs/proposals/joint-planning-implementation-plan.md b/docs/design_docs/proposals/joint-planning-implementation-plan.md index 55831cfbd..81644691b 100644 --- a/docs/design_docs/proposals/joint-planning-implementation-plan.md +++ b/docs/design_docs/proposals/joint-planning-implementation-plan.md @@ -12,14 +12,14 @@ Preserve supported accuracy semantics, physical eligibility, cost objectives, sc | Scope | Current implementation | Responsibility | | --- | --- | --- | -| Logical identity | `logical-asap-optimization/src/stage0.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`), stable initial query/group references. | -| Computation and sharing rules | `logical-asap-optimization/src/pass2/memo/{candidates,rules,state_groups,shared_configurations,window_candidates}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | -| Physical legality | `physical-asap-optimization` | Executable operators, timing and materialization eligibility. | -| Candidate dependency inventory | `physical-asap-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning; Stage 3 builds the selected plan (`PhysicalASAPDAG`) directly. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`plan-selection/src/search/candidates/scopes.rs`). | -| Solver | `plan-selection/src/search/milp` | Activation, producer/root assignment, reachability, acyclicity, resource constraints, and solver warm starts. | -| Search and validation | `plan-selection/src/search/refinement` | Actual evaluator checks, sound cuts, objective-specific ranking, and certified component splitting. | -| Pricing support | `plan-selection/src/search/costing` | Certified local bounds and consistent calibration. Production execution demand is owned by `cost/src/execution_demand.rs`; the search demand reference is test-only. | -| Pipeline and lifecycle | `plan-selection/src/search/{pipeline,route}` | Initial groups, generation accounting, seed/full search budgets, archives, reports, and selected workload outputs. | +| Logical identity | `stage0-memo/src/lib.rs`; `types/src/stages/` | `DagOfLogicalMemoGroups`, `MemoGroup` (`asap_types::stages`; built by `memo_of_queries`), stable initial query/group references. | +| Computation and sharing rules | `stage1-pass2-reuse-candidates/src/memo/{candidates,rules,state_groups,shared_configurations,window_candidates}` | Populate groups, retain local candidates, expose shared states and compatible rewrites. | +| Physical legality | `stage2-physical-optimization` | Executable operators, timing and materialization eligibility. | +| Candidate dependency inventory | `stage2-physical-optimization/src/catalog`; `types/src/stages/physical` | Local `PhysicalCandidate`, `ProducerRequirement` (`asap_types::stages`), compatible producers, structural pruning; Stage 3 builds the selected plan (`PhysicalASAPDAG`) directly. Stage 2 makes no execution-scope copies; today's Stage 3 engine makes its own (`stage3-plan-selection/src/search/candidates/scopes.rs`). | +| Solver | `stage3-plan-selection/src/search/milp` | Activation, producer/root assignment, reachability, acyclicity, resource constraints, and solver warm starts. | +| Search and validation | `stage3-plan-selection/src/search/refinement` | Actual evaluator checks, sound cuts, objective-specific ranking, and certified component splitting. | +| Pricing support | `stage3-plan-selection/src/search/costing` | Certified local bounds and consistent calibration. Production execution demand is owned by `cost/src/execution_demand.rs`; the search demand reference is test-only. | +| Pipeline and lifecycle | `stage3-plan-selection/src/search/{pipeline,route}` | Initial groups, generation accounting, seed/full search budgets, archives, reports, and selected workload outputs. | `PhysicalCandidate` is a local selectable computation; the plan (`PhysicalASAPDAG`) is the complete workload DAG Stage 3 builds; they are not interchangeable. Physical candidate configuration in selection invokes the authoritative Stage 2 legality code rather than duplicating its rules. Logical rules do not depend on the solver crate. diff --git a/docs/design_docs/proposals/joint-planning-search.md b/docs/design_docs/proposals/joint-planning-search.md index 2558a5341..937f21269 100644 --- a/docs/design_docs/proposals/joint-planning-search.md +++ b/docs/design_docs/proposals/joint-planning-search.md @@ -632,7 +632,7 @@ After selection, the planner reconstructs the DAG and evaluates accuracy, latenc The implementation uses `DagOfLogicalMemoGroups` and `MemoGroup` for initial identities, and `Memo` / `CandidateForMemoGroup` for logical expansion. Local `PhysicalCandidate` choices have explicit producer requirements and compatible connections. Selection reconstructs a `PhysicalWorkloadPlan` and validates its resulting graph through the configured evaluators. The sole production engine is `CandidateConnections`; bounded and exhaustive modes use the same candidate representation and solver. Existing supported runtime operators determine which configurations can be generated. -The candidate memo and its Pass 2 rules belong in `logical-asap-optimization`; physical eligibility remains authoritative in `physical-asap-optimization`. Within selection, candidate inventory/reconstruction, MILP construction, evaluator refinement, pricing support, and route orchestration have separate scopes. State-local choices belong in candidates; cross-candidate decisions use connections and constraints. The retired `CouplingSite` selector representation and legacy search routes are removed rather than retained under new names. +The candidate memo and its Pass 2 rules belong in `stage1-pass2-reuse-candidates`; physical eligibility remains authoritative in `stage2-physical-optimization`. Within selection, candidate inventory/reconstruction, MILP construction, evaluator refinement, pricing support, and route orchestration have separate scopes. State-local choices belong in candidates; cross-candidate decisions use connections and constraints. The retired `CouplingSite` selector representation and legacy search routes are removed rather than retained under new names. Evaluation must distinguish generation coverage from selection correctness. Small workloads are exhaustively enumerated over candidate subsets and compatible connections under the same inventory, then compared with MILP results. Cases include multiple selected candidates in one group, shared build costs, separate execution scopes, fusion outputs, and composed accuracy/resource requirements. Scaling experiments measure candidate and connection counts, generation and solve time, planner memory, and validated execution savings. Together these measurements establish whether the representation avoids repeated search work and whether joint selection justifies its planning cost. diff --git a/docs/design_docs/proposals/planner-layering-example2-acceptance.md b/docs/design_docs/proposals/planner-layering-example2-acceptance.md index 0ec29bd09..b0d35a810 100644 --- a/docs/design_docs/proposals/planner-layering-example2-acceptance.md +++ b/docs/design_docs/proposals/planner-layering-example2-acceptance.md @@ -10,7 +10,7 @@ several computations". Tests: This spec was written by a test designer who does not implement the stages. Tests run the stages through `planner_layering_common::run_stages`: Pass 1 and composition, `optimize_physical_asap` and the whole-workload reference -pricer (`asap_plan_selection::reference_pricer::select_among_plans`). A test that +pricer (`asap_stage3_plan_selection::reference_pricer::select_among_plans`). A test that needs a missing feature is `#[ignore]`d with the PR that delivers it. ## Workload diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 0d195901d..5f983defc 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -282,7 +282,7 @@ one. It makes no implementation choice, so this section counts it as Stage 0 (its output); the code runs it as Stage 1 starts, which is why [Stage inputs and outputs](#stage-inputs-and-outputs-whole-workload-candidates-and-memo) lists `memo_of_queries` with Pass 1. `memo_of_queries` runs -`share_common_sub_dags` (`crates/logical-asap-optimization/src/stage0/cse.rs`) over the frontends' +`share_common_sub_dags` (`crates/stage0-memo/src/cse.rs`) over the frontends' roots and numbers the shared nodes as groups. Two structurally equal nodes become one group only when sharing them is legal, that is, when every reader of the shared node reads the rows it would read alone. Equal nodes are @@ -513,11 +513,11 @@ full tumbling-window-width domain, with no time limit. #### Memo code interface -The memo rules are in `crates/logical-asap-optimization/src/` (module -`pass2::memo`, crate-private: the crate root re-exports each entry point); the memo types Stage 1 hands Stage 2 are in +The memo rules are in `crates/stage1-pass2-reuse-candidates/src/` (module +`memo`, crate-private: the crate root re-exports each entry point); the memo types Stage 1 hands Stage 2 are in `crates/types/src/stages/` (`asap_types::stages`); the facade that runs the stages is in `crates/planner/src/run.rs`; the search over Stage 2's -candidates is in `crates/plan-selection/src/` (`select.rs`, `groups/`, `milp/`). +candidates is in `crates/stage3-plan-selection/src/` (`select.rs`, `groups/`, `milp/`). | Step | Entry point | Path | |---|---|---| @@ -526,7 +526,7 @@ candidates is in `crates/plan-selection/src/` (`select.rs`, `groups/`, `milp/`). | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | -| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/plan-selection/src/select.rs` | +| Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/select.rs` | ## ASAPPlanner Detailed Design: Planning Stages and their decisions/Strategies @@ -598,8 +598,8 @@ the memo path's entry points. | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_physical_asap_optimization`) | -| 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | +| 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_stage3_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | What changed with the memo: diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index 953bf57f0..400d7b2e3 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -19,22 +19,26 @@ in `asap-integration-tests` (`crate_graph`) checks these edges with | --- | --- | | `asap-types` (`crates/types`) | Shared vocabulary, by IR section: `ir::operator` (`OperatorNode`, `NonASAPOp`, `ASAPOp`, operator parameters, `AggIntent`), `ir::scalar` (`ScalarExpr`, `ColumnRef`), `ir::schema` (`Schema`, `FieldDataType`, summary-state types such as `SketchAlgorithm` and `SummaryTypeCapability`), `ir::properties` (`AccuracyTarget`, `AccuracyMetric`, `AccuracyGuarantee`, execution timing, summary coverage), plus `ir::flat` and `ir::physical_export`; the planner inputs (`workload`); Stage 2 decision data (`physical`: `PhysicalProperties` (timing and `Lifetime`), tumbling windows) | | `asap-frontend-sql`, `asap-frontend-promql`, `asap-frontend-metricsql` (with `asap-frontend-common`) | Stage 0, the query-language frontends: lower each query into the logical DAG (`QueryRoot`s of `NonASAPOp` nodes, no summaries, no timing) | -| `asap-logical-asap-optimization` (`crates/logical-asap-optimization`) | Stage 1: Pass 1 and Pass 2 (the memo rules) | +| `asap-stage0-memo` (`crates/stage0-memo`) | Stage 0 hash-consing: the memo groups of the workload's query roots (`memo_of_queries`) | +| `asap-stage1-pass1-per-computation-candidates` (`crates/stage1-pass1-per-computation-candidates`) | Stage 1, Pass 1: the initial candidates of each memo group | +| `asap-stage1-pass2-reuse-candidates` (`crates/stage1-pass2-reuse-candidates`) | Stage 1, Pass 2: the reuse rules over memo groups and the logical edges | | `asap-accuracy` (`crates/accuracy`) | The accuracy model the stages share: the analytical rules (per-algorithm estimators, composition, budget split, target check), the deployment's measured accuracy (`AccuracyEvidence`) and α^log (`producer_satisfies_logical_requirements`); depends only on `asap-types` | -| `asap-physical-asap-optimization` (`crates/physical-asap-optimization`) | Stage 2: materialization and physical operator implementation | +| `asap-stage2-physical-optimization` (`crates/stage2-physical-optimization`) | Stage 2: materialization and physical operator implementation | | `asap-cost` (`crates/cost`) | The cost model the stages share: resource-usage estimation (the analytical per-operator model, measured operation costs, state byte sizes) and the cost models (`ResourceUsage`, `CostModel`, `CostInputs`); depends only on `asap-types` | -| `asap-plan-selection` (`crates/plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | +| `asap-stage3-plan-selection` (`crates/stage3-plan-selection`) | Stage 3: `select_plan` over Stage 2's output; accuracy and latency checks, pricing candidates with `asap-cost` | | `asap-planner` (`crates/planner`) | The facade: `e2e_plan` runs Stage 0, then the optimization pass (`Stage1ToStage3Pass` by default), which orchestrates Stages 1–3 (`run_planning`); the planning models (`PlanningModels`) and the `OptimizationPass` trait | | `asap-executor` (`crates/executor`) | Stage 4, the reference executor: operators and summary kernels over `asap_sketchlib`; it runs a `PhysicalASAPDAG` | | `asap-devtools` | `asap_planner` and the other inspection tools, the design-doc example workloads (`planner_layering_examples`) and the candidate labels | ```text asap-types ◄── frontends (Stage 0) -asap-types ◄── asap-cost (shared cost model) ◄── asap-physical-asap-optimization, asap-plan-selection -asap-types ◄── asap-accuracy (shared accuracy model) ◄── asap-logical-asap-optimization, asap-physical-asap-optimization, asap-plan-selection -asap-types ◄── asap-logical-asap-optimization (Stage 1) ◄── asap-planner -asap-types ◄── asap-physical-asap-optimization (Stage 2) ◄── asap-planner -asap-plan-selection (Stage 3) ◄── asap-planner (also depends on the frontends) +asap-types ◄── asap-cost (shared cost model) ◄── asap-stage2-physical-optimization, asap-stage3-plan-selection +asap-types ◄── asap-accuracy (shared accuracy model) ◄── asap-stage1-pass1-per-computation-candidates, asap-stage2-physical-optimization, asap-stage3-plan-selection +asap-types ◄── asap-stage0-memo (Stage 0 hash-consing) ◄── asap-planner +asap-types ◄── asap-stage1-pass1-per-computation-candidates (Stage 1, Pass 1) ◄── asap-planner +asap-types ◄── asap-stage1-pass2-reuse-candidates (Stage 1, Pass 2) ◄── asap-planner +asap-types ◄── asap-stage2-physical-optimization (Stage 2) ◄── asap-planner +asap-stage3-plan-selection (Stage 3) ◄── asap-planner (also depends on the frontends) asap-types ◄── asap-executor (Stage 4; no stage crate depends on it) ``` @@ -42,18 +46,18 @@ asap-types ◄── asap-executor (Stage 4; no stage crate depends on it) ```text QueryRoots (Stage 0) - │ memo_of_queries → DagOfLogicalMemoGroups, then - │ pass1_memo + │ asap_stage0_memo::memo_of_queries → DagOfLogicalMemoGroups, then + │ asap_stage1_pass1_per_computation_candidates::pass1_memo ▼ Stage 1, Pass 1 PerComputationLogicalMemoCandidates (Memo + pruned_candidates) - │ generate_logical_asap_memo_candidates - │ the memo rules (pass2/memo/rules/) over the candidate memo Stage 1, Pass 2 + │ asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates + │ the memo rules (memo/rules/) over the candidate memo Stage 1, Pass 2 ▼ LogicalASAPMemoCandidates - │ asap_physical_asap_optimization::generate_physical_asap_memo_candidates Stage 2 + │ asap_stage2_physical_optimization::generate_physical_asap_memo_candidates Stage 2 ▼ PhysicalASAPMemoCandidates (operator nodes per timing/materialization + pruned_candidates) - │ asap_plan_selection::select_plan Stage 3 + │ asap_stage3_plan_selection::select_plan Stage 3 ▼ (PhysicalASAPDAG, Selection) — the plan, and accepted_accuracy_per_query, pruned_candidates and the selection MILP's size and time @@ -66,17 +70,17 @@ same pipeline behind the `OptimizationPass` trait. | Concern | Where it lives | | --- | --- | -| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) | `asap-logical-asap-optimization`, `pass1/per_computation_candidates.rs` | -| Which summary families serve an intent (`summary_candidates`) | `asap-logical-asap-optimization`, `pass1/realization.rs` | -| Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-logical-asap-optimization`, `pass2/memo/rules/` and `pass2/memo/window_candidates.rs`, over the helpers in `pass2/asap_aware_cse.rs` | -| Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | `asap-logical-asap-optimization`, `pass2/memo/enumeration.rs` | +| Pass 1: which computations a query has (`ComputationTarget`), how each can be computed (`HowToComputeAggregateMeasure`: `Exact`, `ExactAccumulator`, `Summary`), fused computations (`FusedComputation`), pruning (`PruneReason`) | `asap-stage1-pass1-per-computation-candidates`, `per_computation_candidates.rs` | +| Which summary families serve an intent (`summary_candidates`) | `asap-stage1-pass1-per-computation-candidates`, `realization.rs` | +| Pass 2 memo rules: shared producers (identical expressions by memo group identity, one build at different sizes), scan-label widening, window composition (tumbling windows), rollup, exact top-k reuse and finer-aggregate reuse | `asap-stage1-pass2-reuse-candidates`, `memo/rules/` and `memo/window_candidates.rs` | +| Enumeration of the plans a memo stands for (reference pricer, DAG viewer) | none today (deleted in #1207 PR 0) | | What each summary type can merge and answer (`SummaryTypeCapability::of`: `merge_inputs`, `readouts`) | `asap-types`, `ir/schema/summary_type_capability.rs` | -| Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-physical-asap-optimization` | +| Stage 2: physical properties options per summary build (not materialized; materialized at ingestion time `AcrossEvaluations`; materialized at query time `OneEvaluation` or `AcrossEvaluations`), ingestion-time eligibility, physical operator implementation | `asap-stage2-physical-optimization` | | Timing and data-state edge rules applied from a `PhysicalPropertiesAssignment` | `asap-types`, `ir/properties/timing.rs` and `ir/properties/execution_data_state.rs`; `PhysicalProperties` and `Lifetime` in `physical/properties.rs` | -| Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | +| Stage 3: accuracy check, latency check, resource usage (`ResourceUsage`), cost models 1–3 (`CostModel`), measured operation costs | `asap-stage3-plan-selection`, `lib.rs`; `asap-cost`, `lib.rs` and `measured.rs` | | The analytical per-operator resource model Stage 3 measures with | `asap-cost`, `analytical.rs` and `operator_statistics.rs` | | The accuracy algebra: per-family guarantees, composition, budget split; guarantee propagation through a DAG; sizing a summary for its target, (ε, δ) budget and default sizes | `asap-accuracy` (`AccuracyEvidence`, `theoretical_guarantee`, `estimators/`, `composition.rs`, `budget.rs`, `propagation.rs`, `sizing.rs`) | -| What the executor supports (`DeploymentCapabilities`) | `asap-logical-asap-optimization`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | +| What the executor supports (`DeploymentCapabilities`) | `asap-types`, `deployment_capabilities.rs`; Pass 1 receives only `supported_summaries` and Stage 2 only `query_time_results_retained` and `raw_data_retained` (`bool` arguments, so Stage 2 does not depend on the Stage 1 crate); Stage 3 reads none of it | | The inputs every stage reads (`PlanningModels`: accuracy evidence, cost inputs, deployment capabilities) and the `OptimizationPass` trait | `asap-planner`, `pass.rs`; the facade gives Pass 2 the evidence and Stage 3 only `CostInputs` | The current production search organizes these computations in memo result/state @@ -115,9 +119,10 @@ No stage caps the candidates it plans, and neither does `asap_planner`. ## Verification - Unit tests sit beside each module (`cargo nextest run -p - asap-logical-asap-optimization -p asap-physical-asap-optimization -p - asap-plan-selection -E 'test()'`). -- `crates/logical-asap-optimization/tests/per_computation_candidates.rs` covers + asap-stage0-memo -p asap-stage1-pass1-per-computation-candidates -p + asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p + asap-stage3-plan-selection -E 'test()'`). +- `crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs` covers Pass 1. - `crates/integration-tests/tests/planner_layering_example{1,2,3,4}.rs` pin the design document's examples through every stage; diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 4935b3c26..1e29571d3 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -5,8 +5,8 @@ the stage code relies on: what each stage's input and output may contain, and which component owns each decision. Read the [architecture](asap-aware-mapping-architecture.md) first; use the [extension guide](extend-asap-aware-mapping.md) when changing one. Field-level -details are in the rustdoc (`cargo doc -p asap-logical-asap-optimization -p -asap-physical-asap-optimization -p asap-plan-selection -p asap-types +details are in the rustdoc (`cargo doc -p asap-stage0-memo -p asap-stage1-pass1-per-computation-candidates -p +asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap-stage3-plan-selection -p asap-types --no-deps`). ## 1. IR at each stage @@ -18,7 +18,7 @@ asap-physical-asap-optimization -p asap-plan-selection -p asap-types | Stage 2 (`generate_physical_asap_memo_candidates`) and Stage 3 (`select_plan`, which builds the plan) | the same operators, with a timing and a `Lifetime` on each summary build | every node of the plan timed from the physical candidate it was built from (`selected_physical_candidate`) | - Structurally equal sub-DAGs are interned before Stage 1 - (`stage0/cse.rs` in `asap_logical_asap_optimization`), so Stage 1 compares nodes by structure and Pass 2 + (`cse.rs` in `asap_stage0_memo`), so Stage 1 compares nodes by structure and Pass 2 matches readers structurally, never by pointer. - Timing is applied with `apply_physical_properties`, which validates every data-state edge. A violation is an `ExecutionDataStateError`: diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index c2ea719f2..29221a294 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -25,7 +25,7 @@ A new sketch that Pass 1 can offer for an intent. (`crates/executor/src/capability.rs`); add the family's readouts there once the executor answers them. 3. **Offer and size.** Place it in `realization::FAMILY_PREFERENCE` - (`crates/logical-asap-optimization`): `summary_candidates` offers it, in + (`crates/stage1-pass1-per-computation-candidates`): `summary_candidates` offers it, in that order, for every intent whose statistic its capability row answers. Size it in `asap_accuracy::sizing::default_size_params`. 4. **Accuracy.** Add its error model under @@ -44,7 +44,7 @@ A new sketch that Pass 1 can offer for an intent. not with a local copy. 7. **Labels.** Name it in `asap_devtools::candidate_labels`. -Tests: a Pass 1 test in `crates/logical-asap-optimization/tests/per_computation_candidates.rs` +Tests: a Pass 1 test in `crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs` that the intent gets the new candidate (and that a deployment without it prunes it with `UnsupportedByDeployment`); an estimator test beside the estimator; an execution test in `crates/integration-tests/tests/` such as @@ -55,7 +55,7 @@ estimator; an execution test in `crates/integration-tests/tests/` such as A new way for computations of one Pass 1 candidate to share, split or reuse work. -1. Add a memo rule under `crates/logical-asap-optimization/src/pass2/memo/rules/` +1. Add a memo rule under `crates/stage1-pass2-reuse-candidates/src/memo/rules/` (for example `rollup.rs` or `topk.rs`): it adds candidates (and, when it needs them, groups) to the memo groups it implements, keeping every existing candidate. Read summary families only through @@ -81,7 +81,7 @@ contains the candidate and that Stage 3 prices it. (`crates/types/src/physical/properties.rs`, `PhysicalProperties::new` and `readable_at`) and its placement in `crates/types/src/ir/properties/timing.rs`, with its data-state edge rules. -2. Offer it in `crates/physical-asap-optimization/src/lib.rs` (the +2. Offer it in `crates/stage2-physical-optimization/src/lib.rs` (the `MaterializedOutputCandidate` options), after the existing options so default candidate ids stay the same, and gate it on the deployment input that allows it. diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 52b2f114c..564a23955 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -24,22 +24,23 @@ artifact, while preserving the checks required by its intended consumer. ## Dependencies Inside this workspace, depend on the frontend you need, the stage crates -(`asap-logical-asap-optimization` for Stage 1, `asap-physical-asap-optimization` -for Stage 2, `asap-plan-selection` for Stage 3), `asap-planner` for the facade that +(`asap-stage0-memo` for Stage 0 hash-consing, `asap-stage1-pass1-per-computation-candidates` and +`asap-stage1-pass2-reuse-candidates` for Stage 1, `asap-stage2-physical-optimization` +for Stage 2, `asap-stage3-plan-selection` for Stage 3), `asap-planner` for the facade that runs the stages, `asap-cost` for the cost model and its inputs (`CostInputs`), `asap-accuracy` for the accuracy rules and the deployment's measured accuracy (`AccuracyEvidence`), and `asap-types`, or on `asap-planner`, which wraps every frontend and the planning stages. External users can use Git dependencies pinned to a compatible revision; use the same revision across these crates, and pick a revision at least as new as the APIs you call. Replace `REV` below with a -`main` revision that has the stage crates (`crates/plan-selection`). +`main` revision that has the stage crates (`crates/stage3-plan-selection`). For the examples below: ```toml [dependencies] asap-planner = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } -asap-plan-selection = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } +asap-stage3-plan-selection = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } asap-cost = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } asap-accuracy = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "REV" } @@ -183,7 +184,7 @@ asap_planner::run_planning( ) -> Result // Stage 3 alone, over Stage 2's output: -asap_plan_selection::select_plan( +asap_stage3_plan_selection::select_plan( physical: &PhysicalASAPMemoCandidates, entries: &[QueryWorkloadEntry], data: &DataWorkload, @@ -195,7 +196,7 @@ asap_plan_selection::select_plan( | --- | --- | | `PlanningModels::new(evidence)` | Your `&dyn AccuracyEvidence` (measured guarantees; `NoMeasuredGuarantees` has none), with default cost inputs and deployment capabilities; `with_evidence` replaces it; `with_ignore_accuracy_guarantees(true)` gives every query no accuracy target | | `PlanningModels::builtin()` | No measured accuracy evidence. `with_cost_inputs(&settings)` sets the deployment's `CostInputs`; `with_deployment_capabilities(&capabilities)` sets its `DeploymentCapabilities` (field `deployment_capabilities`) | -| `DeploymentCapabilities` (`asap_logical_asap_optimization`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | +| `DeploymentCapabilities` (`asap_types::deployment_capabilities`, re-exported as `asap_planner::DeploymentCapabilities`) | What the executor supports: `supported_summaries` (`None`, the default, supports every summary the planner builds; otherwise `SupportedSummary { summary_type: SketchAlgorithm, readouts: Vec }` entries; Pass 1 prunes others with `PruneReason::UnsupportedByDeployment`), `raw_data_retained` (default `true`; when `false`, a candidate that scans raw data at query time is pruned by Stage 2, and stored data is refused with `asap_planner::SelectionError::InvalidDeploymentCapabilities`) and `query_time_results_retained` (default `false`; when `true`, Stage 2 offers query-time `AcrossEvaluations` for tumbling-window chains) | | `CostInputs` (`asap_cost`) | `cost_model`, `resource_speed` (`ms_per_cpu_op`, `ms_per_scanned_byte`), `non_repeating_query_horizon_seconds` (default 3,600) and `measured_operation_costs` (`MeasuredOperationCosts`, default empty: `operators`, one `OperatorCost { operator, phase, sketch / exact / measure, ns_per_unit, ns_per_invocation, memory_bytes_per_state }` per measured operator kind, phase and family; `ns_per_evaluation` per query request; `provenance` (executor version, machine). A node with an entry is priced from it, a sketch between the two nearest measured sizes; every other node from the formulas times one calibration factor fitted from the measured nodes; each node's `NodeCost::cost_source` says which. The earlier JSON array of `MeasuredOperationCost { sketch, operation, cpu_ns_per_operation }` still loads) | | `CostModel::UsageOverTime(ResourceUnitCost)` | Cost model 1 (default, `ResourceUnitCost::illustrative()`): unit cost × usage per second; the cheapest valid candidate wins | | `CostModel::ProvisionedCapacity(ResourceUnitCost)` | Cost model 2: unit cost × peak memory, peak CPU cores and disk; the cheapest valid candidate wins | @@ -365,14 +366,15 @@ cost/guarantee evidence needed downstream instead of exporting only a bare DAG. For public symbol details, build local API documentation with: ```sh -cargo doc -p asap-logical-asap-optimization -p asap-physical-asap-optimization \ - -p asap-plan-selection -p asap-types --no-deps +cargo doc -p asap-stage0-memo -p asap-stage1-pass1-per-computation-candidates \ + -p asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization \ + -p asap-stage3-plan-selection -p asap-types --no-deps ``` ## Source references - [Frontend PromQL](../../crates/frontend-promql/src/lib.rs), [SQL](../../crates/frontend-sql/src/lib.rs), [MetricsQL](../../crates/frontend-metricsql/src/lib.rs) -- [Planner facade](../../crates/planner/src/lib.rs), [planning stages](../../crates/planner/src/run.rs), [plan selection and cost models 1–3](../../crates/plan-selection/src/lib.rs) -- [Deployment capabilities](../../crates/logical-asap-optimization/src/deployment_capabilities.rs) +- [Planner facade](../../crates/planner/src/lib.rs), [planning stages](../../crates/planner/src/run.rs), [plan selection and cost models 1–3](../../crates/stage3-plan-selection/src/lib.rs) +- [Deployment capabilities](../../crates/types/src/deployment_capabilities.rs) - [Workload types](../../crates/types/src/workload/mod.rs) - [Planner-runtime contract](../design_docs/architecture/planner-runtime-contract.md) diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 9a2d23dc5..f3346279b 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -149,7 +149,7 @@ yet — and calls `asap_frontend_common::resolve_root`, which does three things The result is `Rc`. `lower_sql` / `lower_sql_dialect` and `lower_metricsql` return it; `lower_promql_query_workload` returns a `QueryRoot` per query. -Workload search then runs structural CSE (`share_common_sub_dags` (crate-private in `asap_logical_asap_optimization`, `stage0/cse.rs`)) once +Workload search then runs structural CSE (`share_common_sub_dags` (crate-private in `asap_stage0_memo`, `cse.rs`)) once across every root: bottom-up hash-consing where the structural hash is only a filter and the typed `PartialEq` decides sharing, following scalar references like any other input, and gated by `NonASAPOp::is_shareable_when_equal` for non-ASAP producers. diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index 7d89d4389..904f0f9db 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -49,7 +49,7 @@ or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. | `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | | Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_physical_properties` writes it from a `PhysicalPropertiesAssignment` (default: all query time) | | `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | -| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `share_common_sub_dags` (crate-private in `asap_logical_asap_optimization`, `stage0/cse.rs`) | +| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `asap_frontend_common` `canonicalize` (crate-private), `share_common_sub_dags` (crate-private in `asap_stage0_memo`, `cse.rs`) | | `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::ir::physical_export::compile_physical_asap_dag` (`PhysicalASAPDAG`, carried in the plan file versioned by `asap_planner::INSTALLABLE_PLAN_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | | Exported schema JSON `columns` | `fields` | From 6cdaa2ff23e3286e9867931f50b4d50354144ce3 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:52:35 +0000 Subject: [PATCH 22/30] test: delete the tests that cross stage crates (#1207 PR 0b) Pass 1's tests that build Stage 0's memo and Stage 0's tests that lower PromQL through Pass 1 are deleted, with the dev-dependencies and helpers only they used. lower_promql moves to Pass 1's test fixtures, so Pass 1 no longer depends on the PromQL frontend outside its own tests. The crate-graph test now also checks dev-dependencies between stage crates. --- Cargo.lock | 1 - crates/integration-tests/tests/crate_graph.rs | 162 +++-- crates/stage0-memo/Cargo.toml | 2 +- crates/stage0-memo/src/lib.rs | 4 +- .../Cargo.toml | 10 +- .../src/lib.rs | 8 +- .../src/per_computation_candidates.rs | 210 +----- .../src/per_computation_candidates_tests.rs | 684 +----------------- .../src/test_fixtures.rs | 36 + .../src/test_support.rs | 45 +- 10 files changed, 143 insertions(+), 1019 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a931d4735..be2d17e18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -462,7 +462,6 @@ version = "0.1.0" dependencies = [ "asap-accuracy", "asap-frontend-promql", - "asap-stage0-memo", "asap-types", "thiserror 2.0.18", ] diff --git a/crates/integration-tests/tests/crate_graph.rs b/crates/integration-tests/tests/crate_graph.rs index 1efd73395..3f4d90c3a 100644 --- a/crates/integration-tests/tests/crate_graph.rs +++ b/crates/integration-tests/tests/crate_graph.rs @@ -1,9 +1,11 @@ //! The workspace crate graph follows #1054's target graph (its rule 4). //! //! The test reads `cargo metadata --format-version 1 --no-deps` and checks every -//! `[dependencies]` edge between two workspace crates. Dev- and -//! build-dependencies are not checked. Each crate has a role ([`role_of`]), and -//! [`rule_violated`] says which role may depend on which: +//! `[dependencies]` edge between two workspace crates, and every +//! `[dev-dependencies]` edge between two stage crates (a test of one stage +//! never calls another stage). Build-dependencies are not checked. Each crate +//! has a role ([`role_of`]), and [`rule_violated`] says which role may depend +//! on which: //! //! - **stage crates** (the frontends, Stage 0's memo, Stage 1's Pass 1 and //! Pass 2, Stage 2's physical optimization, Stage 3's plan selection) never @@ -18,7 +20,7 @@ //! other crate may depend on them. //! //! The edges the rules forbid but the workspace has today are listed, each -//! with its reason, in [`EXCEPTIONS`]. A new workspace crate fails the test +//! with its reason, in [`EXCEPTIONS`] and [`DEV_EXCEPTIONS`]. A new workspace crate fails the test //! until it gets a role. use serde_json::Value; @@ -90,38 +92,51 @@ fn rule_violated(from: Role, to: Role) -> Option<&'static str> { } } -/// A forbidden edge the workspace keeps on purpose. -struct Exception { - from: &'static str, - to: &'static str, - /// The exception holds only while the dependency is `optional = true`. - only_if_optional: bool, -} +/// A forbidden edge the workspace keeps on purpose: (from, to). +type Exception = (&'static str, &'static str); -/// The forbidden edges the workspace keeps today. +/// The forbidden `[dependencies]` edges the workspace keeps today. const EXCEPTIONS: &[Exception] = &[ - // Pass 1's `test-support` feature lowers PromQL test queries for other - // crates' tests. The dependency is optional and enabled only by that - // feature, so no production build links a frontend into Stage 1. Made - // non-optional, the edge fails the test. - Exception { - from: "asap-stage1-pass1-per-computation-candidates", - to: "asap-frontend-promql", - only_if_optional: true, - }, // `plan_installable` offers only the summaries the Planner's own executor // implements. #1016 decides where that kernel list and the executor // belong; this exception goes away with it. - Exception { - from: "asap-planner", - to: "asap-executor", - only_if_optional: false, - }, + ("asap-planner", "asap-executor"), +]; + +/// The `[dev-dependencies]` edges between stage crates the workspace keeps +/// today: the query-language frontends' tests and Pass 1's tests. +const DEV_EXCEPTIONS: &[Exception] = &[ + // Pass 1's tests lower PromQL test queries (`test_fixtures::lower_promql`). + ( + "asap-stage1-pass1-per-computation-candidates", + "asap-frontend-promql", + ), + // The frontends' tests compose Pass 1 candidates of the lowered queries + // (`tests/support.rs`'s `logical_candidates`, `frequency_l2.rs`). + ("asap-frontend-promql", "asap-stage0-memo"), + ( + "asap-frontend-promql", + "asap-stage1-pass1-per-computation-candidates", + ), + ("asap-frontend-sql", "asap-stage0-memo"), + ( + "asap-frontend-sql", + "asap-stage1-pass1-per-computation-candidates", + ), + ("asap-frontend-metricsql", "asap-stage0-memo"), + ( + "asap-frontend-metricsql", + "asap-stage1-pass1-per-computation-candidates", + ), + // The MetricsQL tests share the PromQL tests' `support.rs` and compare + // with the PromQL frontend's lowering. + ("asap-frontend-metricsql", "asap-frontend-promql"), ]; -/// One message per `[dependencies]` edge of `metadata` (the JSON of +/// One message per edge of `metadata` (the JSON of /// `cargo metadata --format-version 1 --no-deps`) that breaks a rule, naming -/// the edge and the rule. +/// the edge and the rule: a `[dependencies]` edge that [`rule_violated`] +/// forbids, or a `[dev-dependencies]` edge between two stage crates. fn crate_graph_violations(metadata: &Value) -> Vec { let packages = metadata["packages"] .as_array() @@ -141,28 +156,35 @@ fn crate_graph_violations(metadata: &Value) -> Vec { continue; }; for dependency in package["dependencies"].as_array().into_iter().flatten() { - // `kind` is null for `[dependencies]`, "dev" or "build" otherwise. - if !dependency["kind"].is_null() { - continue; - } let to = dependency["name"].as_str().expect("dependency `name`"); - if !names.contains(&to) { + if !names.contains(&to) || to == from { continue; } let Some(to_role) = role_of(to) else { // Reported once, as its own package above. continue; }; - let Some(rule) = rule_violated(from_role, to_role) else { + // `kind` is null for `[dependencies]`, "dev" or "build" otherwise. + let (section, rule, exceptions) = match dependency["kind"].as_str() { + None => ( + "[dependencies]", + rule_violated(from_role, to_role), + EXCEPTIONS, + ), + Some("dev") => ( + "[dev-dependencies]", + (from_role == Role::Stage && to_role == Role::Stage) + .then_some("a stage crate's tests never call another stage crate"), + DEV_EXCEPTIONS, + ), + Some(_) => continue, + }; + let Some(rule) = rule else { continue; }; - let optional = dependency["optional"].as_bool().unwrap_or(false); - let excepted = EXCEPTIONS - .iter() - .any(|e| e.from == from && e.to == to && (optional || !e.only_if_optional)); - if !excepted { + if !exceptions.contains(&(from, to)) { violations.push(format!( - "forbidden [dependencies] edge {from} -> {to}: {rule} (#1054)" + "forbidden {section} edge {from} -> {to}: {rule} (#1054)" )); } } @@ -196,14 +218,14 @@ fn workspace_crate_graph_follows_the_target_graph() { } /// A package of synthetic `cargo metadata` JSON with `[dependencies]` on -/// `deps` (name, optional) and a dev-dependency on `dev_deps`. -fn package(name: &str, deps: &[(&str, bool)], dev_deps: &[&str]) -> Value { +/// `deps` and `[dev-dependencies]` on `dev_deps`. +fn package(name: &str, deps: &[&str], dev_deps: &[&str]) -> Value { let normal = deps .iter() - .map(|(d, optional)| serde_json::json!({"name": d, "kind": null, "optional": optional})); + .map(|d| serde_json::json!({"name": d, "kind": null})); let dev = dev_deps .iter() - .map(|d| serde_json::json!({"name": d, "kind": "dev", "optional": false})); + .map(|d| serde_json::json!({"name": d, "kind": "dev"})); serde_json::json!({"name": name, "dependencies": normal.chain(dev).collect::>()}) } @@ -211,12 +233,12 @@ fn package(name: &str, deps: &[(&str, bool)], dev_deps: &[&str]) -> Value { fn checker_rejects_an_edge_between_stage_crates() { let metadata = serde_json::json!({"packages": [ package("asap-types", &[], &[]), - package("asap-cost", &[("asap-types", false)], &[]), - package("asap-stage2-physical-optimization", &[("asap-types", false)], &[]), + package("asap-cost", &["asap-types"], &[]), + package("asap-stage2-physical-optimization", &["asap-types"], &[]), // Forbidden: Stage 3 on Stage 2. package( "asap-stage3-plan-selection", - &[("asap-cost", false), ("asap-stage2-physical-optimization", false)], + &["asap-cost", "asap-stage2-physical-optimization"], &[], ), ]}); @@ -231,38 +253,32 @@ fn checker_rejects_an_edge_between_stage_crates() { } #[test] -fn checker_ignores_dev_dependencies_and_external_crates() { +fn checker_ignores_external_crates_and_non_stage_dev_dependencies() { let metadata = serde_json::json!({"packages": [ - package("asap-types", &[("serde", false)], &[]), - package("asap-stage1-pass1-per-computation-candidates", &[("asap-types", false)], &[]), - package( - "asap-stage3-plan-selection", - &[("asap-types", false)], - &["asap-stage1-pass1-per-computation-candidates"], - ), + package("asap-types", &["serde"], &[]), + package("asap-stage1-pass1-per-computation-candidates", &["asap-types"], &[]), + package("asap-executor", &["asap-types"], &["asap-stage1-pass1-per-computation-candidates"]), ]}); assert!(crate_graph_violations(&metadata).is_empty()); } #[test] -fn optional_only_exception_rejects_the_edge_made_non_optional() { - let metadata = |optional| { - serde_json::json!({"packages": [ - package("asap-types", &[], &[]), - package("asap-frontend-promql", &[("asap-types", false)], &[]), - package( - "asap-stage1-pass1-per-computation-candidates", - &[("asap-types", false), ("asap-frontend-promql", optional)], - &[], - ), - ]}) - }; - assert!(crate_graph_violations(&metadata(true)).is_empty()); +fn checker_rejects_a_dev_dependency_between_stage_crates() { + let metadata = serde_json::json!({"packages": [ + package("asap-types", &[], &[]), + package("asap-stage0-memo", &["asap-types"], &[]), + // Forbidden: Pass 1's tests call Stage 0. + package( + "asap-stage1-pass1-per-computation-candidates", + &["asap-types"], + &["asap-stage0-memo"], + ), + ]}); assert_eq!( - crate_graph_violations(&metadata(false)), + crate_graph_violations(&metadata), vec![ - "forbidden [dependencies] edge asap-stage1-pass1-per-computation-candidates -> \ - asap-frontend-promql: stage crates never depend on each other (#1054)" + "forbidden [dev-dependencies] edge asap-stage1-pass1-per-computation-candidates -> \ + asap-stage0-memo: a stage crate's tests never call another stage crate (#1054)" .to_string() ] ); @@ -272,8 +288,8 @@ fn optional_only_exception_rejects_the_edge_made_non_optional() { fn checker_rejects_an_unclassified_crate_and_shared_model_edges() { let metadata = serde_json::json!({"packages": [ package("asap-types", &[], &[]), - package("asap-stage1-pass1-per-computation-candidates", &[("asap-types", false)], &[]), - package("asap-accuracy", &[("asap-stage1-pass1-per-computation-candidates", false)], &[]), + package("asap-stage1-pass1-per-computation-candidates", &["asap-types"], &[]), + package("asap-accuracy", &["asap-stage1-pass1-per-computation-candidates"], &[]), package("asap-new-crate", &[], &[]), ]}); assert_eq!( diff --git a/crates/stage0-memo/Cargo.toml b/crates/stage0-memo/Cargo.toml index 40804bc41..5fda801d9 100644 --- a/crates/stage0-memo/Cargo.toml +++ b/crates/stage0-memo/Cargo.toml @@ -5,6 +5,6 @@ edition = "2021" # #509 Stage 0: hash-consing the workload's logical DAGs into the memo groups # (`asap_types::stages::Memo`, without candidates) that Stage 1 fills. Depends -# only on asap-types; no other stage crate depends on it except in their tests. +# only on asap-types; no other stage crate depends on it. [dependencies] asap-types = { path = "../types" } diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index 79e269fc9..ce293d080 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -5,8 +5,8 @@ //! Stage 1 starts from these groups. //! //! It depends only on `asap-types`, where its output (`Memo`) lives; no -//! other stage crate depends on it except in their tests, and the planner -//! facade calls it first. +//! other stage crate depends on it (the frontends' tests aside, listed in +//! `crate_graph.rs`), and the planner facade calls it first. use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{QueryRoot, SchemaDerivationError}; diff --git a/crates/stage1-pass1-per-computation-candidates/Cargo.toml b/crates/stage1-pass1-per-computation-candidates/Cargo.toml index 9cd188ba4..4af9be734 100644 --- a/crates/stage1-pass1-per-computation-candidates/Cargo.toml +++ b/crates/stage1-pass1-per-computation-candidates/Cargo.toml @@ -6,22 +6,18 @@ edition = "2021" # #509 Stage 1, Pass 1: the initial candidates of each memo group (exact, # exact accumulator, summaries, fused builds), with the shared analytical # accuracy model (`asap-accuracy`). Reads Stage 0's memo through -# `asap_types::stages`; depends on no other stage crate, and on no -# query-language frontend except through the `test-support` feature. +# `asap_types::stages`; depends on no other stage crate and on no +# query-language frontend. [dependencies] asap-types = { path = "../types" } asap-accuracy = { path = "../accuracy" } thiserror = "2" -# Only for `test-support`: lowering PromQL test queries. -asap-frontend-promql = { path = "../frontend-promql", optional = true } [features] # The test helpers (`test_support`) other crates' tests share. -test-support = ["dep:asap-frontend-promql", "asap-types/test-support"] +test-support = [] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } -# Stage 0's memo of the test queries. -asap-stage0-memo = { path = "../stage0-memo" } diff --git a/crates/stage1-pass1-per-computation-candidates/src/lib.rs b/crates/stage1-pass1-per-computation-candidates/src/lib.rs index ec4fb281c..06aa8d712 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/lib.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/lib.rs @@ -7,10 +7,10 @@ //! It depends only on `asap-types` (its input and output memo are in //! `asap_types::stages`) and the shared accuracy model (`asap-accuracy`). //! It depends on no other stage crate, and the planner facade calls it -//! after Stage 0. It depends on no query-language frontend either, except -//! with the `test-support` feature, whose test helpers (`test_support`) -//! lower PromQL test queries through `asap-frontend-promql`; other crates -//! enable that feature only for their tests. +//! after Stage 0. It depends on no query-language frontend either; its own +//! tests lower PromQL through `asap-frontend-promql` (a dev-dependency). +//! The `test-support` feature exports the Pass 1 internals other crates' +//! tests call (`test_support`). //! //! Accuracy guarantees come from the built-in analytical accuracy rules in //! `asap-accuracy` (per-family estimators, composition and budget diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs index 79f48793c..82c1ac4eb 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs @@ -874,7 +874,7 @@ mod tests { /// Guarantee composition through scalar and vector arithmetic (#669). mod guarantee_composition { use super::*; - use crate::test_support::lower_promql; + use crate::test_fixtures::lower_promql; use asap_types::ir::properties::AccuracyTarget; fn with_bound(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { @@ -1397,67 +1397,6 @@ mod tests { } } - fn inventory(query: &str) -> PerComputationCandidates { - let root = crate::test_support::lower_promql( - query, - AccuracyTarget::max_error(0.01).failure_probability(0.001), - ); - // Per-series state ranks each series' identity, which the rows carry - // as a column; a query over aggregated rows is planned over labels. - let root = asap_types::ir::schema::schema_support::with_promql_series_identity(&root) - .unwrap_or(root); - generate_per_computation_candidates( - &asap_stage0_memo::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - } - - fn topk_target(inventory: &PerComputationCandidates) -> usize { - inventory - .targets - .iter() - .position(|t| { - matches!( - inventory.aggregate(t).non_asap(), - Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) - ) - }) - .expect("a top-k target") - } - - /// How many whole-workload choices enumeration keeps: every digit over - /// its target's `choice_count`, less those [`absorbed_choice_varies`] - /// skips. - /// - /// [`absorbed_choice_varies`]: PerComputationCandidates::absorbed_choice_varies - fn enumerated_choices(inventory: &PerComputationCandidates) -> usize { - let mut enumerated = 0; - let mut choice = vec![0; inventory.targets.len()]; - loop { - if !inventory.absorbed_choice_varies(&choice) { - enumerated += 1; - } - let mut carry = true; - for (digit, target) in choice.iter_mut().zip(&inventory.targets).rev() { - *digit += 1; - if *digit < target.choice_count() { - carry = false; - break; - } - *digit = 0; - } - if carry { - return enumerated; - } - } - } - /// Only raw data samples and plain sums of them: arithmetic, `rate` and other /// functions end the requirement. #[test] @@ -1470,7 +1409,7 @@ mod tests { ("rate(m[1m])", None), ("max_over_time(m[1m])", None), ] { - let root = crate::test_support::lower_promql(query, AccuracyTarget::exact()); + let root = crate::test_fixtures::lower_promql(query, AccuracyTarget::exact()); assert_eq!( raw_data_samples_of_metric(&root).as_deref(), metric, @@ -1479,149 +1418,6 @@ mod tests { } } - /// The target whose fused computation is a nested-sum merge. - fn nested_sum_target(inventory: &PerComputationCandidates) -> Option { - let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); - inventory.targets.iter().position(|t| { - t.fused_computation - .as_ref() - .is_some_and(|fused| fused.ways_to_compute == [merge.clone()]) - }) - } - - /// The merge needs a sum over a sum's value: an outer sum over a rate, - /// an outer count, or an outer sum over an inner `max` gets none. - #[test] - fn nested_sum_merge_needs_a_sum_of_sums() { - for query in [ - "sum by (job) (rate(m[5m]))", - "count by (job) (sum_over_time(m[5m]))", - "sum by (job) (max_over_time(m[5m]))", - ] { - assert_eq!(nested_sum_target(&inventory(query)), None, "{query}"); - } - } - - /// A per-series sum drops the metric name, but the raw data samples keep it: - /// grouping the merge by `__name__` would answer the metric's name where - /// the nested sums answer none, so no merge is offered. - #[test] - fn nested_sum_merge_never_groups_by_the_dropped_metric_name() { - let query = "sum by (__name__) (sum_over_time(m[5m]))"; - assert_eq!(nested_sum_target(&inventory(query)), None); - } - - /// Merges do not stack: in `sum(sum by (job) (sum_over_time(m[5m])))` - /// a target with a fused computation is never absorbed, and `combinations` - /// counts exactly the choices enumeration keeps. - #[test] - fn nested_sum_merges_do_not_stack() { - let inventory = inventory("sum(sum by (job) (sum_over_time(m[5m])))"); - let absorbed: Vec = absorbed_targets(&inventory.targets).collect(); - assert_eq!(absorbed.len(), 1, "one merge"); - for &a in &absorbed { - assert!( - inventory.targets[a].fused_computation.is_none(), - "an absorbed target has a fused computation" - ); - } - let enumerated = enumerated_choices(&inventory); - assert_eq!(inventory.combinations(), enumerated); - } - - /// The shapes a heap over the raw data samples cannot answer do not get one: - /// `bottomk` stays a sort and limit (no top-k target), and a grouped sum - /// `without` labels or by the metric name (which the per-series sum drops) - /// gets no heap that absorbs the per-series sum and reads the samples. - #[test] - fn grouped_range_sums_a_raw_data_samples_heap_cannot_rank_get_no_fused_heap() { - let is_topk = |inventory: &PerComputationCandidates, t: &ComputationTarget| { - matches!( - inventory.aggregate(t).non_asap(), - Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) - ) - }; - let bottom = inventory("bottomk(3, sum by (region) (sum_over_time(m[1m])))"); - assert!( - !bottom.targets.iter().any(|t| is_topk(&bottom, t)), - "bottomk has no top-k target" - ); - for query in [ - "topk(3, sum without (instance) (sum_over_time(m[1m])))", - "topk(3, sum by (__name__) (sum_over_time(m[1m])))", - ] { - let inventory = inventory(query); - for target in inventory.targets.iter().filter(|t| is_topk(&inventory, t)) { - assert!( - target - .fused_computation - .as_ref() - .is_none_or(|f| f.absorbed_targets.len() == 1), - "{query}: {:?}", - target.fused_computation - ); - } - } - } - - /// A fused computation picked leaves every target beneath it uncomputed, - /// also one that an absorbed target's own merge absorbs: Pass 1 closes - /// `absorbed_targets` over those, so enumeration keeps only their first choice and - /// `combinations` still counts exactly the choices kept. Here the top-k - /// heap is made to absorb the grouped sum only, whose merge absorbs the - /// per-series sum. - #[test] - fn a_fused_computation_absorbs_what_its_absorbed_targets_merges_absorb() { - let mut inventory = inventory("topk(10, sum by (job) (sum_over_time(m[1m])))"); - let topk = topk_target(&inventory); - let fused = inventory.targets[topk] - .fused_computation - .as_mut() - .expect("a fused heap"); - let grouped = fused.absorbed_targets[0]; - fused.absorbed_targets.truncate(1); - let per_series = inventory.targets[grouped] - .fused_computation - .as_ref() - .expect("the grouped sum's merge") - .absorbed_targets[0]; - close_absorptions(&mut inventory.targets); - assert_eq!( - inventory.targets[topk] - .fused_computation - .as_ref() - .unwrap() - .absorbed_targets, - vec![grouped, per_series] - ); - assert_eq!(inventory.combinations(), enumerated_choices(&inventory)); - } - - /// `combinations` counts each choice once: picking a fused - /// whole-expression candidate with any but the first candidate of the - /// target it absorbs is the same candidate. An absorbed target with three - /// candidates (an approximate inner count: its accumulator, Count-Min and - /// UnivMon) gives 1 exact · 3 + 2 whole-expression = 5, not 3 · 3. - #[test] - fn combinations_skip_choices_a_whole_expression_candidate_absorbs() { - let inventory = inventory("topk by (job) (10, count_over_time(m[1m]))"); - let topk = topk_target(&inventory); - let target = &inventory.targets[topk]; - let Some(fused) = &target.fused_computation else { - panic!("a fused computation: {:?}", target.ways_to_compute) - }; - let inner = fused.absorbed_targets[0]; - let n = target.ways_to_compute.len(); - let whole = fused.ways_to_compute.len(); - let m = inventory.targets[inner].ways_to_compute.len(); - let enumerated = enumerated_choices(&inventory); - assert!(whole > 0 && m > 1, "{n} {whole} {m}"); - assert_eq!(inventory.combinations(), n * m + whole); - assert_eq!(inventory.combinations(), 5); - assert_eq!(inventory.combinations(), enumerated); - } - /// A Count-Min estimate has a bound only over weights proven /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. #[test] @@ -1663,7 +1459,7 @@ mod tests { #[test] fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { let hll = FieldDataType::Sketch; - let input = |query| crate::test_support::lower_promql(query, AccuracyTarget::exact()); + let input = |query| crate::test_fixtures::lower_promql(query, AccuracyTarget::exact()); let guarantee = |family: &FieldDataType, query| { source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) }; diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs index 5985738d0..7111f60f0 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs @@ -1,22 +1,9 @@ //! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. -use crate::per_computation_candidates::{ - computation_candidates_for_intent, generate_per_computation_candidates, - PerComputationCandidateError, -}; -use asap_stage0_memo::memo_of_queries; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use crate::per_computation_candidates::computation_candidates_for_intent; use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{ - DataType, ExactKind, Field, FieldDataType, Schema, SketchAlgorithm, SummaryUpdate, -}; -use asap_types::ir::{ - ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot, ScalarExpr, -}; +use asap_types::ir::schema::SketchAlgorithm; -use asap_types::stages::{HavingEntry, HavingPlacement, HowToComputeAggregateMeasure}; -use std::rc::Rc; +use asap_types::stages::HowToComputeAggregateMeasure; fn candidates_for(intent: AggIntent) -> Vec { computation_candidates_for_intent(&intent).candidates @@ -30,44 +17,6 @@ fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec }) .collect() } -fn scan(nullable_value: bool) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "flows".into(), - }, - predicates: vec![], - schema: Schema::lifted( - vec![ - Field::plain("src_ip", DataType::Utf8, false), - Field::plain("value", DataType::Float64, nullable_value), - ], - None, - ), - })) - .unwrap() -} -fn aggregate_over( - child: Rc, - measures: Vec, - filters: Vec>, - having: Option, -) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child, - reduction: Reduction::by(vec![]), - measures, - output_names: vec![], - filters, - having, - })) - .unwrap() -} -fn aggregate(intent: AggIntent) -> Rc { - aggregate_over(scan(false), vec![intent], vec![], None) -} -fn always() -> Predicate { - Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))) -} /// Example 2 preserves specialized distinct summaries and the universal alternative. #[test] @@ -99,111 +48,6 @@ fn frequency_statistics_keep_universal_choices() { } } -/// Scalar roots expose their producer targets; repeated references retain one target identity. -#[test] -fn scalar_root_producers_are_discovered_once() { - let producer = aggregate(AggIntent::Cardinality { cols: vec![0] }); - let roots = vec![ - QueryRoot::Scalar(ScalarExpr::ScalarSubquery(producer.clone())), - QueryRoot::Operator(producer.clone()), - ]; - let candidates = generate_per_computation_candidates( - &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), - ) - .unwrap(); - assert_eq!(candidates.roots.len(), 2); - for root in &candidates.roots { - root.validate_structure().unwrap(); - } - assert_eq!(candidates.targets.len(), 1); - assert_eq!(**candidates.aggregate(&candidates.targets[0]), *producer); - - assert!(producer.guarantee.is_none()); - producer.validate_structure().unwrap(); -} - -/// #509 Example 1 Q1 is exact: each target's one exact option is its exact -/// accumulator state (grouped sum, rate); no plain exact form and no summary. -/// One PromQL query with the default exact target, lowered through the -/// workload frontend. -fn exact_promql(query: &str) -> Vec { - use asap_types::workload::{ - BatchEntry, PlanningWorkload, Query, QueryLanguage, QueryRequirements, QueryWorkload, - }; - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy: AccuracyTarget::exact(), - ..Default::default() - }, - predictability: Default::default(), - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(asap_types::workload::DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - asap_types::workload::DurationMs(1000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload).unwrap() -} - -#[test] -fn example_1_exact_q1_keeps_exact_accumulators_only() { - let roots = exact_promql("sum by (job) (rate(http_requests_total[1m]))"); - let candidates = generate_per_computation_candidates( - &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), - ) - .unwrap(); - let found: Vec<_> = candidates - .targets - .iter() - .map(|target| target.ways_to_compute.clone()) - .collect(); - assert_eq!( - found, - vec![ - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])], - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Rate - ])], - ] - ); -} - -/// ASAP operators fail at the stage -/// boundary. -#[test] -fn asap_input_is_rejected() { - let summary = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: scan(false), - summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - filter: None, - })) - .unwrap(); - assert!(matches!( - generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(summary)], - &[None], - &Default::default() - ) - .unwrap() - ), - Err(PerComputationCandidateError::ContainsAsap) - )); -} - /// TopK keeps both declared heap substrates without choosing an implementation. #[test] fn topk_keeps_both_specialized_heap_choices() { @@ -225,525 +69,3 @@ fn summary_candidates_are_unsized() { let tight = candidates_for(AggIntent::Count); assert_eq!(loose, tight); } - -/// Each measure of a multi-measure aggregate is its own target; a measure whose -/// only realization is its exact accumulator is still a target. -#[test] -fn multi_measure_aggregate_has_one_target_per_measure() { - let node = aggregate_over( - scan(false), - vec![AggIntent::Sum { col: Some(1) }, AggIntent::Count], - vec![], - None, - ); - let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(node)], &[None], &Default::default()).unwrap(), - ) - .unwrap(); - let candidates = &inventory.targets; - assert_eq!( - candidates - .iter() - .map(|target| target.measure_index) - .collect::>(), - vec![0, 1] - ); - assert_eq!( - candidates[0].ways_to_compute, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Sum - ])] - ); - assert_sql_count_choices(&choices_of(&inventory, 1)); -} - -/// An aggregate with no realization besides exact execution is not a target. -#[test] -fn aggregate_without_realization_is_not_a_target() { - let node = aggregate(AggIntent::StdDev { - col: Some(1), - population: true, - }); - assert!(generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(node)], &[None], &Default::default()).unwrap() - ) - .unwrap() - .targets - .is_empty()); -} - -/// A filtered SQL count keeps its executable accumulator and records its -/// filter, which composition applies to the state. -#[test] -fn filtered_measure_keeps_candidates_and_records_its_filter() { - let count = || AggIntent::Count; - let filtered = aggregate_over( - scan(false), - vec![count(), count()], - vec![Some(always()), None], - None, - ); - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(filtered)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 2); - for (index, target) in targets.iter().enumerate() { - assert_sql_count_choices(&choices_of(&inventory, index)); - assert!(target.having_filters.is_empty()); - } - assert_eq!(targets[0].measure_filter, Some(always())); - assert_eq!(targets[1].measure_filter, None); -} - -/// Target `target`'s own ways to compute. -fn choices_of( - inventory: &crate::per_computation_candidates::PerComputationCandidates, - target: usize, -) -> Vec { - inventory.targets[target].ways_to_compute.clone() -} - -fn assert_sql_count_choices(choices: &[HowToComputeAggregateMeasure]) { - assert_eq!( - choices, - vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![ - ExactKind::Count - ])] - ); -} - -/// `count(*) by (src_ip)`, the form the SQL frontend gives `GROUP BY src_ip`. -fn grouped_count(having: Option) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: Reduction::by(vec![0]), - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![], - having, - })) - .unwrap() -} -/// `HAVING` as the SQL frontend lowers it: a `Filter` above the aggregate. -fn having_filter(child: Rc, pred: Predicate) -> Rc { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Filter { pred, child })).unwrap() -} -/// `column = 'a'` over the aggregate output (0: `src_ip`, 1: the count). -fn equals(column: usize, value: ScalarValue) -> Predicate { - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op: CompareOpKind::Eq, - right: Box::new(ScalarExpr::Literal(value)), - semantics: ExprSemantics::Sql, - }) -} - -/// A `HAVING` over group keys only is pushed below the aggregate and keeps -/// every executable candidate. -#[test] -fn key_only_having_is_pushed_below_and_keeps_candidates() { - let node = grouped_count(None); - let root = having_filter(node.clone(), equals(0, ScalarValue::Utf8("a".into()))); - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(root.clone())], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 1); - assert_eq!(**inventory.aggregate(&targets[0]), *node); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(root.clone()), - predicate: equals(0, ScalarValue::Utf8("a".into())), - placement: HavingPlacement::BelowAggregate, - }] - ); - assert_sql_count_choices(&choices_of(&inventory, 0)); -} - -/// A `HAVING` over an aggregate value, as a `Filter` above or as the -/// aggregate's own `having`, stays above the output and keeps executable candidates. -#[test] -fn value_having_stays_above_output_and_keeps_candidates() { - let value = || equals(1, ScalarValue::Int64(10)); - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &[ - QueryRoot::Operator(having_filter(grouped_count(None), value())), - QueryRoot::Operator(grouped_count(Some(value()))), - ], - &vec![None; 2], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 2); - for (index, target) in targets.iter().enumerate() { - let placements: Vec<_> = target - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!(placements, vec![HavingPlacement::AboveOutput]); - assert_sql_count_choices(&choices_of(&inventory, index)); - } -} - -/// A key-only `HAVING` above a shared aggregate stays above the output when -/// another consumer reads the aggregate without it. -#[test] -fn having_on_one_consumer_of_a_shared_aggregate_stays_above() { - let node = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let filtered = having_filter(node.clone(), key.clone()); - let targets = generate_per_computation_candidates( - &memo_of_queries( - &[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(node.clone()), - ], - &vec![None; 2], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(filtered), - predicate: key.clone(), - placement: HavingPlacement::AboveOutput, - }] - ); - - // Two consumers applying the same key-only predicate push it below. - let first = having_filter(node.clone(), key.clone()); - let second = having_filter(node, key); - let targets = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(first), QueryRoot::Operator(second)], - &[None, None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - .targets; - assert!(targets[0] - .having_filters - .iter() - .all(|entry| entry.placement == HavingPlacement::BelowAggregate)); - // The two equal `Filter`s are one memo group, so one HAVING entry. - assert_eq!(targets[0].having_filters.len(), 1); -} - -/// A scalar reference to a grouped aggregate (`Filter(scan, EXISTS G)` as a -/// second root) is a consumer that reads it without the `HAVING`, so the -/// first root's key-only `HAVING` stays above the output. -#[test] -fn a_scalar_reference_consumer_keeps_a_key_only_having_above() { - let node = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let filtered = having_filter(node.clone(), key.clone()); - let exists = having_filter( - scan(false), - Predicate(ScalarExpr::Exists { - subquery: node.clone(), - negated: false, - }), - ); - let inventory = generate_per_computation_candidates( - &memo_of_queries( - &[ - QueryRoot::Operator(filtered.clone()), - QueryRoot::Operator(exists), - ], - &vec![None; 2], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let targets = &inventory.targets; - let target = targets - .iter() - .find(|target| **inventory.aggregate(target) == *node) - .expect("the grouped count's target"); - assert_eq!( - target.having_filters, - vec![HavingEntry { - filter: Some(filtered), - predicate: key, - placement: HavingPlacement::AboveOutput, - }] - ); -} - -/// Of two stacked `Filter`s above an aggregate, only the adjacent one is -/// its `HAVING`; the outer one filters the inner `Filter`'s output. -#[test] -fn only_the_adjacent_of_stacked_filters_is_having() { - let node = grouped_count(None); - let inner_key = equals(0, ScalarValue::Utf8("a".into())); - let inner = having_filter(node.clone(), inner_key.clone()); - let outer = having_filter(inner.clone(), equals(0, ScalarValue::Utf8("b".into()))); - let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(outer)], &[None], &Default::default()).unwrap(), - ) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(inner), - predicate: inner_key, - placement: HavingPlacement::BelowAggregate, - }] - ); -} - -/// A global aggregate's `HAVING` always stays above the output, even a -/// constant one: the one output row exists with no input rows. -#[test] -fn global_aggregate_having_stays_above() { - let count = || vec![AggIntent::Count]; - let never = Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))); - let targets = generate_per_computation_candidates( - &memo_of_queries( - &[ - QueryRoot::Operator(having_filter( - aggregate_over(scan(false), count(), vec![], None), - never.clone(), - )), - QueryRoot::Operator(aggregate_over(scan(false), count(), vec![], Some(never))), - ], - &vec![None; 2], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - .targets; - assert_eq!(targets.len(), 2); - for target in &targets { - let placements: Vec<_> = target - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!(placements, vec![HavingPlacement::AboveOutput]); - } -} - -/// The `HAVING` entries of the target whose measure satisfies `is`. -fn having_of( - inventory: &crate::per_computation_candidates::PerComputationCandidates, - is: impl Fn(&AggIntent) -> bool, -) -> Vec { - let target = inventory - .targets - .iter() - .find(|target| match inventory.aggregate(target).non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => is(&measures[target.measure_index]), - _ => false, - }) - .expect("target"); - target.having_filters.clone() -} - -/// `Column(column) > ScalarSubquery(subquery)`. -fn above_scalar(column: usize, subquery: Rc) -> Predicate { - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(column)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::ScalarSubquery(subquery)), - semantics: ExprSemantics::Sql, - }) -} - -/// A `Filter` that reads an aggregate only from its predicate is not that -/// aggregate's `HAVING`; one that also filters the aggregate gives one entry. -#[test] -fn scalar_reference_filter_is_not_having() { - let count = || vec![AggIntent::Count]; - let agg = aggregate_over(scan(false), count(), vec![], None); - let root = having_filter(scan(false), above_scalar(1, agg.clone())); - let inventory = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()).unwrap(), - ) - .unwrap(); - let targets = &inventory.targets; - assert_eq!(targets.len(), 1); - assert_eq!(**inventory.aggregate(&targets[0]), *agg); - assert!( - targets[0].having_filters.is_empty(), - "{:?}", - targets[0].having_filters - ); - - let agg = aggregate_over(scan(false), count(), vec![], None); - let root = having_filter(agg.clone(), above_scalar(0, agg.clone())); - let targets = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(root.clone())], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - assert_eq!( - targets[0].having_filters, - vec![HavingEntry { - filter: Some(root), - predicate: above_scalar(0, agg), - placement: HavingPlacement::AboveOutput, - }] - ); -} - -/// PromQL `scalar(...)` operands are scalar references, not `HAVING`. -#[test] -fn promql_scalar_operand_is_not_having() { - let roots = exact_promql("http_requests_total > scalar(sum(http_requests_total))"); - let targets = generate_per_computation_candidates( - &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), - ) - .unwrap() - .targets; - assert!(!targets.is_empty()); - for target in &targets { - assert!( - target.having_filters.is_empty(), - "{:?}", - target.having_filters - ); - } - - let roots = exact_promql("sum by (job) (x) > scalar(count(x))"); - let inventory = generate_per_computation_candidates( - &memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(), - ) - .unwrap(); - assert!(having_of(&inventory, |intent| matches!(intent, AggIntent::Count)).is_empty()); - let sum = having_of(&inventory, |intent| matches!(intent, AggIntent::Sum { .. })); - assert_eq!(sum.len(), 1, "{sum:?}"); - assert_eq!(sum[0].placement, HavingPlacement::AboveOutput); -} - -/// A non-root, non-`Filter` consumer keeps a key-only `HAVING` above. -#[test] -fn having_shared_with_a_project_consumer_stays_above() { - let node = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let filtered = having_filter(node.clone(), key); - let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { - cols: vec![asap_types::ir::ProjectItem { - alias: None, - expr: ScalarExpr::Column(0), - }], - qualifier: None, - child: node, - })) - .unwrap(); - let targets = generate_per_computation_candidates( - &memo_of_queries( - &[QueryRoot::Operator(filtered), QueryRoot::Operator(project)], - &[None, None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap() - .targets; - assert_eq!(targets.len(), 1); - let placements: Vec<_> = targets[0] - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!(placements, vec![HavingPlacement::AboveOutput]); -} - -/// `without(...)` and per-entity aggregates have no fixed key prefix, so a -/// `HAVING` over column 0 stays above. -#[test] -fn without_and_per_entity_having_stays_above() { - for reduction in [ - Reduction::Reduce( - asap_types::ir::operator::operator_properties::GroupKeys::without(vec![1]), - ), - Reduction::PerEntity, - ] { - let node = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - child: scan(false), - reduction: reduction.clone(), - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![], - having: None, - })) - .unwrap(); - let root = having_filter(node, equals(0, ScalarValue::Utf8("a".into()))); - let targets = generate_per_computation_candidates( - &memo_of_queries(&[QueryRoot::Operator(root)], &[None], &Default::default()).unwrap(), - ) - .unwrap() - .targets; - assert_eq!(targets.len(), 1, "{reduction:?}"); - let placements: Vec<_> = targets[0] - .having_filters - .iter() - .map(|entry| entry.placement) - .collect(); - assert_eq!( - placements, - vec![HavingPlacement::AboveOutput], - "{reduction:?}" - ); - } -} - -// Structurally equal aggregates are one memo group with one target, and its -// HAVING placement is decided over every consumer of the group: an unfiltered -// consumer keeps the key-only HAVING above the output for the filtered ones. -#[test] -fn memo_cached_rules_preserve_occurrence_specific_having() { - let shared = grouped_count(None); - let other = grouped_count(None); - let key = equals(0, ScalarValue::Utf8("a".into())); - let roots = vec![ - QueryRoot::Operator(having_filter(shared.clone(), key.clone())), - QueryRoot::Operator(shared), - QueryRoot::Operator(having_filter(other, key)), - ]; - let memo = memo_of_queries(&roots, &vec![None; roots.len()], &Default::default()).unwrap(); - let cached = generate_per_computation_candidates(&memo).unwrap(); - assert_eq!(cached.targets.len(), 1); - // Roots 0 and 2 apply the same `Filter`, one memo group: one entry. - let [entry] = &cached.targets[0].having_filters[..] else { - panic!("one HAVING entry: {:?}", cached.targets[0].having_filters); - }; - assert_eq!(entry.placement, HavingPlacement::AboveOutput); -} diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs index a69c984be..4eb7afb7c 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -5,8 +5,13 @@ use std::rc::Rc; use asap_types::ir::operator::operator_properties::Source; +use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; +use asap_types::workload::{ + BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, + QueryRequirements, QueryWorkload, +}; /// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, /// no unique key. @@ -47,3 +52,34 @@ pub fn scan_from(source: Source, schema: Schema) -> Rc { })) .unwrap() } + +/// `query` lowered by the PromQL frontend, with accuracy target `accuracy`. +pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy, + ..Default::default() + }, + predictability: Predictability::Unknown, + invocations: 1, + execute_at: None, + range_evaluation: None, + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload::sampled_every( + asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), + DurationMs(1_000), + )), + }; + asap_frontend_promql::lower_promql_query_workload(&workload) + .unwrap() + .remove(0) + .as_operator() + .cloned() + .expect("operator root") +} diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_support.rs b/crates/stage1-pass1-per-computation-candidates/src/test_support.rs index 66c497a1a..f44495593 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_support.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_support.rs @@ -1,45 +1,4 @@ -//! Test helpers: the Pass 1 internals and the PromQL lowering other crates' -//! tests call (feature `test-support`), and the fixtures this crate's own -//! tests share (`cfg(test)`). +//! The Pass 1 internals other crates' tests call (feature `test-support`): +//! Pass 1's working form and single-candidate composition. -use std::rc::Rc; - -// Pass 1 internals other crates' tests call: Pass 1's working form. pub use crate::per_computation_candidates::generate_per_computation_candidates; - -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::OperatorNode; -use asap_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; - -pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .remove(0) - .as_operator() - .cloned() - .expect("operator root") -} From d18a86f6719fe55891b7fb8fbac85bccef0ee018 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:03:10 +0000 Subject: [PATCH 23/30] test: delete the frontend and Pass 1 tests that cross stage crates (#1207 PR 0b) Frontend tests that compose Pass 1 candidates and Pass 1 tests that lower PromQL through the frontend are deleted, with the helpers, dev-dependencies and test targets only they used. MetricsQL's test that compares with PromQL's lowering goes too, so no frontend tests depend on another frontend. The crate-graph test checks dev-dependencies between stage crates with no exceptions. --- Cargo.lock | 8 - crates/frontend-metricsql/Cargo.toml | 4 - .../tests/external_corpus.rs | 61 -- .../tests/lowering_semantics.rs | 13 - crates/frontend-metricsql/tests/main.rs | 6 - crates/frontend-promql/Cargo.toml | 3 - .../frontend-promql/tests/count_planning.rs | 11 - crates/frontend-promql/tests/support.rs | 20 - crates/frontend-sql/Cargo.toml | 3 - crates/frontend-sql/tests/frequency_l2.rs | 38 -- crates/integration-tests/tests/crate_graph.rs | 37 +- .../Cargo.toml | 1 - .../src/per_computation_candidates.rs | 590 +----------------- .../src/test_fixtures.rs | 36 -- .../metrics-observability-corpora.md | 5 + 15 files changed, 11 insertions(+), 825 deletions(-) delete mode 100644 crates/frontend-metricsql/tests/external_corpus.rs diff --git a/Cargo.lock b/Cargo.lock index be2d17e18..ec1019f84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -360,9 +360,6 @@ version = "0.1.0" dependencies = [ "asap-frontend-common", "asap-frontend-metricsql", - "asap-frontend-promql", - "asap-stage0-memo", - "asap-stage1-pass1-per-computation-candidates", "asap-types", "metricsql_parser", "thiserror 2.0.18", @@ -375,8 +372,6 @@ dependencies = [ "asap-accuracy", "asap-frontend-common", "asap-planner", - "asap-stage0-memo", - "asap-stage1-pass1-per-computation-candidates", "asap-types", "promql-parser", ] @@ -389,8 +384,6 @@ dependencies = [ "asap-frontend-sql", "asap-planner", "asap-sql-function-catalog", - "asap-stage0-memo", - "asap-stage1-pass1-per-computation-candidates", "asap-types", "datafusion", "serde", @@ -461,7 +454,6 @@ name = "asap-stage1-pass1-per-computation-candidates" version = "0.1.0" dependencies = [ "asap-accuracy", - "asap-frontend-promql", "asap-types", "thiserror 2.0.18", ] diff --git a/crates/frontend-metricsql/Cargo.toml b/crates/frontend-metricsql/Cargo.toml index caa418db2..912119261 100644 --- a/crates/frontend-metricsql/Cargo.toml +++ b/crates/frontend-metricsql/Cargo.toml @@ -22,7 +22,3 @@ path = "tests/main.rs" [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-frontend-metricsql = { path = ".", features = ["test-support"] } -asap-frontend-promql = { path = "../frontend-promql" } -# `support.rs` (shared with frontend-promql's tests) binds through Pass 1. -asap-stage0-memo = { path = "../stage0-memo" } -asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } diff --git a/crates/frontend-metricsql/tests/external_corpus.rs b/crates/frontend-metricsql/tests/external_corpus.rs deleted file mode 100644 index aafe36049..000000000 --- a/crates/frontend-metricsql/tests/external_corpus.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! An out-of-tree MetricsQL corpus: every query lowers and binds without -//! panicking. The queries live outside this repository (e.g. a private -//! corpus); the test reads every `*.txt` in `$ASAP_EXTERNAL_CORPUS_DIR` and -//! passes trivially when it is unset, so CI still compiles it on every PR. -//! -//! Each file holds one query per line (`#` lines are comments) and a -//! `# floor: N` line: the minimum number of its queries that must lower. -//! Raise a floor to the current `lowered` count when coverage improves. -//! -//! `ASAP_EXTERNAL_CORPUS_DIR= cargo test -p asap-frontend-metricsql -//! --test metricsql external_corpus:: -- --nocapture` - -fn queries(corpus: &str) -> impl Iterator { - corpus - .lines() - .map(str::trim) - .filter(|l| !l.is_empty() && !l.starts_with('#')) -} - -/// The `# floor: N` line's `N`. -fn floor(corpus: &str) -> Option { - corpus - .lines() - .find_map(|l| l.trim().strip_prefix("# floor:")) - .and_then(|n| n.trim().parse().ok()) -} - -/// `msg` with sanitized identifiers replaced by `_` and digit runs by `N`, so -/// the same failure in different queries groups under one message. -fn normalize(msg: &str) -> String { - let mut out = String::new(); - let mut word = String::new(); - for c in msg.chars().chain([' ']) { - if c.is_alphanumeric() || c == '_' { - word.push(c); - continue; - } - if word.starts_with("sanitized_") { - out.push('_'); - } else if !word.is_empty() && word.chars().all(|c| c.is_ascii_digit()) { - out.push('N'); - } else { - out.push_str(&word); - } - word.clear(); - out.push(c); - } - out.pop(); - out -} - -#[test] -fn floor_line() { - assert_eq!(floor("# floor: 25\nrate(x[5m])"), Some(25)); - assert_eq!(floor("rate(x[5m])"), None); - assert_eq!( - queries("# floor: 3\n\n up \n# c").collect::>(), - ["up"] - ); - assert_eq!(normalize("sanitized_foo at 123"), "_ at N"); -} diff --git a/crates/frontend-metricsql/tests/lowering_semantics.rs b/crates/frontend-metricsql/tests/lowering_semantics.rs index 37284b71b..c369e4e8e 100644 --- a/crates/frontend-metricsql/tests/lowering_semantics.rs +++ b/crates/frontend-metricsql/tests/lowering_semantics.rs @@ -145,16 +145,3 @@ fn workload_lowering_uses_the_declared_sample_interval() { }); lower_metricsql_query_workload(&at_rest).unwrap(); } - -/// Vector negation lowers to the same IR as PromQL's: a sample-value -/// `Negative` that drops the metric name, not a multiplication by -1. -#[test] -fn vector_negation_lowers_as_in_promql() { - let query = "-rate(up[5m])"; - let metricsql = - lower_metricsql_query_workload(&workload(QueryLanguage::MetricsQL, query)).unwrap(); - let promql = - asap_frontend_promql::lower_promql_query_workload(&workload(QueryLanguage::PromQL, query)) - .unwrap(); - assert_eq!(metricsql, promql); -} diff --git a/crates/frontend-metricsql/tests/main.rs b/crates/frontend-metricsql/tests/main.rs index f3aa9f7c9..0c8315d09 100644 --- a/crates/frontend-metricsql/tests/main.rs +++ b/crates/frontend-metricsql/tests/main.rs @@ -2,12 +2,6 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-frontend-metricsql --test metricsql ::`. -mod external_corpus; mod lowering; mod lowering_semantics; mod victoriametrics_go_golden; - -// The PromQL tests' binding helpers (`logical_plan`), for `external_corpus`. -#[path = "../../frontend-promql/tests/support.rs"] -#[allow(dead_code)] -mod support; diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index 194c97579..63890e6d3 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -19,9 +19,6 @@ promql-parser = { git = "https://github.com/ProjectASAP/promql-parser", rev = "1 [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -# Stage 0's memo and Pass 1's target list (`per_computation_candidates`) for the tests. -asap-stage0-memo = { path = "../stage0-memo" } -asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } # The planning stages through the facade (a dev-dependency cycle, for # integration tests only). asap-planner = { path = "../planner" } diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index a5496d1e4..ee2b2c82d 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -116,14 +116,3 @@ fn count_over_time_counts_scrapes_not_sample_values() { vec![10.0, 10.0] ); } - -// The assertion helper must fail on an injected composition error; its coverage -// must not depend on a real planner bug remaining unfixed. -#[test] -#[should_panic(expected = "does not compose")] -fn logical_candidates_fails_on_a_candidate_that_does_not_compose() { - support::require_composed_operator_candidates(vec![support::LogicalCandidate { - id: "invalid-composition".into(), - roots: Err("invalid choice".into()), - }]); -} diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index fe2867e3f..6867db91d 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -143,23 +143,3 @@ pub struct LogicalCandidate { pub id: String, pub roots: Result, String>, } - -/// Fail on a rejected candidate rather than silently dropping it from assertions. -#[allow(dead_code)] -pub fn require_composed_operator_candidates( - candidates: Vec, -) -> Vec> { - candidates - .into_iter() - .filter_map(|candidate| { - match candidate - .roots - .unwrap_or_else(|e| panic!("{} does not compose: {e}", candidate.id)) - .remove(0) - { - asap_types::ir::QueryRoot::Operator(node) => Some(node), - asap_types::ir::QueryRoot::Scalar(_) => None, - } - }) - .collect() -} diff --git a/crates/frontend-sql/Cargo.toml b/crates/frontend-sql/Cargo.toml index df4998a97..2656f0622 100644 --- a/crates/frontend-sql/Cargo.toml +++ b/crates/frontend-sql/Cargo.toml @@ -36,9 +36,6 @@ test-support = [] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-frontend-sql = { path = ".", features = ["test-support"] } -# Stage 0's memo and Pass 1's target list (`per_computation_candidates`) for the tests. -asap-stage0-memo = { path = "../stage0-memo" } -asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } # The planning stages through the facade (a dev-dependency cycle, for # integration tests only). asap-planner = { path = "../planner" } diff --git a/crates/frontend-sql/tests/frequency_l2.rs b/crates/frontend-sql/tests/frequency_l2.rs index 4111c4770..1a433905c 100644 --- a/crates/frontend-sql/tests/frequency_l2.rs +++ b/crates/frontend-sql/tests/frequency_l2.rs @@ -6,7 +6,6 @@ use asap_frontend_sql::{lower_sql, SqlCatalog}; use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::stages::HowToComputeAggregateMeasure; use std::rc::Rc; fn catalog(nullable: bool) -> SqlCatalog { @@ -103,43 +102,6 @@ async fn frequency_empty_input_guard_uses_an_exact_count() { assert!(matches!(measures.as_slice(), [AggIntent::Count])); } -/// Stage 1 offers the exact option and summary candidates for the -/// recognized L2 intent. -#[tokio::test] -async fn stage1_offers_exact_and_summary_l2_candidates() { - use asap_stage1_pass1_per_computation_candidates::test_support::generate_per_computation_candidates; - use asap_types::ir::QueryRoot; - let root = lower_sql(CAST_COUNT, &catalog(false)).await.unwrap(); - let inventory = generate_per_computation_candidates( - &asap_stage0_memo::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), - ) - .unwrap(); - let l2 = inventory - .targets - .iter() - .find(|t| has_l2(inventory.aggregate(t))) - .expect("L2 target"); - let candidates = &l2.ways_to_compute; - assert!( - candidates.iter().any(|c| matches!( - c, - HowToComputeAggregateMeasure::Exact | HowToComputeAggregateMeasure::ExactAccumulator(_) - )), - "{candidates:?}" - ); - assert!( - candidates - .iter() - .any(|c| matches!(c, HowToComputeAggregateMeasure::Summary(_))), - "{candidates:?}" - ); -} - /// #509 Example 2 over `flows(ts, src_ip)` with its one-minute window: Q2 /// lowers to entropy and the Float64 Q3 to L2; the design's literal Int64 /// Q3 stays relational. diff --git a/crates/integration-tests/tests/crate_graph.rs b/crates/integration-tests/tests/crate_graph.rs index 3f4d90c3a..9e4a15a35 100644 --- a/crates/integration-tests/tests/crate_graph.rs +++ b/crates/integration-tests/tests/crate_graph.rs @@ -19,8 +19,9 @@ //! - **devtools** and **integration-tests** may depend on anything, and no //! other crate may depend on them. //! -//! The edges the rules forbid but the workspace has today are listed, each -//! with its reason, in [`EXCEPTIONS`] and [`DEV_EXCEPTIONS`]. A new workspace crate fails the test +//! The `[dependencies]` edges the rules forbid but the workspace has today are +//! listed, each with its reason, in [`EXCEPTIONS`]; no `[dev-dependencies]` +//! edge between stage crates is excepted. A new workspace crate fails the test //! until it gets a role. use serde_json::Value; @@ -103,36 +104,6 @@ const EXCEPTIONS: &[Exception] = &[ ("asap-planner", "asap-executor"), ]; -/// The `[dev-dependencies]` edges between stage crates the workspace keeps -/// today: the query-language frontends' tests and Pass 1's tests. -const DEV_EXCEPTIONS: &[Exception] = &[ - // Pass 1's tests lower PromQL test queries (`test_fixtures::lower_promql`). - ( - "asap-stage1-pass1-per-computation-candidates", - "asap-frontend-promql", - ), - // The frontends' tests compose Pass 1 candidates of the lowered queries - // (`tests/support.rs`'s `logical_candidates`, `frequency_l2.rs`). - ("asap-frontend-promql", "asap-stage0-memo"), - ( - "asap-frontend-promql", - "asap-stage1-pass1-per-computation-candidates", - ), - ("asap-frontend-sql", "asap-stage0-memo"), - ( - "asap-frontend-sql", - "asap-stage1-pass1-per-computation-candidates", - ), - ("asap-frontend-metricsql", "asap-stage0-memo"), - ( - "asap-frontend-metricsql", - "asap-stage1-pass1-per-computation-candidates", - ), - // The MetricsQL tests share the PromQL tests' `support.rs` and compare - // with the PromQL frontend's lowering. - ("asap-frontend-metricsql", "asap-frontend-promql"), -]; - /// One message per edge of `metadata` (the JSON of /// `cargo metadata --format-version 1 --no-deps`) that breaks a rule, naming /// the edge and the rule: a `[dependencies]` edge that [`rule_violated`] @@ -175,7 +146,7 @@ fn crate_graph_violations(metadata: &Value) -> Vec { "[dev-dependencies]", (from_role == Role::Stage && to_role == Role::Stage) .then_some("a stage crate's tests never call another stage crate"), - DEV_EXCEPTIONS, + &[][..], ), Some(_) => continue, }; diff --git a/crates/stage1-pass1-per-computation-candidates/Cargo.toml b/crates/stage1-pass1-per-computation-candidates/Cargo.toml index 4af9be734..0a4df4c1a 100644 --- a/crates/stage1-pass1-per-computation-candidates/Cargo.toml +++ b/crates/stage1-pass1-per-computation-candidates/Cargo.toml @@ -20,4 +20,3 @@ test-support = [] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } -asap-frontend-promql = { path = "../frontend-promql" } diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs index 82c1ac4eb..3cec2afed 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs @@ -815,22 +815,10 @@ pub(super) fn whole_expression_input( mod tests { use super::*; - use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind}; - use asap_types::ir::properties::{AccuracyGuarantee, AccuracyTarget}; - use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; - use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::schema::{NonNegativeWeightProof, WeightDomain}; - use asap_types::ir::{BinaryOperator, ExprSemantics, Operator, ProjectItem, ScalarExpr}; - - use asap_accuracy::propagation::raw_data_samples_of_metric; - use asap_accuracy::composition::PRODUCT_OPERAND_DOMAINS; + use asap_types::ir::schema::{NonNegativeWeightProof, WeightDomain}; - use asap_accuracy::test_support::{ - composed_guarantee, estimate_has_bound, scalar_guarantee, source_distinct_guarantee, - QUOTIENT_OPERAND_DOMAINS, - }; - use asap_types::ir::properties::guarantee::{AccuracyMetric, BoundExpr, ProbabilityExpr}; + use asap_accuracy::test_support::estimate_has_bound; use asap_types::ir::schema::SketchStatistic; /// Approximate count keeps one exact option, its accumulator, beside the @@ -871,553 +859,6 @@ mod tests { } } - /// Guarantee composition through scalar and vector arithmetic (#669). - mod guarantee_composition { - use super::*; - use crate::test_fixtures::lower_promql; - use asap_types::ir::properties::AccuracyTarget; - - fn with_bound(metric: AccuracyMetric, bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - metric, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - with_bound(AccuracyMetric::ABSOLUTE, bound, delta) - } - fn rel(bound: f64) -> AccuracyGuarantee { - with_bound(AccuracyMetric::Relative, bound, 0.0) - } - /// `sum(m)` carrying `guarantee`. - fn node(guarantee: AccuracyGuarantee) -> Rc { - node_over("sum(m)", guarantee) - } - /// `query` carrying `guarantee`. - fn node_over(query: &str, guarantee: AccuracyGuarantee) -> Rc { - let node = lower_promql(query, AccuracyTarget::exact()); - Rc::new((*node).clone().with_guarantee(Some(guarantee))) - } - fn scalar(guarantee: AccuracyGuarantee) -> ScalarExpr { - ScalarExpr::PromqlScalarFromVector(node(guarantee)) - } - fn lit(value: f64) -> ScalarExpr { - ScalarExpr::literal_f64(value) - } - fn arith(op: ArithmeticOpKind, left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { - ScalarExpr::Arithmetic { - op, - left: Box::new(left), - right: Box::new(right), - semantics: ExprSemantics::Promql, - } - } - fn call(name: &str, args: Vec) -> ScalarExpr { - ScalarExpr::FunctionCall { - name: name.into(), - args, - } - } - fn of(expr: &ScalarExpr) -> Option { - scalar_guarantee(expr, &|_| None) - } - fn bound(g: &AccuracyGuarantee) -> f64 { - g.spec.error.evaluate().expect("a resolved bound") - } - fn delta(g: &AccuracyGuarantee) -> f64 { - g.spec - .failure_probability - .evaluate() - .expect("a resolved probability") - } - fn close(a: f64, b: f64) -> bool { - (a - b).abs() < 1e-12 - } - - /// Absolute errors add through `x ± y`, with the union bound on the - /// failure probability; `−x` and `x ± c` keep the error, `c · x` and - /// `x / c` scale it by `|c|` and `1/|c|`, division by a literal zero - /// has none. - #[test] - fn absolute_errors_compose_through_arithmetic() { - use ArithmeticOpKind::*; - let (x, y) = (scalar(abs(0.01, 0.001)), scalar(abs(0.02, 0.002))); - for op in [Add, Sub] { - let g = of(&arith(op.clone(), x.clone(), y.clone())).unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE, "{op:?}"); - assert!(close(bound(&g), 0.03) && close(delta(&g), 0.003), "{op:?}"); - for shifted in [ - arith(op.clone(), x.clone(), lit(5.0)), - arith(op.clone(), lit(5.0), x.clone()), - arith(op.clone(), x.clone(), ScalarExpr::EvalTimestamp), - ] { - let g = of(&shifted).unwrap(); - assert!(close(bound(&g), 0.01) && close(delta(&g), 0.001), "{op:?}"); - } - } - let negated = ScalarExpr::Negative { - expr: Box::new(x.clone()), - semantics: ExprSemantics::Promql, - }; - assert!(close(bound(&of(&negated).unwrap()), 0.01)); - for (expr, expected) in [ - (arith(Mul, lit(-3.0), x.clone()), 0.03), - (arith(Mul, x.clone(), lit(3.0)), 0.03), - (arith(Div, x.clone(), lit(-4.0)), 0.0025), - ] { - let g = of(&expr).unwrap(); - assert!( - close(bound(&g), expected) && close(delta(&g), 0.001), - "{expr:?}" - ); - } - for none in [ - arith(Div, x.clone(), lit(0.0)), - arith(Mul, x.clone(), ScalarExpr::EvalTimestamp), - arith(Pow, x.clone(), lit(2.0)), - ] { - assert_eq!(of(&none), None, "{none:?}"); - } - } - - /// A product of two estimates has an absolute bound - /// `|x|·e_y + |y|·e_x + e_x·e_y`, and a quotient with an approximate - /// denominator one that needs the denominator bounded away from - /// zero: both need operand domains, so Stage 1 leaves them symbolic - /// for plan selection to bind. Relative errors need no domain: a - /// product has the closed form `(1 + a)(1 + b) − 1`, a quotient - /// `(a + b) / (1 − b)` (`a = 0` for an exact numerator). The failure - /// probability is the union bound. A rank error composes to none. - #[test] - fn products_and_quotients_of_estimates_have_closed_or_symbolic_bounds() { - use ArithmeticOpKind::*; - let unknown = |statistic: &str| BoundExpr::Unknown { - statistic: statistic.into(), - }; - let (x, y) = (scalar(abs(0.01, 0.001)), scalar(abs(0.02, 0.002))); - let product = of(&arith(Mul, x.clone(), y.clone())).unwrap(); - assert_eq!(product.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(product.spec.error, unknown(PRODUCT_OPERAND_DOMAINS)); - assert!(close(delta(&product), 0.003)); - for quotient in [ - arith(Div, x.clone(), y.clone()), - arith(Div, lit(1.0), x.clone()), - arith(Div, x.clone(), scalar(rel(0.01))), - ] { - let g = of("ient).unwrap(); - assert_eq!( - g.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE, - "{quotient:?}" - ); - assert_eq!( - g.spec.error, - unknown(QUOTIENT_OPERAND_DOMAINS), - "{quotient:?}" - ); - } - let ratio = of(&arith( - Div, - scalar(with_bound(AccuracyMetric::Relative, 0.01, 0.001)), - scalar(with_bound(AccuracyMetric::Relative, 0.02, 0.002)), - )) - .unwrap(); - assert_eq!(ratio.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&ratio), (0.01 + 0.02) / (1.0 - 0.02))); - assert!(close(delta(&ratio), 0.003)); - let reciprocal = of(&arith(Div, lit(2.0), scalar(rel(0.02)))).unwrap(); - assert_eq!(reciprocal.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&reciprocal), 0.02 / (1.0 - 0.02))); - // A denominator bound of 1 or more leaves the ratio unbounded. - assert_eq!(of(&arith(Div, scalar(rel(0.01)), scalar(rel(1.0)))), None); - // A relative bound still symbolic in Stage 1 has no ratio rule. - let symbolic = AccuracyGuarantee::new( - AccuracyMetric::Relative, - unknown("values_sign"), - ProbabilityExpr::Zero, - vec![], - ); - assert_eq!(of(&arith(Div, scalar(symbolic), scalar(rel(0.01)))), None); - let relative = of(&arith(Mul, scalar(rel(0.01)), scalar(rel(0.02)))).unwrap(); - assert_eq!(relative.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&relative), 0.01 + 0.02 + 0.0002)); - let rank = scalar(with_bound(AccuracyMetric::Rank, 0.01, 0.001)); - assert_eq!(of(&arith(Mul, rank.clone(), x.clone())), None); - assert_eq!(of(&arith(Div, x, rank)), None); - } - - /// A relative error survives negation and scaling by an exact value, - /// but not a shift (`x + c`) or a sum or difference of estimates - /// (cancellation); a rank error composes to no guarantee (a rank - /// interval counts as none in the MVP). - #[test] - fn relative_and_rank_errors_compose_only_where_sound() { - use ArithmeticOpKind::*; - let x = scalar(rel(0.01)); - for kept in [ - ScalarExpr::Negative { - expr: Box::new(x.clone()), - semantics: ExprSemantics::Promql, - }, - arith(Mul, x.clone(), lit(-2.0)), - arith(Div, x.clone(), ScalarExpr::EvalTimestamp), - ] { - let g = of(&kept).unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative, "{kept:?}"); - assert!(close(bound(&g), 0.01), "{kept:?}"); - } - for none in [ - arith(Add, x.clone(), lit(1.0)), - arith(Sub, x.clone(), scalar(rel(0.01))), - ] { - assert_eq!(of(&none), None, "{none:?}"); - } - let rank = scalar(with_bound(AccuracyMetric::Rank, 0.01, 0.001)); - for none in [ - arith(Sub, rank.clone(), rank.clone()), - arith(Mul, rank.clone(), lit(2.0)), - call("promql_clamp_max", vec![rank, lit(1.0)]), - ] { - assert_eq!(of(&none), None, "{none:?}"); - } - } - - /// `clamp`, `clamp_min`, `clamp_max` and `abs` are 1-Lipschitz and - /// monotone in each argument: the result keeps the largest absolute - /// (or relative) error of its arguments, with the union bound on the - /// failure probability. - #[test] - fn clamp_and_abs_keep_the_largest_argument_error() { - let x = scalar(abs(0.01, 0.001)); - for expr in [ - call("promql_clamp", vec![x.clone(), lit(0.0), lit(10.0)]), - call("promql_clamp_min", vec![x.clone(), lit(0.0)]), - call("promql_clamp_max", vec![x.clone(), lit(10.0)]), - call("promql_abs", vec![x.clone()]), - call("abs", vec![x.clone()]), - ] { - let g = of(&expr).unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE, "{expr:?}"); - assert!( - close(bound(&g), 0.01) && close(delta(&g), 0.001), - "{expr:?}" - ); - } - let bounded = of(&call( - "promql_clamp", - vec![x.clone(), scalar(abs(0.03, 0.002)), lit(10.0)], - )) - .unwrap(); - assert!(close(bound(&bounded), 0.03) && close(delta(&bounded), 0.003)); - let relative = of(&call( - "promql_clamp", - vec![scalar(rel(0.01)), scalar(rel(0.02)), lit(10.0)], - )) - .unwrap(); - assert_eq!(relative.spec.accuracy_metric, AccuracyMetric::Relative); - assert!(close(bound(&relative), 0.02)); - let mixed = call("promql_clamp_min", vec![x, scalar(rel(0.02))]); - assert_eq!(of(&mixed), None); - assert_eq!(of(&call("promql_floor", vec![scalar(rel(0.01))])), None); - } - - /// A comparison over an estimate has no guarantee on its output (a - /// band guarantee is later work); over exact operands it is exact. - #[test] - fn comparisons_over_estimates_have_no_guarantee() { - let compare = |left: ScalarExpr| ScalarExpr::Compare { - left: Box::new(left), - op: CompareOpKind::Gt, - right: Box::new(lit(5.0)), - semantics: ExprSemantics::Promql, - }; - assert_eq!(of(&compare(scalar(abs(0.01, 0.001)))), None); - assert!(of(&compare(scalar(AccuracyGuarantee::exact("exact")))) - .is_some_and(|g| g.is_exact())); - } - - /// A SQL scalar subquery inherits its relation's guarantee, as - /// PromQL `scalar(v)` does. - #[test] - fn a_scalar_subquery_inherits_its_guarantee() { - let g = abs(0.01, 0.001); - let expr = arith( - ArithmeticOpKind::Add, - ScalarExpr::ScalarSubquery(node(g.clone())), - lit(1.0), - ); - assert!(close(bound(&of(&expr).unwrap()), 0.01)); - } - - /// Two estimates count their failure probabilities once each (union - /// bound), even when they read one shared summary: the built-in - /// guarantees hold per estimate, not simultaneously. The same - /// planned estimate read twice fails or holds as one event, so its - /// failure probability counts once. - #[test] - fn failure_probability_counts_once_per_distinct_estimate() { - let shared = node(abs(0.01, 0.001)); - // Equal to `shared`, but planned separately. - let other = node(abs(0.01, 0.001)); - let same = ScalarExpr::PromqlScalarFromVector(shared.clone()); - let twice = of(&arith(ArithmeticOpKind::Add, same.clone(), same.clone())).unwrap(); - assert!(close(bound(&twice), 0.02) && close(delta(&twice), 0.001)); - let distinct = of(&arith( - ArithmeticOpKind::Add, - same, - ScalarExpr::PromqlScalarFromVector(other.clone()), - )) - .unwrap(); - assert!(close(bound(&distinct), 0.02) && close(delta(&distinct), 0.002)); - let vector = |lhs: &Rc, rhs: &Rc| { - let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub), - vector_match: None, - }, - return_bool: false, - lhs: lhs.clone(), - rhs: rhs.clone(), - })) - .unwrap(); - composed_guarantee(&node).unwrap() - }; - assert!(close(delta(&vector(&shared, &shared)), 0.001)); - assert!(close(delta(&vector(&shared, &other)), 0.002)); - } - - /// `left JOIN right ON pred`, composed. - fn join( - kind: JoinKind, - left: &Rc, - right: &Rc, - pred: ScalarExpr, - ) -> OperatorNode { - let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Join { - kind, - pred: Predicate(pred), - left: left.clone(), - right: right.clone(), - })) - .unwrap(); - node.guarantee = composed_guarantee(&node); - node - } - fn column(id: usize) -> ScalarExpr { - ScalarExpr::Column(id) - } - fn equals(left: ScalarExpr, right: ScalarExpr) -> ScalarExpr { - ScalarExpr::Compare { - left: Box::new(left), - op: CompareOpKind::Eq, - right: Box::new(right), - semantics: ExprSemantics::Sql, - } - } - /// `CASE WHEN condition THEN NULL ELSE otherwise END` over `child`, - /// composed. - fn case_over( - child: &OperatorNode, - condition: ScalarExpr, - otherwise: ScalarExpr, - ) -> Option { - let node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols: vec![ProjectItem { - alias: Some("v".into()), - expr: ScalarExpr::Case { - operand: None, - branches: vec![(condition, ScalarExpr::Literal(ScalarValue::Null))], - else_expr: Some(Box::new(otherwise)), - }, - }], - qualifier: None, - child: Rc::new(child.clone()), - })) - .unwrap(); - composed_guarantee(&node) - } - /// An approximate node and an exact one, and the approximate one's - /// column count (its columns come first in a join). - fn approximate_and_exact() -> (Rc, Rc, usize) { - let approximate = node(rel(0.01)); - let exact = node_over("count(m)", AccuracyGuarantee::exact("test")); - let width = approximate.schema.fields.len(); - (approximate, exact, width) - } - - /// A join whose other input is exact and whose predicate reads only - /// exact columns keeps the approximate input's guarantee: a cross - /// join, or a key equality on the exact input's columns. - #[test] - fn join_beside_an_exact_input_on_exact_columns_keeps_the_guarantee() { - let (approximate, exact, width) = approximate_and_exact(); - for (kind, pred) in [ - ( - JoinKind::Cross, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ), - (JoinKind::Inner, equals(column(width), lit(0.0))), - (JoinKind::Left, equals(column(width), lit(0.0))), - ] { - assert_eq!( - join(kind.clone(), &approximate, &exact, pred).guarantee, - Some(rel(0.01)), - "{kind:?}" - ); - } - // Either side. - assert_eq!( - join( - JoinKind::Cross, - &exact, - &approximate, - ScalarExpr::Literal(ScalarValue::Boolean(true)) - ) - .guarantee, - Some(rel(0.01)) - ); - } - - /// A join keyed on an approximate input's column, a semi-join, or - /// two approximate inputs have no guarantee (column-level tracking - /// is #689). - #[test] - fn join_on_an_approximate_key_has_no_guarantee() { - let (approximate, exact, width) = approximate_and_exact(); - assert_eq!( - join( - JoinKind::Inner, - &approximate, - &exact, - equals(column(0), column(width)) - ) - .guarantee, - None - ); - assert_eq!( - join( - JoinKind::Semi, - &exact, - &approximate, - ScalarExpr::Literal(ScalarValue::Boolean(true)) - ) - .guarantee, - None - ); - assert_eq!( - join( - JoinKind::Cross, - &approximate, - &approximate, - ScalarExpr::Literal(ScalarValue::Boolean(true)) - ) - .guarantee, - None - ); - } - - /// A `CASE` whose condition reads only exact columns keeps each - /// branch's guarantee (a NULL branch is exact); one testing an - /// approximate column has none. - #[test] - fn case_on_exact_columns_keeps_the_branch_guarantee() { - let (approximate, exact, width) = approximate_and_exact(); - let joined = join( - JoinKind::Cross, - &approximate, - &exact, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ); - let value = approximate.schema.fields.len() - 1; - assert_eq!( - case_over(&joined, equals(column(width), lit(0.0)), column(value)), - Some(rel(0.01)) - ); - assert_eq!( - case_over(&joined, equals(column(value), lit(0.0)), column(value)), - None - ); - } - - /// `node` projecting `cols` over `child`, composed. - fn project(child: &OperatorNode, cols: Vec) -> OperatorNode { - let mut node = OperatorNode::new(Operator::NonASAP(NonASAPOp::Project { - cols: cols - .into_iter() - .enumerate() - .map(|(i, expr)| ProjectItem { - alias: Some(format!("c{i}")), - expr, - }) - .collect(), - qualifier: None, - child: Rc::new(child.clone()), - })) - .unwrap(); - node.guarantee = composed_guarantee(&node); - node - } - - /// Over a join, a projection reads each column's own guarantee, - /// not the child's sample value (which may be an exact input's - /// column): `sum(m) * 1000` beside the exact side's `value` has - /// the scaled error 1000, not the approximate side's 1. A - /// projection over that computed projection has no per-column - /// guarantees, so it has none (#689). - #[test] - fn projection_over_a_join_reads_each_columns_guarantee() { - let approximate = node(abs(1.0, 0.01)); - let exact = node_over("m", AccuracyGuarantee::exact("test")); - let joined = join( - JoinKind::Cross, - &approximate, - &exact, - ScalarExpr::Literal(ScalarValue::Boolean(true)), - ); - assert_eq!(joined.guarantee, Some(abs(1.0, 0.01))); - let sum = approximate.schema.fields.len() - 1; - let value = approximate.schema.fields.len() + exact.schema.fields.len() - 1; - let scaled = project( - &joined, - vec![ - arith(ArithmeticOpKind::Mul, column(sum), lit(1000.0)), - column(value), - ], - ); - assert_eq!(scaled.guarantee.as_ref().map(bound), Some(1000.0)); - let stacked = project( - &scaled, - vec![arith(ArithmeticOpKind::Add, column(1), lit(1.0)), column(0)], - ); - assert_eq!(stacked.guarantee, None); - } - } - - /// Only raw data samples and plain sums of them: arithmetic, `rate` and other - /// functions end the requirement. - #[test] - fn raw_data_samples_of_metric_ends_at_any_other_operator() { - for (query, metric) in [ - ("sum_over_time(m[1m])", Some("m")), - ("sum by (job) (sum_over_time(m[1m]))", Some("m")), - ("sum_over_time(m[1m]) - 100", None), - ("-sum_over_time(m[1m])", None), - ("rate(m[1m])", None), - ("max_over_time(m[1m])", None), - ] { - let root = crate::test_fixtures::lower_promql(query, AccuracyTarget::exact()); - assert_eq!( - raw_data_samples_of_metric(&root).as_deref(), - metric, - "{query}" - ); - } - } - /// A Count-Min estimate has a bound only over weights proven /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. #[test] @@ -1450,31 +891,4 @@ mod tests { &query )); } - - /// Classic HLL's failure probability is left symbolic in a source's - /// distinct bound only for one instance per group over that source's - /// own rows (#693's scope condition): not over a join of two sources, - /// not over computed rows, and not for one Hydra structure shared by - /// every group, whose distinct items the source's bound does not bound. - #[test] - fn classic_hll_confidence_needs_one_instance_over_a_sources_own_rows() { - let hll = FieldDataType::Sketch; - let input = |query| crate::test_fixtures::lower_promql(query, AccuracyTarget::exact()); - let guarantee = |family: &FieldDataType, query| { - source_distinct_guarantee(family, &SketchStatistic::Cardinality, &input(query)) - }; - let own = hll(SketchAlgorithm::Hll); - let symbolic = guarantee(&own, "a").expect("a source's own rows"); - assert_eq!( - symbolic.spec.failure_probability, - ProbabilityExpr::Unknown { - statistic: asap_accuracy::estimators::hll::classic_failure_statistic(15, 0.05, "a") - } - ); - for (query, over) in [("a * on() b", "a join"), ("a * 2", "computed rows")] { - assert!(guarantee(&own, query).is_none(), "{over}: {query}"); - } - let hydra = hll(SketchAlgorithm::HydraHll); - assert!(guarantee(&hydra, "a").is_none()); - } } diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs index 4eb7afb7c..a69c984be 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -5,13 +5,8 @@ use std::rc::Rc; use asap_types::ir::operator::operator_properties::Source; -use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::workload::{ - BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Predictability, Query, QueryLanguage, - QueryRequirements, QueryWorkload, -}; /// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, /// no unique key. @@ -52,34 +47,3 @@ pub fn scan_from(source: Source, schema: Schema) -> Rc { })) .unwrap() } - -/// `query` lowered by the PromQL frontend, with accuracy target `accuracy`. -pub fn lower_promql(query: &str, accuracy: AccuracyTarget) -> Rc { - let workload = PlanningWorkload { - query_workload: QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some(vec![BatchEntry { - query: Query(query.into()), - requirements: QueryRequirements { - accuracy, - ..Default::default() - }, - predictability: Predictability::Unknown, - invocations: 1, - execute_at: None, - range_evaluation: None, - }]), - repeating_queries: None, - }, - data_workload: Some(DataWorkload::sampled_every( - asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), - DurationMs(1_000), - )), - }; - asap_frontend_promql::lower_promql_query_workload(&workload) - .unwrap() - .remove(0) - .as_operator() - .cloned() - .expect("operator root") -} diff --git a/docs/develop_docs/metrics-observability-corpora.md b/docs/develop_docs/metrics-observability-corpora.md index f9684a0ec..181f75045 100644 --- a/docs/develop_docs/metrics-observability-corpora.md +++ b/docs/develop_docs/metrics-observability-corpora.md @@ -1,5 +1,10 @@ # Metrics-observability corpus coverage +> **Status.** The executable corpus test, `metrics_observability.rs`, was deleted +> when each planning stage became its own crate: a frontend crate's tests may not +> call Stage 0 or Stage 1. The fixtures stay, for the Pass 1 coverage tests that +> replace it. The sections below describe the deleted test. + This note describes the PromQL corpora derived from the metrics-observability benchmark dump and the commands used to reproduce their measurements. The audience is developers evaluating changes to the PromQL front end or ASAP From a4290eeef846473be7e5e1505ca9dd59ae30718f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:11:30 +0000 Subject: [PATCH 24/30] test: delete the frontend tests that plan through the facade (#1207 PR 0b) The PromQL test that planned through asap-planner is deleted with its planner_support helpers; the PromQL and SQL frontends drop their asap-planner dev-dependency, and PromQL its unused asap-accuracy one. The crate-graph test now also forbids a stage crate's dev-dependency on the facade. --- Cargo.lock | 3 -- crates/frontend-promql/Cargo.toml | 4 -- .../frontend-promql/tests/count_planning.rs | 24 ---------- crates/frontend-promql/tests/main.rs | 1 - .../frontend-promql/tests/planner_support.rs | 45 ------------------ crates/frontend-promql/tests/support.rs | 46 ------------------- crates/frontend-sql/Cargo.toml | 3 -- crates/integration-tests/tests/crate_graph.rs | 39 +++++++++++++--- 8 files changed, 32 insertions(+), 133 deletions(-) delete mode 100644 crates/frontend-promql/tests/planner_support.rs diff --git a/Cargo.lock b/Cargo.lock index ec1019f84..6b77c8a73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,9 +369,7 @@ dependencies = [ name = "asap-frontend-promql" version = "0.1.0" dependencies = [ - "asap-accuracy", "asap-frontend-common", - "asap-planner", "asap-types", "promql-parser", ] @@ -382,7 +380,6 @@ version = "0.1.0" dependencies = [ "asap-frontend-common", "asap-frontend-sql", - "asap-planner", "asap-sql-function-catalog", "asap-types", "datafusion", diff --git a/crates/frontend-promql/Cargo.toml b/crates/frontend-promql/Cargo.toml index 63890e6d3..c65fb30ff 100644 --- a/crates/frontend-promql/Cargo.toml +++ b/crates/frontend-promql/Cargo.toml @@ -18,10 +18,6 @@ promql-parser = { git = "https://github.com/ProjectASAP/promql-parser", rev = "1 [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } -# The planning stages through the facade (a dev-dependency cycle, for -# integration tests only). -asap-planner = { path = "../planner" } # Corpus tests live in domain folders (observability) but must still be declared # as test targets — Cargo only auto-discovers `.rs` files directly in `tests/`. diff --git a/crates/frontend-promql/tests/count_planning.rs b/crates/frontend-promql/tests/count_planning.rs index ee2b2c82d..3c4d6095c 100644 --- a/crates/frontend-promql/tests/count_planning.rs +++ b/crates/frontend-promql/tests/count_planning.rs @@ -1,32 +1,8 @@ //! Query text through summary selection: counts use observations, never value weights. -use crate::planner_support::selected_dag; use crate::support; use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::schema::{ExactKind, FieldDataType}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; use support::lower_promql; -// Exact series and temporal counts must select a count accumulator, not distinct or sum. -#[test] -fn exact_counts_select_count_accumulators() { - for query in ["count(up)", "count by(job)(up)", "count_over_time(up[5m])"] { - let root = lower_promql(query, AccuracyTarget::exact()).unwrap(); - let selected = selected_dag(root, AccuracyTarget::exact()); - assert!( - OperatorNode::reachable(&selected).iter().any(|node| { - matches!( - &node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(ExactKind::Count, _), - .. - }) - ) - }), - "{query}: {selected:?}" - ); - } -} - // Fixtures are already selected at one instant or within one five-minute window. // This narrow test oracle interprets the emitted aggregate, not Prometheus ingestion, // staleness, or scrape scheduling. Unsupported plan shapes fail explicitly. diff --git a/crates/frontend-promql/tests/main.rs b/crates/frontend-promql/tests/main.rs index b962cb00b..6055452b4 100644 --- a/crates/frontend-promql/tests/main.rs +++ b/crates/frontend-promql/tests/main.rs @@ -2,7 +2,6 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-frontend-promql --test promql ::`. -mod planner_support; mod support; mod count_planning; diff --git a/crates/frontend-promql/tests/planner_support.rs b/crates/frontend-promql/tests/planner_support.rs deleted file mode 100644 index 04aa13096..000000000 --- a/crates/frontend-promql/tests/planner_support.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! Planner-facade helpers for this crate's tests. Kept out of `support.rs`, -//! which the private corpus also compiles without depending on asap-planner. -use std::rc::Rc; - -use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::OperatorNode; - -use crate::support::declared_data; - -/// Every planning stage's output for the single query `root` at `accuracy`, -/// with the built-in models. -pub fn run_all_stages( - root: Rc, - accuracy: AccuracyTarget, -) -> asap_planner::PlanningRun { - let data = declared_data(&root); - asap_planner::run_planning( - vec![asap_types::ir::QueryRoot::Operator(root)], - &[asap_types::workload::QueryWorkloadEntry { - query: Default::default(), - requirements: asap_types::workload::QueryRequirements { - accuracy, - max_latency_ms: None, - }, - predictability: Default::default(), - recurrence: Default::default(), - range_evaluation: None, - }], - &data, - asap_planner::PlanningModels::builtin(), - asap_planner::PlanSearchSettings::default(), - ) - .expect("Pass 1 accepts the query") -} - -/// The logical DAG the planner pipeline selects for the single query `root` -/// at `accuracy`. -pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { - let run = run_all_stages(root, accuracy); - let (plan, _) = run.selection.as_ref().expect("a plan is selected"); - match plan.query_roots().unwrap().as_slice() { - [asap_types::ir::QueryRoot::Operator(node)] => Rc::clone(node), - other => panic!("one operator root: {other:?}"), - } -} diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 6867db91d..185c99e38 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -76,16 +76,6 @@ pub fn promql_scalar(node: &ScalarExpr) -> Option { } } -/// Time `root` under the default physical properties assignment (every summary -/// at query time) and export the physical DAG — export needs every node -/// timed first. -#[allow(dead_code)] -pub fn physical_asap_dag( - root: &Rc, -) -> asap_types::ir::physical_export::PhysicalASAPDAG { - asap_types::test_support::physical_asap_dag_at_query_time(root).expect("post-ASAP DAG export") -} - #[allow(dead_code)] pub fn scalar_root(query: &str) -> ScalarExpr { match lower_promql_query_workload(&workload(query, AccuracyTarget::exact())) @@ -107,39 +97,3 @@ pub fn sample_expression(node: &OperatorNode) -> &ScalarExpr { other => panic!("expected sample expression, got {other:?}"), } } - -/// The data these tests plan over, declared: 1,000 rows arriving per -/// second, and each metric `root` reads holding 1,000 series. -#[allow(dead_code)] -pub fn declared_data(root: &Rc) -> DataWorkload { - let sources = OperatorNode::reachable(root) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { - source: asap_types::ir::operator::Source::TimeSeries { metric }, - .. - }) => Some(( - metric.clone(), - asap_types::workload::SourceFacts { - series_count: Some(1_000), - ..Default::default() - }, - )), - _ => None, - }) - .collect(); - DataWorkload { - arrival: asap_types::workload::DataArrival::Ingesting { - rows_per_second: 1_000.0, - max_out_of_order_ms: None, - }, - sources, - } -} - -/// A composed logical candidate, or why it does not compose. -#[allow(dead_code)] -pub struct LogicalCandidate { - pub id: String, - pub roots: Result, String>, -} diff --git a/crates/frontend-sql/Cargo.toml b/crates/frontend-sql/Cargo.toml index 2656f0622..70b1333eb 100644 --- a/crates/frontend-sql/Cargo.toml +++ b/crates/frontend-sql/Cargo.toml @@ -36,9 +36,6 @@ test-support = [] [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-frontend-sql = { path = ".", features = ["test-support"] } -# The planning stages through the facade (a dev-dependency cycle, for -# integration tests only). -asap-planner = { path = "../planner" } tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } # bgp_jan2024_workload corpus is sourced verbatim as YAML (ASAPQuery PR #561) # rather than transcribed into the flat .sql shape the other corpora use. diff --git a/crates/integration-tests/tests/crate_graph.rs b/crates/integration-tests/tests/crate_graph.rs index 9e4a15a35..1dcb0ec12 100644 --- a/crates/integration-tests/tests/crate_graph.rs +++ b/crates/integration-tests/tests/crate_graph.rs @@ -2,8 +2,9 @@ //! //! The test reads `cargo metadata --format-version 1 --no-deps` and checks every //! `[dependencies]` edge between two workspace crates, and every -//! `[dev-dependencies]` edge between two stage crates (a test of one stage -//! never calls another stage). Build-dependencies are not checked. Each crate +//! `[dev-dependencies]` edge from a stage crate to another stage crate or to +//! the facade (a test of one stage never calls another stage, directly or +//! through `asap-planner`). Build-dependencies are not checked. Each crate //! has a role ([`role_of`]), and [`rule_violated`] says which role may depend //! on which: //! @@ -21,7 +22,7 @@ //! //! The `[dependencies]` edges the rules forbid but the workspace has today are //! listed, each with its reason, in [`EXCEPTIONS`]; no `[dev-dependencies]` -//! edge between stage crates is excepted. A new workspace crate fails the test +//! edge is excepted. A new workspace crate fails the test //! until it gets a role. use serde_json::Value; @@ -107,7 +108,8 @@ const EXCEPTIONS: &[Exception] = &[ /// One message per edge of `metadata` (the JSON of /// `cargo metadata --format-version 1 --no-deps`) that breaks a rule, naming /// the edge and the rule: a `[dependencies]` edge that [`rule_violated`] -/// forbids, or a `[dev-dependencies]` edge between two stage crates. +/// forbids, or a `[dev-dependencies]` edge from a stage crate to another +/// stage crate or the facade. fn crate_graph_violations(metadata: &Value) -> Vec { let packages = metadata["packages"] .as_array() @@ -144,8 +146,11 @@ fn crate_graph_violations(metadata: &Value) -> Vec { ), Some("dev") => ( "[dev-dependencies]", - (from_role == Role::Stage && to_role == Role::Stage) - .then_some("a stage crate's tests never call another stage crate"), + (from_role == Role::Stage && matches!(to_role, Role::Stage | Role::Facade)) + .then_some( + "a stage crate's tests never call another stage crate, \ + directly or through the facade", + ), &[][..], ), Some(_) => continue, @@ -249,7 +254,27 @@ fn checker_rejects_a_dev_dependency_between_stage_crates() { crate_graph_violations(&metadata), vec![ "forbidden [dev-dependencies] edge asap-stage1-pass1-per-computation-candidates -> \ - asap-stage0-memo: a stage crate's tests never call another stage crate (#1054)" + asap-stage0-memo: a stage crate's tests never call another stage crate, \ + directly or through the facade (#1054)" + .to_string() + ] + ); +} + +#[test] +fn checker_rejects_a_frontend_dev_dependency_on_the_facade() { + let metadata = serde_json::json!({"packages": [ + package("asap-types", &[], &[]), + package("asap-planner", &["asap-types"], &[]), + // Forbidden: the PromQL frontend's tests plan through the facade. + package("asap-frontend-promql", &["asap-types"], &["asap-planner"]), + ]}); + assert_eq!( + crate_graph_violations(&metadata), + vec![ + "forbidden [dev-dependencies] edge asap-frontend-promql -> asap-planner: \ + a stage crate's tests never call another stage crate, directly or through \ + the facade (#1054)" .to_string() ] ); From 13063ab218c5c787558ddd94feeb320e9891615c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:18:23 +0000 Subject: [PATCH 25/30] refactor(types): properties and requirements from #1207 (#1207 PR 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LogicalProperties::new(expression, parameters): a sketch build (a chain included) needs θ from parameter_list(algorithm) of its FieldDataType::Sketch(algorithm); an exact operator has none. - parameter_list(algorithm): structures only. Count-Min lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, 81, 117 and 155 (its median-of-rows δ levels 1e-2 … 1e-5); heap variants each of those with the 1-2-5 heap sizes; UnivMon 40 entries; KLL, DDSketch, HLL, KMV one entry per ε level. A test keeps the lists equal to today's sizing, which is test-only. - PhysicalProperties::new rejects (IngestionTime, OneEvaluation); producer_satisfies_physical_requirements(producer, consumer_timing). - PhysicalCandidate::delivers_on(candidates, context): None in every context, OneEvaluation only in Evaluation contexts, a chain with AcrossEvaluations when its width divides the grid's period and phase (or the refresh window's width and phase), any other AcrossEvaluations result everywhere when its group does not depend on the evaluation time, else when its group's evaluation grid holds the context's times; Materialized(v) contexts stay todo!() for PR 7. - Chains: one schema (window_end at column 0, keys, state), TimeRange and TimeShift over a chain select windows (an instant TimeRange is rejected), a SummaryMerge over them finalizes and rolls up as the chain's build over that range (the two PR 0 todos). - summary_type_of_operator; QueryRecurrence::evaluation_grid_ms is None for a fixed interval without a phase. --- crates/accuracy/src/estimators/cardinality.rs | 2 + crates/accuracy/src/estimators/cms.rs | 1 + .../accuracy/src/estimators/count_sketch.rs | 2 + crates/accuracy/src/estimators/ddsketch.rs | 1 + crates/accuracy/src/estimators/hll.rs | 1 + crates/accuracy/src/estimators/kll.rs | 2 + crates/accuracy/src/estimators/mod.rs | 109 ++++- crates/accuracy/src/estimators/univmon.rs | 10 + crates/accuracy/src/propagation.rs | 95 ++-- crates/accuracy/src/sizing.rs | 31 +- crates/accuracy/src/test_support.rs | 20 +- crates/types/src/ir/operator/asap.rs | 172 +++++-- crates/types/src/ir/operator/non_asap.rs | 25 + .../src/ir/operator/tumbling_window_chain.rs | 36 +- crates/types/src/physical/properties.rs | 61 +-- crates/types/src/stages/candidates.rs | 459 +++++++++++++++++- .../src/stages/physical/memo_candidates.rs | 393 ++++++++++++++- crates/types/src/workload/mod.rs | 112 +++-- crates/types/tests/tumbling_window_chain.rs | 214 +++++++- 19 files changed, 1501 insertions(+), 245 deletions(-) diff --git a/crates/accuracy/src/estimators/cardinality.rs b/crates/accuracy/src/estimators/cardinality.rs index a437da6de..5e03bd0b6 100644 --- a/crates/accuracy/src/estimators/cardinality.rs +++ b/crates/accuracy/src/estimators/cardinality.rs @@ -33,6 +33,7 @@ use super::*; /// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. const KMV_MIN_K: u32 = 2; /// The largest `k` sizing returns. +#[cfg(test)] const KMV_MAX_K: u32 = 1 << 26; pub(super) fn guarantee( @@ -81,6 +82,7 @@ pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { /// The smallest `k` whose KMV failure probability at relative error `eps` /// is at most `delta` ([`kmv_failure_probability`]), up to `2^26`; a /// degenerate `eps` or `delta` saturates to `2^26`. +#[cfg(test)] pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return KMV_MAX_K; diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/estimators/cms.rs index 11f78d4ac..77d4dcd56 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/estimators/cms.rs @@ -25,6 +25,7 @@ pub(super) fn guarantee( } /// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. +#[cfg(test)] pub(crate) fn cms_width(eps: f64) -> u32 { saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) } diff --git a/crates/accuracy/src/estimators/count_sketch.rs b/crates/accuracy/src/estimators/count_sketch.rs index 5cda666b1..9e027377c 100644 --- a/crates/accuracy/src/estimators/count_sketch.rs +++ b/crates/accuracy/src/estimators/count_sketch.rs @@ -57,12 +57,14 @@ fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { } /// CountSketch `L2` point-query width: ε = sqrt(3/w). +#[cfg(test)] pub(crate) fn count_sketch_width(eps: f64) -> u32 { saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) } /// The smallest positive odd depth whose binomial median failure /// probability at `width` is at most `delta`, capped at 255. +#[cfg(test)] pub(crate) fn count_sketch_depth(width: u32, delta: f64) -> u32 { if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { return 255; diff --git a/crates/accuracy/src/estimators/ddsketch.rs b/crates/accuracy/src/estimators/ddsketch.rs index 9cde75477..5f1a7310c 100644 --- a/crates/accuracy/src/estimators/ddsketch.rs +++ b/crates/accuracy/src/estimators/ddsketch.rs @@ -20,6 +20,7 @@ pub(super) fn guarantee( )) } +#[cfg(test)] pub(super) fn size_params(epsilon: f64) -> SketchParams { SketchParams::DDSketch { alpha: epsilon } } diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/estimators/hll.rs index efaef2a51..066a991b5 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/estimators/hll.rs @@ -255,6 +255,7 @@ pub fn min_precision( /// bound up to `⌈1/ε⌉` (18 when none up to 18 does). Pass 1 then raises /// the precision by the same rule ([`min_precision`]) when the source's own /// bound or a smaller failure probability needs it. +#[cfg(test)] pub(crate) fn hll_precision(eps: f64, delta: f64) -> u8 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return 18; diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index a0303c103..f7f76ff28 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -52,6 +52,7 @@ pub(crate) fn kll_rank_error_99(k: u32) -> f64 { /// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. Capped at /// [`KLL_MAX_K`], the largest `k` `asap_sketchlib` builds; an `eps` that /// needs more gets that `k`, whose guarantee then misses the target. +#[cfg(test)] pub(crate) fn kll_k(eps: f64) -> u32 { saturating_ceil( (KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / KLL_RANK_ERROR_EXPONENT_99), @@ -133,6 +134,7 @@ fn normal_quantile(p: f64) -> f64 { /// `k` for rank error `eps` at failure probability `delta`: the empirical /// 99% calibration for `delta >= 0.01`; below, the calibration at /// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. +#[cfg(test)] pub(crate) fn size_params(eps: f64, delta: f64) -> SketchParams { if delta.is_nan() || delta <= 0.0 || delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { return SketchParams::Kll { k: kll_k(eps) }; diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 3f4045d98..315a64ff8 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -1,5 +1,6 @@ //! Dispatch committed estimator parameters to their accuracy models. use super::*; +#[cfg(test)] use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::SummaryTypeCapability; @@ -80,7 +81,10 @@ pub fn theoretical_guarantee( } } -#[allow(dead_code)] // Read again for the predefined θ lists (`parameter_list`), in #1207 PR 1. +/// Today's sizing: the parameters of algorithm `kind` for error `eps` at failure +/// probability `delta`. Test-only: the predefined lists (`parameter_list`) +/// are this sizing over an error grid, which a test keeps equal. +#[cfg(test)] pub(crate) fn size_params( kind: SketchAlgorithm, intent: &AggIntent, @@ -185,6 +189,7 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { /// Accuracy-dependent candidate budget, not a completeness theorem. The /// membership model must still certify the selected set independently. +#[cfg(test)] pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { u32::try_from(k) .unwrap_or(u32::MAX) @@ -204,6 +209,108 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { mod tests { use super::*; + /// Each algorithm's predefined list (`parameter_list`) is today's sizing + /// over the error grid ε = 0.1 … 0.001 at failure probability 0.01, + /// smallest first, without repeats; KMV's and UnivMon's distinct counts + /// stop at ε = 0.01. Count-Min lists each sized width with depths 5, 7, + /// 10 and 12, Count Sketch with 47, 81, 117 and 155, and a heap variant + /// each of those with + /// every heap size of the 1-2-5 series 10 … 10 000. UnivMon's list also + /// combines each L2 shape with each distinct-count shape (28 entries). + #[test] + fn parameter_list_is_the_sizing_over_the_error_grid() { + use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; + use asap_types::stages::parameter_list; + let full_grid = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; + let distinct_grid = [0.1, 0.05, 0.02, 0.01]; + let cms_depths = [5, 7, 10, 12]; + let count_sketch_depths = [47, 81, 117, 155]; + let heap_sizes = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; + let l2 = AggIntent::FrequencyL2 { col: None }; + let top_1 = AggIntent::TopK { k: 1 }; + let quantile = default_quantile(0.5); + let distinct = default_cardinality(); + for (algorithm, intents) in [ + (SketchAlgorithm::Kll, vec![&quantile]), + (SketchAlgorithm::DDSketch, vec![&quantile]), + (SketchAlgorithm::Hll, vec![&distinct]), + (SketchAlgorithm::Kmv, vec![&distinct]), + (SketchAlgorithm::Cms, vec![&distinct]), + (SketchAlgorithm::CountSketch, vec![&distinct]), + (SketchAlgorithm::CmsWithHeap, vec![&top_1]), + (SketchAlgorithm::CountSketchWithHeap, vec![&top_1]), + (SketchAlgorithm::UnivMon, vec![&l2, &distinct, &quantile]), + ] { + let mut sized: Vec = Vec::new(); + for intent in intents { + let grid: &[f64] = match (algorithm, intent) { + ( + SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon, + AggIntent::Cardinality { .. }, + ) => &distinct_grid, + _ => &full_grid, + }; + for &eps in grid { + let listed: Vec = match size_params(algorithm, intent, eps, 0.01) + { + SketchParams::Cms { width, .. } => cms_depths + .iter() + .map(|&depth| SketchParams::Cms { width, depth }) + .collect(), + SketchParams::CountSketch { width, .. } => count_sketch_depths + .iter() + .map(|&depth| SketchParams::CountSketch { width, depth }) + .collect(), + SketchParams::CmsWithHeap { width, .. } => cms_depths + .iter() + .flat_map(|&depth| { + heap_sizes + .iter() + .map(move |&heap_size| SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }) + }) + .collect(), + SketchParams::CountSketchWithHeap { width, .. } => count_sketch_depths + .iter() + .flat_map(|&depth| { + heap_sizes.iter().map(move |&heap_size| { + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + } + }) + }) + .collect(), + other => vec![other], + }; + for params in listed { + if !sized.contains(¶ms) { + sized.push(params); + } + } + } + } + let list = parameter_list(algorithm); + let (listed_sized, combined): (Vec<_>, Vec<_>) = + list.iter().partition(|params| sized.contains(params)); + assert_eq!( + format!("{listed_sized:?}"), + format!("{:?}", sized.iter().collect::>()), + "{algorithm:?}" + ); + let expected_combined = if algorithm == SketchAlgorithm::UnivMon { + 28 + } else { + 0 + }; + assert_eq!(combined.len(), expected_combined, "{algorithm:?}"); + } + } + /// Classic HLL's precision grows until its failure probability over the /// source's distinct bound meets the target, up to 18. #[test] diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index 2bf1d8a56..c0545c20c 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -17,18 +17,22 @@ //! (OctoSketch aggregators fed promoted deltas) are not covered: the //! executor rejects them. use super::*; +#[cfg(test)] use asap_types::ir::operator::AggIntent; /// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and /// seed 19 picks the layers a key reaches. +#[cfg(test)] pub(crate) const MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; /// The most rows an L2 sketch is sized with: the deepest odd shape within /// the 20 rows the executor builds ([`hash_independent`]), so the row median /// is one row's estimate, and within the hash budget at the more than 20 /// columns an L2 guarantee needs (5 column bits: 19 · 6 = 114 ≤ 128). +#[cfg(test)] const MAX_L2_ROWS: u32 = 19; /// Each row of a layer reads its column and sign bits from one hash. +#[cfg(test)] const HASH_BITS: u32 = 128; /// The per-row failure probability the L2 guarantee fixes: a row misses by /// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it @@ -39,6 +43,7 @@ const L2_ROW_FAILURE: f64 = 0.1; const DISTINCT_VARIANCE: f64 = 1.35; /// The smallest heap the distinct-count variance bound holds for. const DISTINCT_MIN_HEAP: u32 = 8; +#[cfg(test)] const CARD_CAP: u32 = 1 << 26; const L2_CONTRACT: &str = "univmon_layer0_f2_median_chebyshev_binomial_v1"; @@ -50,6 +55,7 @@ const MAX_DISTINCT: &str = "max_distinct"; /// The shape baseline for readouts without a size model: the total (exact /// whatever the shape) and entropy (uncertified). +#[cfg(test)] const BASELINE: SketchParams = SketchParams::UnivMon { heap_size: 256, sketch_rows: 5, @@ -147,6 +153,7 @@ fn l2_failure_probability(rows: u32) -> f64 { /// clamped to the hash budget: with more rows than [`MAX_L2_ROWS`], or more /// columns than `rows` leave bits for, the shape is the largest one built /// and its guarantee misses the target. +#[cfg(test)] fn l2_shape(eps: f64, delta: f64) -> (u32, u32) { let rows = (1..=MAX_L2_ROWS) .find(|&rows| l2_failure_probability(rows) <= delta) @@ -186,6 +193,7 @@ fn incomplete_top_layer_bound(heap_size: u32) -> f64 { /// The Chebyshev share of the distinct count's failure probability at /// heap `k` and relative error `eps`: `1.35/(k ε²)`. +#[cfg(test)] fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { DISTINCT_VARIANCE / (f64::from(heap_size) * eps * eps) } @@ -194,6 +202,7 @@ fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { /// `eps · F0` with probability at most `delta`, top-layer completeness /// included: `1.35/(k ε²) + (e/4)^(k/2) ≤ δ`. Capped; a capped heap's /// guarantee misses the target. +#[cfg(test)] fn distinct_heap(eps: f64, delta: f64) -> u32 { if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { return CARD_CAP; @@ -320,6 +329,7 @@ pub(crate) fn distinct_guarantee( /// counter; /// - total and entropy: the baseline shape (the total is exact; entropy /// has no certified bound). +#[cfg(test)] pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { match intent { AggIntent::FrequencyL2 { .. } => { diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index 84274bb21..2a78698b9 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -7,7 +7,6 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::tumbling_window_chain::merged_chain; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::guarantee::{ AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, @@ -527,32 +526,31 @@ fn rolled_up_guarantee(node: &OperatorNode) -> Option> } /// The guarantee of an estimate or finalized value read from merged -/// tumbling windows ([`merged_chain`]): the guarantee of the same read of one -/// build over the merged tumbling windows' time range (the whole chain's, or -/// its newest tumbling windows') when the -/// family's merged error for that read is exact or the same as that build's -/// (the family-capability table); none when it is approximate (evidence +/// tumbling windows (a state whose [`OperatorNode::summary_build`] is a +/// chain): the guarantee of the same read of one build over the time range +/// the merged windows cover when the +/// summary type's merged error for that read is exact or the same as that build's +/// (the summary-type capability table); none when it is approximate (evidence /// only, for plan selection to judge) or the family has no merge. `None` for /// any node that does not read merged tumbling windows. fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let merged = match node.asap()? { + let (state, query) = match node.asap()? { ASAPOp::SummaryEstimate { summary_input, query, - } => { - let chain = merged_chain(summary_input)?; - let family = chain.asap()?.produced_state()?; - SummaryTypeCapability::of(family).merged_error(query) - } - ASAPOp::FinalizeExactAccumulator { child } => { - let chain = merged_chain(child)?; - let family = chain.asap()?.produced_state()?; - // An exact accumulator's merged state is exact. - let merged = SummaryTypeCapability::of(family).merge_inputs; - merged.map(|_| MergedError::Exact) - } + } => (summary_input, Some(query)), + ASAPOp::FinalizeExactAccumulator { child } => (child, None), _ => return None, }; + let ASAPOp::SummaryTumblingWindowChain { summary_type, .. } = state.summary_build()? else { + return None; + }; + let capability = SummaryTypeCapability::of(&summary_type); + let merged = match query { + Some(query) => capability.merged_error(query), + // An exact accumulator's merged state is exact. + None => capability.merge_inputs.map(|_| MergedError::Exact), + }; if !matches!( merged, Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) @@ -662,40 +660,43 @@ pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option match merged_chain(summary_input) - .unwrap_or_else(|| Rc::clone(summary_input)) - .asap()? - { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. + }) if (0.0..=1.0).contains(q) => { + let build = match summary_input.summary_build()? { + chain @ ASAPOp::SummaryTumblingWindowChain { .. } => chain, + _ => summary_input.asap()?.clone(), + }; + match &build { + ASAPOp::SummaryAgg { + input: + SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + .. + }, + filter: None, + child, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + input: + SummaryUpdate { + item: None, + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + .. + }, + filter: None, + child, + .. + } => estimates_raw_data_samples_of(child), + _ => None, } - | ASAPOp::SummaryTumblingWindowChain { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - }, + } Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { match summary_input.asap()? { ASAPOp::SummaryAgg { diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index b13ab82e7..bf8194ddd 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -1,9 +1,10 @@ -//! Sizing: the statistic a summary is read for, and the default sketch -//! parameters for an `(ε, δ)`. +//! The statistic a summary is read for. (Today's sizing for an `(ε, δ)` is +//! test-only, `estimators::size_params`: the predefined lists are checked +//! against it.) use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::ColumnRef; -use asap_types::ir::schema::{SketchAlgorithm, SketchParams, SketchStatistic}; +use asap_types::ir::schema::SketchStatistic; /// The statistic a summary is read for to answer `intent`, or `None` when /// no sketch statistic serves the intent. @@ -22,47 +23,31 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { }) } -/// `asap-plan`'s built-in `SketchParams` sizing, keyed off the resolved -/// `(eps, delta)` accuracy budget. -/// -/// Each formula inverts the sketch family's standard error bound to the -/// smallest parameter satisfying the target, clamped to the family's sane -/// range. A non-positive ε saturates to the clamp maximum (tightest -/// allowed). -#[allow(dead_code)] // Read by `test_support` until #1207 PR 1 replaces it with `parameter_list`. -pub(crate) fn default_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - crate::estimators::size_params(kind, intent, eps, delta) -} - #[cfg(test)] mod tests { use super::*; + use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. #[test] fn epsilon_delta_sizes_cms_depth() { let intent = AggIntent::Count; assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), + crate::estimators::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), SketchParams::Cms { width: 2719, depth: 7 } // ⌈e/0.001⌉, ⌈ln 1000⌉ ); assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), + crate::estimators::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), SketchParams::Cms { width: 2719, depth: 5 } ); assert_eq!( - default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), + crate::estimators::size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), SketchParams::Cms { width: 272, depth: 5 diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs index 672ee25ae..9fd42c4a9 100644 --- a/crates/accuracy/src/test_support.rs +++ b/crates/accuracy/src/test_support.rs @@ -5,11 +5,8 @@ use std::rc::Rc; use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::guarantee::AccuracyGuarantee; -use asap_types::ir::schema::{ - FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, -}; +use asap_types::ir::schema::{FieldDataType, SketchStatistic, SummaryUpdate}; use asap_types::ir::{OperatorNode, ScalarExpr}; use crate::composition::StreamNorm; @@ -71,11 +68,6 @@ pub fn source_distinct_guarantee( propagation::source_distinct_guarantee(family, query, input) } -/// `crate::estimators::cms::cms_width`. -pub fn cms_width(eps: f64) -> u32 { - crate::estimators::cms::cms_width(eps) -} - /// `crate::estimators::cms::cms_depth`. pub fn cms_depth(delta: f64) -> u32 { crate::estimators::cms::cms_depth(delta) @@ -86,16 +78,6 @@ pub fn kll_rank_error_99(k: u32) -> f64 { crate::estimators::kll::kll_rank_error_99(k) } -/// `crate::sizing::default_size_params`. -pub fn default_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - crate::sizing::default_size_params(kind, intent, eps, delta) -} - /// `crate::composition::QUOTIENT_OPERAND_DOMAINS`. pub const QUOTIENT_OPERAND_DOMAINS: &str = crate::composition::QUOTIENT_OPERAND_DOMAINS; diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 0adde2639..1cf4992d3 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -8,6 +8,8 @@ use serde::{Deserialize, Serialize}; use super::input::{InputOrTemporaryBuild, OperatorInput}; use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::non_asap::NonASAPOp; +use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::maintained_population::{MaintainedPopulation, PopulationStatistic}; use crate::ir::operator::operator_properties::Reduction; use crate::ir::schema::sketch::{SketchStatistic, SummaryUpdate}; @@ -75,8 +77,9 @@ pub enum ASAPOp> { /// `reduction` and `filter`, built over each window /// `(k·w, (k+1)·w]` of `child`, `w` = `tumbling_window_width_ms`, on a /// grid anchored at the epoch. Each window is built when it ends. Output: - /// one row per window, (window end, keys, state), a state with a time - /// index. It has no range and no window count: a `TimeRange` (and + /// one row per window and key, (window_end, keys, state), a state with + /// a time index (`window_end`, column 0), unique by (window_end, keys). + /// It has no range and no window count: a `TimeRange` (and /// `TimeShift`) over it selects the windows a query reads, and how many /// windows it retains is $h_v$, a Stage 3 choice. // Written as `PaneChain` before #1163; the aliases read those documents. @@ -281,13 +284,13 @@ impl ASAPOp { summary_type: family, reduction, .. - } - | SummaryTumblingWindowChain { + } => summary_agg_schema(child.schema(), family, reduction)?, + SummaryTumblingWindowChain { child, summary_type: family, reduction, .. - } => summary_agg_schema(child.schema(), family, reduction)?, + } => chain_schema(child.schema(), family, reduction)?, SummaryEstimate { summary_input, query, @@ -336,9 +339,8 @@ impl ASAPOp { if let Some((build, schema)) = rolled_up_build(child) { return finalized_schema(Some(&build), &schema); } - if super::tumbling_window_chain::merged_chain(child).is_some() { - todo!("#1207 PR 1: finalize the windows a TimeRange over a chain selects as one build over that TimeRange") - } + // The windows a `TimeRange` over a chain selects, merged, + // finalize as the chain's build over that range. finalized_schema(summary_build(child).as_ref(), child.schema())? } MaintainPopulation { child, .. } => child.schema().clone(), @@ -410,9 +412,22 @@ impl ASAPOp { } => { self.validate_inputs()?; let input = children[0].schema(); - match group_by { - None => input.clone(), - Some(keys) => regrouped_schema(input, keys)?, + match (group_by, summary_build(&children[0])) { + (Some(keys), _) => regrouped_schema(input, keys)?, + // The windows a `TimeRange` over a chain selects merge + // into the state of the chain's build over that range, + // which has no window column (a `TimeRange` and a + // `TimeShift` keep their input's schema). + ( + None, + Some(SummaryTumblingWindowChain { + child, + summary_type, + reduction, + .. + }), + ) => summary_agg_schema(child.schema(), &summary_type, &reduction)?, + (None, _) => input.clone(), } } SummarySubtract { .. } @@ -474,6 +489,19 @@ impl ASAPOp { "summary merge requires exactly one state column".into(), )); } + // Over a time-indexed state: the windows one `TimeRange` + // over a chain selects, not the whole chain. + if children.iter().any(is_time_indexed_state) + && !matches!( + children.as_slice(), + [child] if matches!(child.operator(), Operator::NonASAP(NonASAPOp::TimeRange { .. })) + ) + { + return Err(SchemaDerivationError::InvalidScalarSignature( + "a summary merge over a time-indexed state reads one TimeRange over a chain" + .into(), + )); + } for child in children { needs_state(child, "SummaryMerge")?; if child.schema() != first.schema() { @@ -655,22 +683,31 @@ fn rolled_up_build( return None; }; let input = selected_from(input).unwrap_or_else(|| input.clone()); - let build = match input.operator() { - Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => { + // A build, or the windows a `TimeRange` over a chain selects, merged: + // the chain's build over that range, whose schema is the build's over + // the chain's input. + let build = match (input.operator(), summary_build(&input)) { + (Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }), _) + | (_, Some(op @ ASAPOp::SummaryTumblingWindowChain { .. })) => { op.map_children(|child| InputOrTemporaryBuild::Input(child.clone())) } - _ if super::tumbling_window_chain::merged_chain(&input).is_some() => { - todo!("#1207 PR 1: roll up the windows a TimeRange over a chain selects as one build over that TimeRange") - } _ => return None, }; - let ASAPOp::SummaryAgg { + let (ASAPOp::SummaryAgg { + child, + summary_type, + input: update, + reduction, + filter, + } + | ASAPOp::SummaryTumblingWindowChain { child, - summary_type: family, + summary_type, input: update, reduction, filter, - } = build + .. + }) = build else { return None; }; @@ -683,11 +720,11 @@ fn rolled_up_build( }) .collect::>>()?; let reduction = Reduction::by(mapped); - let schema = summary_agg_schema(child.schema(), &family, &reduction).ok()?; + let schema = summary_agg_schema(child.schema(), &summary_type, &reduction).ok()?; Some(( ASAPOp::SummaryAgg { child, - summary_type: family, + summary_type, input: update, reduction, filter, @@ -722,6 +759,39 @@ pub(crate) fn summary_agg_schema( Ok(schema) } +/// The schema of a `SummaryTumblingWindowChain` of `summary_type` by `reduction` +/// over rows of schema `child`: a state with a time index, one row per +/// window and key, (window_end, keys…, state). `window_end` (column 0, the +/// time index) is computed, so it takes no name or table from `child`; the +/// build's own time column, which a per-entity build keeps, is dropped. The +/// unique key is (window_end, keys). +fn chain_schema( + child: &Schema, + summary_type: &FieldDataType, + reduction: &Reduction, +) -> Result { + let mut schema = summary_agg_schema(child, summary_type, reduction)?; + let time = child + .time_index + .and_then(|index| child.fields.get(index)) + .ok_or_else(|| { + SchemaDerivationError::InvalidScalarSignature( + "a tumbling-window chain reads rows with a time column".into(), + ) + })?; + if let Some(build_time) = schema.time_index.take() { + schema.fields.remove(build_time); + } + schema + .fields + .insert(0, Field::new("window_end", time.dtype.clone(), false)); + schema.time_index = Some(0); + schema.unique_keys = vec![(0..schema.fields.len()) + .filter(|&column| schema.fields[column].is_plain()) + .collect()]; + Ok(schema) +} + /// The schema `FinalizeExactAccumulator` derives over state of /// `state_schema` that `build` (the state's `SummaryAgg`, if any) builds: /// each exact aggregate replaced by the value it finalizes to. @@ -729,14 +799,24 @@ fn finalized_schema( build: Option<&ASAPOp>, state_schema: &Schema, ) -> Result { - let value_result = if let Some(ASAPOp::SummaryAgg { - child: source, - summary_type: FieldDataType::ExactAggregate(kind, _), - input, - reduction, - filter, - .. - }) = build + let value_result = if let Some( + ASAPOp::SummaryAgg { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + child: source, + summary_type: FieldDataType::ExactAggregate(kind, _), + input, + reduction, + filter, + .. + }, + ) = build { { use crate::ir::operator::AggIntent; @@ -842,22 +922,24 @@ fn finalized_data_type(kind: &crate::ir::schema::sketch::ExactKind) -> DataType } } -/// The states `node` selects from when it is a `Filter` that keeps whole -/// summary states by their plain columns (its output is state); `None` for +/// The states `node` selects from when it keeps whole summary states (its +/// output is state): a `Filter` by their plain columns, or a `TimeRange` or +/// `TimeShift` over a time-indexed state, which selects windows. `None` for /// any other input. pub fn selected_from(node: &I) -> Option { match node.operator() { - Operator::NonASAP(crate::ir::NonASAPOp::Filter { child, .. }) - if node.result_kind() == OperatorResultKind::State => - { - Some(child) - } + Operator::NonASAP( + NonASAPOp::Filter { child, .. } + | NonASAPOp::TimeRange { child, .. } + | NonASAPOp::TimeShift { child, .. }, + ) if node.result_kind() == OperatorResultKind::State => Some(child), _ => None, } } /// The build whose state this summary state carries: a `SummaryAgg` or -/// `SummaryTumblingWindowChain` itself, or the first input's build under +/// `SummaryTumblingWindowChain` itself, the chain under the `TimeRange` and +/// `TimeShift`s that select its windows, or the first input's build under /// `SummaryMerge`s (merged inputs have one definition, which summary /// coverage checks). `None` under a regrouping merge, whose states no build /// carries (its coverage's definition is the build it stands for), and for @@ -872,6 +954,22 @@ pub fn summary_build(state: &I) -> Option> { group_by: None, .. }) => summary_build(children.first()?), + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => summary_build(&child), + _ => None, + } +} + +/// The summary type a build operator builds: a `SummaryAgg`'s or a +/// `SummaryTumblingWindowChain`'s `summary_type`; `None` for every other +/// operator. +pub fn summary_type_of_operator(operator: &Operator) -> Option<&FieldDataType> { + match operator { + Operator::ASAP( + ASAPOp::SummaryAgg { summary_type, .. } + | ASAPOp::SummaryTumblingWindowChain { summary_type, .. }, + ) => Some(summary_type), _ => None, } } diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index dd1755456..42800d44b 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use super::input::OperatorInput; use super::node::{Operator, OperatorNode, OperatorResultKind}; +use super::tumbling_window_chain::is_time_indexed_state; use crate::ir::operator::agg_intent::AggIntent; use crate::ir::operator::operator_properties::{ BinaryOpKind, GroupKeys, InfoMatcher, JoinKind, Reduction, RelationalSetOpKind, SampleKind, @@ -921,6 +922,12 @@ impl NonASAPOp { TimeRange { child, .. } if child.result_kind() == OperatorResultKind::Relation => { OperatorResultKind::Relation } + // Over a time-indexed state (a chain): the windows it selects. + TimeRange { child, .. } | TimeShift { child, .. } + if child.result_kind() == OperatorResultKind::State => + { + OperatorResultKind::State + } TimeRange { kind, .. } => match kind { TimeRangeKind::Instant => OperatorResultKind::InstantVector, TimeRangeKind::Range => OperatorResultKind::RangeVector, @@ -1162,6 +1169,24 @@ impl NonASAPOp { return Err(invalid("label expression requires a string")); } } + TimeRange { child, .. } | TimeShift { child, .. } + if child.result_kind() == OperatorResultKind::State => + { + if !is_time_indexed_state(child) { + return Err(invalid( + "a time selection over state requires a time-indexed state", + )); + } + if matches!( + self, + TimeRange { + kind: TimeRangeKind::Instant, + .. + } + ) { + return Err(invalid("a time selection over a chain is a range")); + } + } TimeRange { child, .. } => { if child.result_kind() != OperatorResultKind::Relation { instant(child)?; diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index aca6285b0..065a3a310 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -1,33 +1,23 @@ //! Tumbling-window chains ([`ASAPOp::SummaryTumblingWindowChain`]): one //! row per window of width `w`, on a grid anchored at the epoch, a state //! with a time index. A chain has no range and no window count: a -//! `TimeRange` (and `TimeShift`) over it selects the windows a query reads -//! (#1207 PR 1), and how many windows it retains is $h_v$ (#1207 PR 7). +//! `TimeRange` (and `TimeShift`s) over it select the windows a query reads, +//! a `SummaryMerge` over that selection is the state of the chain's build +//! over the same range, and how many windows it retains is $h_v$ (#1207 +//! PR 7). use super::asap::ASAPOp; use super::input::OperatorInput; use super::node::Operator; +use super::non_asap::NonASAPOp; -/// The `SummaryTumblingWindowChain` a summary state carries merged: the -/// chain itself, or the one chain a `SummaryMerge` that keeps its groups -/// reads. `None` for any other input, a rollup (regrouping merge) included. -pub fn merged_chain(state: &I) -> Option { +/// Whether `state` is a state with a time index: a chain, or a `TimeRange` +/// or `TimeShift` over one. +pub(crate) fn is_time_indexed_state(state: &I) -> bool { match state.operator() { - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => Some(state.clone()), - Operator::ASAP(ASAPOp::SummaryMerge { - children, - group_by: None, - .. - }) => match children.as_slice() { - [child] - if matches!( - child.operator(), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) - ) => - { - Some(child.clone()) - } - _ => None, - }, - _ => None, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => true, + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => is_time_indexed_state(&child), + _ => false, } } diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 2841c733d..5c1827096 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -64,8 +64,12 @@ impl PhysicalProperties { timing: ExecutionTiming, lifetime: Lifetime, ) -> Result { - let _ = (timing, lifetime); - todo!("#1207 PR 1: reject the invalid combination of section 2") + match (timing, lifetime) { + (ExecutionTiming::IngestionTime, Lifetime::OneEvaluation(_)) => { + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + } + _ => Ok(Self { timing, lifetime }), + } } /// $\tau_v$. @@ -89,32 +93,35 @@ mod tests { use super::*; use ExecutionTiming::{IngestionTime, QueryTime}; - const MEMORY: StorageMedium = StorageMedium::Memory; - - /// The one invalid combination of the concepts doc, section 2. - #[test] - fn new_rejects_one_evaluation_at_ingestion() { - assert_eq!( - PhysicalProperties::new(IngestionTime, Lifetime::OneEvaluation(MEMORY)), - Err(PhysicalPropertiesError::OneEvaluationAtIngestion) - ); - } - - /// The valid (τ, ℓ) pairs of section 2. + /// `new` accepts every (τ, ℓ) except (IngestionTime, OneEvaluation), for + /// both media, and keeps what it was given. #[test] - fn new_accepts_each_valid_pair() { - for (timing, lifetime) in [ - (QueryTime, Lifetime::None), - (IngestionTime, Lifetime::None), - (QueryTime, Lifetime::OneEvaluation(MEMORY)), - (QueryTime, Lifetime::AcrossEvaluations(MEMORY)), - ( - IngestionTime, - Lifetime::AcrossEvaluations(StorageMedium::Disk), - ), - ] { - let properties = PhysicalProperties::new(timing, lifetime).unwrap(); - assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + fn new_over_every_timing_and_lifetime() { + for medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (timing, lifetime, valid) in [ + (QueryTime, Lifetime::None, true), + (QueryTime, Lifetime::OneEvaluation(medium), true), + (QueryTime, Lifetime::AcrossEvaluations(medium), true), + (IngestionTime, Lifetime::None, true), + (IngestionTime, Lifetime::OneEvaluation(medium), false), + (IngestionTime, Lifetime::AcrossEvaluations(medium), true), + ] { + let properties = PhysicalProperties::new(timing, lifetime); + match valid { + true => { + let properties = properties.unwrap(); + assert_eq!( + (properties.timing(), properties.lifetime()), + (timing, lifetime) + ); + assert_eq!(properties.is_materialized(), lifetime != Lifetime::None); + } + false => assert_eq!( + properties, + Err(PhysicalPropertiesError::OneEvaluationAtIngestion) + ), + } + } } } } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 89ab3b284..d47f785f6 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -6,9 +6,10 @@ use std::collections::HashSet; use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; +use crate::ir::operator::asap::summary_type_of_operator; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::properties::{AccuracyMetric, AccuracyTarget}; -use crate::ir::schema::{SketchAlgorithm, SketchParams}; +use crate::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; use crate::ir::ASAPOp; use crate::ir::Operator; use crate::ir::QueryRoot; @@ -90,8 +91,7 @@ impl Memo { /// The candidate `id`. pub fn candidate(&self, id: CandidateForMemoGroupId) -> &CandidateForMemoGroup { - let _ = id; - todo!("#1207 PR 2") + &self.groups[id.group.0 as usize].candidates[id.index_in_group] } /// Replaces candidate `id`'s input requirements $r^{\log}_{c,i}$, one @@ -263,8 +263,21 @@ impl LogicalProperties { expression: &CandidateExpression, parameters: Option, ) -> Result { - let _ = (expression, parameters); - todo!("#1207 PR 1: reject invalid parameters") + let algorithm = match summary_type_of_operator(&expression.operator) { + Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), + _ => None, + }; + match (algorithm, ¶meters) { + (Some(_), None) => return Err(LogicalPropertiesError::SketchBuildWithoutParameters), + (None, Some(_)) => return Err(LogicalPropertiesError::ParametersOnExactOperator), + (Some(algorithm), Some(parameters)) => { + if !parameter_list(algorithm).contains(parameters) { + return Err(LogicalPropertiesError::ParametersNotInList); + } + } + (None, None) => {} + } + Ok(Self { parameters }) } /// $\boldsymbol\theta_v$. @@ -288,9 +301,239 @@ pub enum LogicalPropertiesError { /// The algorithm's predefined $\boldsymbol\theta$ list: structures only; the /// error of each, for a statistic at a failure probability, is its guarantee /// (`asap_accuracy::theoretical_guarantee`). +/// +/// The widths, `k`s, precisions, `alpha`s and heap sizes are today's sizing +/// (`asap_accuracy`'s test-only `size_params`) over the errors ε = 0.1, +/// 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, without +/// repeats; an `asap_accuracy` test keeps them equal. Distinct counts (KMV, +/// and UnivMon's distinct-count shapes) stop at ε = 0.01. Where δ is set by +/// a structural parameter, the list covers the δ levels with it: Count-Min +/// lists each width with each depth of [`CMS_DEPTHS`], Count Sketch with +/// each of [`COUNT_SKETCH_DEPTHS`]. A +/// UnivMon's list is its L2-norm shapes, its distinct-count shapes, each L2 +/// shape combined with each distinct-count shape, then its shape for every +/// other statistic. A heap variant combines each width × depth with each +/// heap size of [`HEAP_SIZES`]: a top-k needs heap_size ≥ k, and the next +/// listed step up is at most 2.5× too large. Empty for an algorithm without +/// an error model (the Hydra algorithms included), so no build of it is a +/// valid candidate. pub fn parameter_list(algorithm: SketchAlgorithm) -> &'static [SketchParams] { - let _ = algorithm; - todo!("#1207 PR 1: the predefined θ list per algorithm") + match algorithm { + SketchAlgorithm::Kll => &KLL_PARAMETERS, + SketchAlgorithm::DDSketch => &DDSKETCH_PARAMETERS, + SketchAlgorithm::Hll => &HLL_PARAMETERS, + SketchAlgorithm::Kmv => &KMV_PARAMETERS, + SketchAlgorithm::Cms => &CMS_PARAMETERS, + SketchAlgorithm::CountSketch => &COUNT_SKETCH_PARAMETERS, + SketchAlgorithm::CmsWithHeap => &CMS_WITH_HEAP_PARAMETERS, + SketchAlgorithm::CountSketchWithHeap => &COUNT_SKETCH_WITH_HEAP_PARAMETERS, + SketchAlgorithm::UnivMon => &UNIVMON_PARAMETERS, + // Listed, not `_`: an algorithm `asap_sketchlib` adds fails to + // compile here until it gets a list or is listed without one. + SketchAlgorithm::CountSketchWithL2Norm + | SketchAlgorithm::CmsWithSpaceSaving + | SketchAlgorithm::CountSketchWithSpaceSaving + | SketchAlgorithm::CmsFold + | SketchAlgorithm::CountSketchFold + | SketchAlgorithm::HllHip + | SketchAlgorithm::KllDynamic + | SketchAlgorithm::SpaceSaving + | SketchAlgorithm::Elastic + | SketchAlgorithm::Coco + | SketchAlgorithm::Bloom + | SketchAlgorithm::SetAggregator + | SketchAlgorithm::UnivMonTwoTier + | SketchAlgorithm::UnivMonQ + | SketchAlgorithm::HydraCms + | SketchAlgorithm::HydraCountSketch + | SketchAlgorithm::HydraKll + | SketchAlgorithm::HydraHll + | SketchAlgorithm::HydraUnivMon => &[], + } +} + +const KLL_PARAMETERS: [SketchParams; 7] = [ + SketchParams::Kll { k: 26 }, + SketchParams::Kll { k: 52 }, + SketchParams::Kll { k: 132 }, + SketchParams::Kll { k: 269 }, + SketchParams::Kll { k: 547 }, + SketchParams::Kll { k: 1_404 }, + SketchParams::Kll { k: 2_863 }, +]; + +const DDSKETCH_PARAMETERS: [SketchParams; 7] = [ + SketchParams::DDSketch { alpha: 0.1 }, + SketchParams::DDSketch { alpha: 0.05 }, + SketchParams::DDSketch { alpha: 0.02 }, + SketchParams::DDSketch { alpha: 0.01 }, + SketchParams::DDSketch { alpha: 0.005 }, + SketchParams::DDSketch { alpha: 0.002 }, + SketchParams::DDSketch { alpha: 0.001 }, +]; + +/// HLL's precision saturates at 18 from ε = 0.01 on. +const HLL_PARAMETERS: [SketchParams; 4] = [ + SketchParams::Hll { precision: 13 }, + SketchParams::Hll { precision: 15 }, + SketchParams::Hll { precision: 17 }, + SketchParams::Hll { precision: 18 }, +]; + +const KMV_PARAMETERS: [SketchParams; 4] = [ + SketchParams::Kmv { k: 1_118 }, + SketchParams::Kmv { k: 4_323 }, + SketchParams::Kmv { k: 26_656 }, + SketchParams::Kmv { k: 106_263 }, +]; + +/// The depths each Count-Min width is listed with: δ = e^−d ≤ 1e-2, 1e-3, +/// 1e-4 and 1e-5. +const CMS_DEPTHS: [u32; 4] = [5, 7, 10, 12]; +/// The depths each Count Sketch width is listed with, for the same δ +/// levels: Count Sketch reads the median of its rows, each wrong with +/// probability about 1/3, so δ = P(Binomial(d, 1/3) ≥ (d+1)/2), the +/// smallest odd d at or below each level. +const COUNT_SKETCH_DEPTHS: [u32; 4] = [47, 81, 117, 155]; + +const CMS_WIDTHS: [u32; 7] = [28, 55, 136, 272, 544, 1_360, 2_719]; +const COUNT_SKETCH_WIDTHS: [u32; 7] = [300, 1_200, 7_500, 30_000, 120_000, 750_000, 3_000_000]; + +const CMS_PARAMETERS: [SketchParams; 28] = with_depths( + &CMS_WIDTHS, + &CMS_DEPTHS, + SketchParams::Cms { width: 0, depth: 0 }, +); +const COUNT_SKETCH_PARAMETERS: [SketchParams; 28] = with_depths( + &COUNT_SKETCH_WIDTHS, + &COUNT_SKETCH_DEPTHS, + SketchParams::CountSketch { width: 0, depth: 0 }, +); + +/// Each of `widths` with each of `depths`, by width, then depth, as +/// `shape`'s variant (`Cms` or `CountSketch`). +const fn with_depths( + widths: &[u32; 7], + depths: &[u32; 4], + shape: SketchParams, +) -> [SketchParams; 28] { + const UNSET: SketchParams = SketchParams::Cms { width: 0, depth: 0 }; + let mut list = [UNSET; 28]; + let mut width = 0; + while width < widths.len() { + let mut depth = 0; + while depth < depths.len() { + let (width_value, depth_value) = (widths[width], depths[depth]); + list[width * depths.len() + depth] = match shape { + SketchParams::Cms { .. } => SketchParams::Cms { + width: width_value, + depth: depth_value, + }, + SketchParams::CountSketch { .. } => SketchParams::CountSketch { + width: width_value, + depth: depth_value, + }, + _ => panic!("a Count-Min or Count Sketch shape"), + }; + depth += 1; + } + width += 1; + } + list +} + +/// The heap sizes of a heap variant's list: the 1-2-5 series from 10 to +/// 10 000, so a top-k with k ≤ 10 000 has a listed heap. +const HEAP_SIZES: [u32; 10] = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; + +const CMS_WITH_HEAP_PARAMETERS: [SketchParams; 280] = with_heap_sizes(&CMS_PARAMETERS); +const COUNT_SKETCH_WITH_HEAP_PARAMETERS: [SketchParams; 280] = + with_heap_sizes(&COUNT_SKETCH_PARAMETERS); + +/// Each `Cms` (or `CountSketch`) shape of `shapes` with each heap size of +/// [`HEAP_SIZES`], as `CmsWithHeap` (or `CountSketchWithHeap`). +const fn with_heap_sizes(shapes: &[SketchParams; 28]) -> [SketchParams; 280] { + const UNSET: SketchParams = SketchParams::Cms { width: 0, depth: 0 }; + let mut list = [UNSET; 280]; + let mut shape = 0; + while shape < shapes.len() { + let mut heap = 0; + while heap < HEAP_SIZES.len() { + let heap_size = HEAP_SIZES[heap]; + list[shape * HEAP_SIZES.len() + heap] = match shapes[shape] { + SketchParams::Cms { width, depth } => SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + SketchParams::CountSketch { width, depth } => SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + }, + _ => panic!("a heap variant extends a Cms or CountSketch shape"), + }; + heap += 1; + } + shape += 1; + } + list +} + +/// UnivMon's L2-norm column counts and distinct-count heap sizes. +const UNIVMON_L2_COLUMNS: [u32; 7] = [624, 2_367, 14_349, 56_817, 226_130, 1_409_067, 5_630_630]; +const UNIVMON_DISTINCT_HEAP_SIZES: [u32; 4] = [13_500, 54_000, 337_500, 1_350_000]; + +const UNIVMON_PARAMETERS: [SketchParams; 40] = univmon_parameters(); + +/// The L2-norm shapes, the distinct-count shapes, then each L2 column +/// count × distinct heap size (by column count, then heap size), last the +/// shape for every other statistic. +const fn univmon_parameters() -> [SketchParams; 40] { + const fn univmon( + heap_size: u32, + sketch_rows: u32, + sketch_cols: u32, + layers: u8, + ) -> SketchParams { + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + } + } + const BASELINE: SketchParams = univmon(256, 5, 1_024, 16); + let mut list = [BASELINE; 40]; + let mut next = 0; + let mut columns = 0; + while columns < UNIVMON_L2_COLUMNS.len() { + list[next] = univmon(1, 5, UNIVMON_L2_COLUMNS[columns], 1); + next += 1; + columns += 1; + } + let mut heap = 0; + while heap < UNIVMON_DISTINCT_HEAP_SIZES.len() { + list[next] = univmon(UNIVMON_DISTINCT_HEAP_SIZES[heap], 1, 1, 19); + next += 1; + heap += 1; + } + columns = 0; + while columns < UNIVMON_L2_COLUMNS.len() { + heap = 0; + while heap < UNIVMON_DISTINCT_HEAP_SIZES.len() { + list[next] = univmon( + UNIVMON_DISTINCT_HEAP_SIZES[heap], + 5, + UNIVMON_L2_COLUMNS[columns], + 19, + ); + next += 1; + heap += 1; + } + columns += 1; + } + list } /// $r^{\log}_{c,i}$: only what $G_{c,i}$'s identity does not fix. Only a @@ -344,3 +587,205 @@ pub struct ErrorPropagation { /// denominator). pub validity_cap_per_input: Vec>, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::operator::operator_properties::Reduction; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::schema::{ExactKind, FieldDataType, HydraParams, SummaryUpdate}; + + /// `SummaryAgg` over group 0 building a state of `summary_type`. + fn summary_agg(summary_type: FieldDataType) -> CandidateExpression { + CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryAgg { + child: MemoGroupId(0), + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + } + } + + const KLL_269: SketchParams = SketchParams::Kll { k: 269 }; + const CMS: SketchParams = SketchParams::Cms { + width: 272, + depth: 5, + }; + const HYDRA_CMS: SketchParams = SketchParams::Hydra(HydraParams::HydraCms { + width: 272, + depth: 5, + shared_rows: 3, + shared_columns: 1_024, + }); + + /// `LogicalProperties::new` over every operator kind and parameters: a + /// sketch build (a chain included) needs parameters of its algorithm from + /// its list; parameters are rejected on an operator that builds no + /// sketch. + #[test] + fn logical_properties_new_over_every_operator_and_parameters() { + use LogicalPropertiesError as Invalid; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let kll_chain = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }), + }; + let readout = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: MemoGroupId(0), + query: crate::ir::schema::SketchStatistic::Quantile { q: 0.5 }, + }), + }; + // k = 270 is KLL's structure but not in its list. + let parameters = [ + None, + Some(KLL_269), + Some(CMS), + Some(HYDRA_CMS), + Some(SketchParams::Kll { k: 270 }), + ]; + let no_parameters = Err(Invalid::ParametersOnExactOperator); + let needs_parameters = Err(Invalid::SketchBuildWithoutParameters); + let not_in_list = Err(Invalid::ParametersNotInList); + let exact = [ + Ok(()), + no_parameters, + no_parameters, + no_parameters, + no_parameters, + ]; + // Another algorithm's parameters are not in KLL's list either. + let kll_build = [ + needs_parameters, + Ok(()), + not_in_list, + not_in_list, + not_in_list, + ]; + // Per operator: the result for each of `parameters`. + let table = [ + (summary_agg(kll), kll_build), + // No Hydra θ is listed yet, so no Hydra build is valid. + ( + summary_agg(FieldDataType::Sketch(SketchAlgorithm::HydraCms)), + [ + needs_parameters, + not_in_list, + not_in_list, + not_in_list, + not_in_list, + ], + ), + (summary_agg(FieldDataType::exact(ExactKind::Sum)), exact), + (kll_chain, kll_build), + (readout, exact), + ]; + for (expression, by_parameters) in table { + for (parameters, expected) in parameters.iter().zip(by_parameters) { + let properties = LogicalProperties::new(&expression, parameters.clone()); + assert_eq!( + properties.as_ref().map(|_| ()).map_err(|e| *e), + expected, + "{:?} {parameters:?}", + expression.operator.kind_name() + ); + if let Ok(properties) = properties { + assert_eq!(properties.parameters(), parameters.as_ref()); + } + } + } + } + + /// `summary_type_of_operator` reads the summary type of a `SummaryAgg` + /// and of a chain, and of no other operator. + #[test] + fn summary_type_of_operator_reads_the_builds_summary_type() { + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let chain = Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: MemoGroupId(0), + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 5_000, + }); + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![MemoGroupId(0)], + group_by: None, + }); + assert_eq!( + summary_type_of_operator(&summary_agg(kll.clone()).operator), + Some(&kll) + ); + assert_eq!(summary_type_of_operator(&chain), Some(&kll)); + assert_eq!(summary_type_of_operator(&merge), None); + } + + /// Each predefined list holds one parameter type, without repeats, and + /// is empty exactly for the algorithms without an error model. Count-Min + /// lists each width with depths 5, 7, 10 and 12, Count Sketch with 47, + /// 81, 117 and 155, and a + /// heap variant each of those with every heap size; UnivMon's list has 40 + /// entries. + #[test] + fn parameter_list_holds_its_algorithm_without_repeats() { + let listed = [ + SketchAlgorithm::Kll, + SketchAlgorithm::DDSketch, + SketchAlgorithm::Hll, + SketchAlgorithm::Kmv, + SketchAlgorithm::Cms, + SketchAlgorithm::CountSketch, + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + SketchAlgorithm::UnivMon, + ]; + for algorithm in listed.iter().copied().chain([ + SketchAlgorithm::HydraCms, + SketchAlgorithm::HydraCountSketch, + SketchAlgorithm::HydraKll, + SketchAlgorithm::HllHip, + ]) { + let list = parameter_list(algorithm); + assert_eq!( + list.is_empty(), + !listed.contains(&algorithm), + "{algorithm:?}" + ); + for (i, parameters) in list.iter().enumerate() { + assert_eq!( + std::mem::discriminant(parameters), + std::mem::discriminant(&list[0]) + ); + assert!(!list[..i].contains(parameters), "{parameters:?} repeats"); + } + } + let depths = |algorithm| -> Vec { + parameter_list(algorithm) + .iter() + .take(4) + .map(|parameters| match parameters { + SketchParams::Cms { depth, .. } + | SketchParams::CountSketch { depth, .. } + | SketchParams::CmsWithHeap { depth, .. } + | SketchParams::CountSketchWithHeap { depth, .. } => *depth, + other => panic!("{other:?}"), + }) + .collect() + }; + assert_eq!(depths(SketchAlgorithm::Cms), [5, 7, 10, 12]); + assert_eq!(depths(SketchAlgorithm::CountSketch), [47, 81, 117, 155]); + assert_eq!(depths(SketchAlgorithm::CmsWithHeap), [5, 5, 5, 5]); + assert_eq!(parameter_list(SketchAlgorithm::Cms).len(), 28); + assert_eq!(parameter_list(SketchAlgorithm::CmsWithHeap).len(), 280); + assert_eq!(parameter_list(SketchAlgorithm::UnivMon).len(), 40); + } +} diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index f94b2c5ff..dcef7d916 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -8,7 +8,8 @@ use std::collections::HashMap; use super::execution_context::ExecutionContext; use crate::ir::QueryRoot; -use crate::physical::{ExecutionTiming, PhysicalProperties}; +use crate::ir::{ASAPOp, Operator}; +use crate::physical::{ExecutionTiming, Lifetime, PhysicalProperties}; use crate::stages::{CandidateForMemoGroupId, LogicalASAPMemoCandidates, PrunedCandidate}; /// Index into [`PhysicalASAPMemoCandidates::physical_candidates`]. @@ -30,24 +31,70 @@ pub struct PhysicalCandidate { impl PhysicalCandidate { /// Whether a consumer running in `context` can read this producer (the - /// concepts doc's alignment table, section 2). True for $\ell$ = None, - /// and for `OneEvaluation` in the same context. A chain producer - /// (`SummaryTumblingWindowChain`) with $\ell$ = `AcrossEvaluations`, with - /// $w$ the chain operator's width, needs every time of the context on - /// its grid: $w$ divides the context's period and phase. Any other - /// producer with $\ell$ = `AcrossEvaluations` is stored on its group's - /// evaluation grid (`evaluation_grid_of_group`): it delivers on every - /// context when the group does not depend on the evaluation time, else - /// when that grid holds every time of the context. In - /// `Materialized(v)`, the context's times are v's, read from - /// `candidates`. + /// concepts doc's alignment table, section 2): true for $\ell$ = None, + /// which runs in the consumer's own context; for `OneEvaluation` only in + /// an `Evaluation` context, the one evaluation its result belongs to. + /// With $\ell$ = `AcrossEvaluations`, every execution time of the + /// context must be a time the producer has stored: for a chain + /// (`SummaryTumblingWindowChain`) of width $w$, $w$ divides an + /// evaluation grid's period and phase, or a refresh window's width and + /// phase; any other result is stored on its group's evaluation grid + /// (`evaluation_grid_of_group`), which must hold every time of the + /// context, unless the group does not depend on the evaluation time + /// (then it delivers on every context). pub fn delivers_on( &self, candidates: &PhysicalASAPMemoCandidates, context: &ExecutionContext, ) -> bool { - let _ = (candidates, context); - todo!("#1207 PR 1: the alignment table of section 2") + match self.physical_properties.lifetime() { + Lifetime::None => true, + Lifetime::OneEvaluation(_) => matches!(context, ExecutionContext::Evaluation { .. }), + Lifetime::AcrossEvaluations(_) => { + let logical = &candidates.logical_candidates; + let group = self.logical_candidate_id.group; + let chain_width_ms = match logical + .memo + .candidate(self.logical_candidate_id) + .expression + .operator + { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => Some(tumbling_window_width_ms), + _ => None, + }; + let depends_on_evaluation_time = logical.memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time; + if chain_width_ms.is_none() && !depends_on_evaluation_time { + return true; + } + let (period_ms, phase_ms) = match context { + ExecutionContext::Evaluation { grid } => (grid.period_ms, grid.phase_ms), + ExecutionContext::IngestionPipeline(window) => { + (window.width_ms, window.phase_ms) + } + ExecutionContext::Materialized(_) => { + todo!( + "#1207 PR 7: the times of a Materialized(v) context, v's own schedule" + ) + } + }; + match chain_width_ms { + Some(width_ms) => { + period_ms.is_multiple_of(width_ms) && phase_ms.is_multiple_of(width_ms) + } + None => { + logical.evaluation_grid_of_group[group.0 as usize].is_some_and(|grid| { + period_ms.is_multiple_of(grid.period_ms) + && phase_ms % grid.period_ms == grid.phase_ms % grid.period_ms + }) + } + } + } + } } } @@ -58,8 +105,19 @@ pub fn producer_satisfies_physical_requirements( producer: &PhysicalCandidate, consumer_timing: ExecutionTiming, ) -> bool { - let _ = (producer, consumer_timing); - todo!("#1207 PR 1: the timing x lifetime table of section 2") + use ExecutionTiming::{IngestionTime, QueryTime}; + match ( + producer.physical_properties.timing(), + producer.physical_properties.lifetime(), + ) { + (QueryTime, _) => consumer_timing == QueryTime, + (IngestionTime, Lifetime::None) => consumer_timing == IngestionTime, + // Materialized at ingestion: read at ingestion time and at query time. + (IngestionTime, Lifetime::AcrossEvaluations(_)) => true, + (IngestionTime, Lifetime::OneEvaluation(_)) => { + unreachable!("rejected by `PhysicalProperties::new`") + } + } } /// $(u, c, i)$ over physical candidates. @@ -162,3 +220,306 @@ pub fn output_references(query_roots: &[QueryRoot]) -> Vec<(usize, } references } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::materialized_view::RefreshWindow; + use crate::ir::operator::operator_properties::{Reduction, Source}; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, SummaryUpdate, + }; + use crate::ir::{NonASAPOp, TimeRangeKind}; + use crate::physical::StorageMedium; + use crate::stages::physical::execution_context::EvaluationGrid; + use crate::stages::{ + CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, + MemoGroupId, OriginOfCandidateForMemoGroup, + }; + use ExecutionTiming::{IngestionTime, QueryTime}; + + /// The KLL build over the scan: not a chain. + const BUILD: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(1), + index_in_group: 0, + }; + /// The chain of tumbling windows 5 s wide over the scan. + const WINDOWS_5S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(2), + index_in_group: 0, + }; + /// The chain of tumbling windows 10 s wide over the scan. + const WINDOWS_10S: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(3), + index_in_group: 0, + }; + /// X = the scan's last 5 minutes (`m[5m]`): a result that depends on + /// the evaluation time and is not a chain. + const LAST_5M: CandidateForMemoGroupId = CandidateForMemoGroupId { + group: MemoGroupId(4), + index_in_group: 0, + }; + + /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains + /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last + /// 5 minutes, each with its own expression as its one candidate. Built + /// through `find_or_add_group` and `add_candidate` (#1207 PR 2). + fn memo() -> Memo { + let mut memo = Memo::default(); + let scan = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let build = Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }); + let chain = |tumbling_window_width_ms| { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: kll.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms, + }) + }; + for operator in [build, chain(5_000), chain(10_000)] { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new( + &expression, + Some(SketchParams::Kll { k: 269 }), + ) + .unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(group, candidate).unwrap(); + } + let operator = Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }); + let group = memo.find_or_add_group(operator.clone()).unwrap(); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(group, candidate).unwrap(); + memo + } + + /// Stage 2's candidates over `memo`, with `evaluation_grid_of_group`; + /// `delivers_on` reads nothing else. + fn candidates( + memo: Memo, + evaluation_grid_of_group: Vec>, + ) -> PhysicalASAPMemoCandidates { + PhysicalASAPMemoCandidates { + logical_candidates: LogicalASAPMemoCandidates { + memo, + edges: vec![], + pruned: vec![], + evaluation_grid_of_group, + }, + physical_candidates: vec![], + pruned_candidates: vec![], + edges: vec![], + admissible_edges_into_input: HashMap::new(), + admissible_edges_out_of_candidate: vec![], + } + } + + fn candidate( + logical_candidate_id: CandidateForMemoGroupId, + timing: ExecutionTiming, + lifetime: Lifetime, + ) -> PhysicalCandidate { + PhysicalCandidate { + logical_candidate_id, + physical_properties: PhysicalProperties::new(timing, lifetime).unwrap(), + } + } + + fn evaluation(period_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::Evaluation { + grid: EvaluationGrid { + period_ms, + phase_ms, + }, + } + } + + fn ingestion(width_ms: u64, phase_ms: u64) -> ExecutionContext { + ExecutionContext::IngestionPipeline(RefreshWindow { width_ms, phase_ms }) + } + + const MEMORY: StorageMedium = StorageMedium::Memory; + + /// A `None` producer runs in the consumer's context and delivers on + /// every one; a `OneEvaluation` producer only on an evaluation context, + /// the one evaluation its result belongs to. Neither reads the memo. + #[test] + fn delivers_on_by_lifetime_without_windows() { + let candidates = candidates(Memo::default(), vec![]); + let unmaterialized = candidate(BUILD, QueryTime, Lifetime::None); + let one_evaluation = candidate(BUILD, QueryTime, Lifetime::OneEvaluation(MEMORY)); + let materialized = ExecutionContext::Materialized(PhysicalCandidateId(0)); + for (producer, context, delivers) in [ + (&unmaterialized, evaluation(7_000, 3_000), true), + (&unmaterialized, ingestion(5_000, 0), true), + (&unmaterialized, materialized, true), + (&one_evaluation, evaluation(7_000, 3_000), true), + (&one_evaluation, ingestion(5_000, 0), false), + (&one_evaluation, materialized, false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} in {context:?}", + producer.physical_properties + ); + } + } + + /// A chain producer with `AcrossEvaluations` of width $w$ (grid at phase + /// 0) delivers when $w$ divides the context's period (or refresh-window + /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s + /// serves both. + #[test] + #[ignore = "filled in by #1207 PR 2 (#1230)"] + fn delivers_on_a_chain_when_its_width_divides_the_context_times() { + let candidates = candidates(memo(), vec![None; 5]); + let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + let ingestion_10s = candidate( + WINDOWS_10S, + IngestionTime, + Lifetime::AcrossEvaluations(MEMORY), + ); + for (producer, context, delivers) in [ + // Evaluation grids: w divides period and phase. + (&across_5s, evaluation(10_000, 0), true), + (&across_5s, evaluation(10_000, 5_000), true), + (&across_5s, evaluation(10_000, 2_000), false), + (&across_5s, evaluation(7_000, 0), false), + (&ingestion_10s, evaluation(10_000, 0), true), + (&ingestion_10s, evaluation(10_000, 5_000), false), + // Refresh windows: w divides width and phase. + (&across_5s, ingestion(5_000, 0), true), + (&across_5s, ingestion(10_000, 5_000), true), + (&across_5s, ingestion(10_000, 2_500), false), + (&ingestion_10s, ingestion(5_000, 0), false), + ] { + assert_eq!( + producer.delivers_on(&candidates, &context), + delivers, + "{:?} {:?} in {context:?}", + producer.logical_candidate_id, + producer.physical_properties + ); + } + } + + /// A stored result that is not a chain delivers when its group's + /// evaluation grid holds every time of the context: X = + /// `sum_over_time(m[5m])`'s range, read every 60 s and every 120 s, is + /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or + /// another phase. Without a known grid it serves none. + #[test] + #[ignore = "filled in by #1207 PR 2 (#1230)"] + fn delivers_on_a_stored_result_on_its_group_grid() { + let grid_60s = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, + }); + let on_grid = candidates(memo(), vec![None, None, None, None, grid_60s]); + let without_grid = candidates(memo(), vec![None; 5]); + let x = candidate(LAST_5M, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); + for (candidates, context, delivers) in [ + (&on_grid, evaluation(60_000, 0), true), + (&on_grid, evaluation(120_000, 0), true), + (&on_grid, evaluation(120_000, 60_000), true), + (&on_grid, evaluation(30_000, 0), false), + (&on_grid, evaluation(60_000, 15_000), false), + (&on_grid, ingestion(60_000, 0), true), + (&without_grid, evaluation(60_000, 0), false), + ] { + assert_eq!(x.delivers_on(candidates, &context), delivers, "{context:?}"); + } + } + + /// `producer_satisfies_physical_requirements` over every valid producer state and every valid + /// consumer state: a query-time producer serves only query-time + /// consumers, an ingestion-time producer that is not materialized only + /// ingestion-time consumers, and one materialized across evaluations + /// serves both. Chain widths do not matter: alignment is Stage 3's + /// per-context check. + #[test] + fn producer_satisfies_physical_requirements_over_every_valid_timing_and_lifetime() { + let states = |medium| { + [ + candidate(BUILD, QueryTime, Lifetime::None), + candidate(BUILD, QueryTime, Lifetime::OneEvaluation(medium)), + candidate(WINDOWS_10S, QueryTime, Lifetime::AcrossEvaluations(medium)), + candidate(BUILD, IngestionTime, Lifetime::None), + candidate( + WINDOWS_5S, + IngestionTime, + Lifetime::AcrossEvaluations(medium), + ), + ] + }; + // Rows: producer state; columns: consumer state, in `states` order. + let expected = [ + [true, true, true, false, false], + [true, true, true, false, false], + [true, true, true, false, false], + [false, false, false, true, true], + [true, true, true, true, true], + ]; + for producer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for consumer_medium in [StorageMedium::Memory, StorageMedium::Disk] { + for (producer, row) in states(producer_medium).iter().zip(expected) { + for (consumer, satisfies) in states(consumer_medium).iter().zip(row) { + assert_eq!( + producer_satisfies_physical_requirements( + producer, + consumer.physical_properties.timing() + ), + satisfies, + "{:?} -> {:?}", + producer.physical_properties, + consumer.physical_properties + ); + } + } + } + } + } +} diff --git a/crates/types/src/workload/mod.rs b/crates/types/src/workload/mod.rs index d013fd66f..45ec657fe 100644 --- a/crates/types/src/workload/mod.rs +++ b/crates/types/src/workload/mod.rs @@ -210,6 +210,49 @@ impl QueryRecurrence { _ => None, } } + + /// The grid every evaluation with this recurrence and `range_evaluation` + /// falls on, in milliseconds, anchored at the epoch (every evaluation + /// time is a multiple of it). The times' grid is the greatest common + /// divisor of a [`RepeatedDemand::FixedIntervalAt`]'s interval and phase + /// (every 20 s at phase 5 s is on a 5 s grid), of the scheduled times, or + /// a one-off query's time. A range evaluation of more than one step also + /// takes the greatest common divisor with its step, since each run + /// evaluates at every step up to its end. `None` when the times are not + /// known: a fixed interval without a declared phase (Prometheus staggers + /// rule groups by a hash, so it does not mean phase 0), an estimated + /// rate, a one-off query without a time, an unknown recurrence, or a + /// zero interval. Pass 2's chain widths and the evaluation grids of + /// Stage 3's contexts, which + /// [`PhysicalCandidate::delivers_on`](crate::stages::PhysicalCandidate::delivers_on) + /// reads, both come from it. + pub fn evaluation_grid_ms(&self, range_evaluation: Option) -> Option { + let times_grid_ms = match self { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + evaluation_phase, .. + }) => greatest_common_divisor(self.fixed_interval_ms()?, evaluation_phase.0), + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => times + .iter() + .fold(0, |grid_ms, time| greatest_common_divisor(grid_ms, time.0)), + QueryRecurrence::OneTime { + execute_at: Some(time), + .. + } => time.0, + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(_) | RepeatedDemand::EstimatedRate(_), + ) + | QueryRecurrence::OneTime { + execute_at: None, .. + } + | QueryRecurrence::Unknown => return None, + }; + Some(match range_evaluation { + Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { + greatest_common_divisor(times_grid_ms, range.step_ms) + } + _ => times_grid_ms, + }) + } } /// The most evaluation steps one range request may have, as Prometheus @@ -370,34 +413,6 @@ pub struct QueryWorkloadEntry { pub range_evaluation: Option, } -impl QueryWorkloadEntry { - /// The grid every evaluation of this query falls on, in milliseconds, - /// anchored at the epoch (every evaluation time is a multiple of it): - /// its fixed interval ([`QueryRecurrence::fixed_interval_ms`]), or with - /// a range evaluation of more than one step the greatest common divisor - /// of the interval and the step, since each run evaluates at every step - /// up to its end; with a declared evaluation phase - /// ([`RepeatedDemand::FixedIntervalAt`]) also the greatest common - /// divisor with that phase (every 20 s at phase 5 s is on a 5 s grid). - /// `None` when it does not repeat at a positive fixed interval. - pub fn evaluation_grid_ms(&self) -> Option { - let interval_ms = self.recurrence.fixed_interval_ms()?; - let step_ms = match self.range_evaluation { - Some(range) if range.step_ms > 0 && range.evaluations_per_request() > 1 => { - greatest_common_divisor(interval_ms, range.step_ms) - } - _ => interval_ms, - }; - let phase_ms = match &self.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - evaluation_phase, .. - }) => evaluation_phase.0, - _ => 0, - }; - Some(greatest_common_divisor(step_ms, phase_ms)) - } -} - impl From<&BatchEntry> for QueryWorkloadEntry { fn from(entry: &BatchEntry) -> Self { Self { @@ -1474,24 +1489,35 @@ mod tests { assert_eq!(range.evaluations_per_request(), 61); let entry = QueryWorkloadEntry::from(&hour); assert_eq!(entry.range_evaluation, Some(range)); - // Refreshes every 30 s and steps every 60 s: evaluations on a 30 s grid. - assert_eq!(entry.evaluation_grid_ms(), Some(30_000)); - assert_eq!( - QueryWorkloadEntry::from(&plain).evaluation_grid_ms(), - Some(30_000) - ); + // A fixed interval without a declared phase: the evaluation times + // are not known, so there is no grid. + let grid = |entry: &QueryWorkloadEntry| { + entry.recurrence.evaluation_grid_ms(entry.range_evaluation) + }; + assert_eq!(grid(&entry), None); + assert_eq!(grid(&QueryWorkloadEntry::from(&plain)), None); + let every = |interval_ms, phase_ms| { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(phase_ms), + }) + }; + // Refreshes every 30 s at phase 0 and steps every 60 s: evaluations + // on a 30 s grid. + let stepped = QueryWorkloadEntry { + recurrence: every(30_000, 0), + ..entry.clone() + }; + assert_eq!(grid(&stepped), Some(30_000)); // At a declared phase, also on that phase from the epoch: every // minute at phase 125 s is on a 5 s grid; at phase 120 s, on 60 s. let at_phase = |phase_ms| QueryWorkloadEntry { - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(60_000), - evaluation_phase: TimestampMs(phase_ms), - }), + recurrence: every(60_000, phase_ms), range_evaluation: None, ..entry.clone() }; - assert_eq!(at_phase(125_000).evaluation_grid_ms(), Some(5_000)); - assert_eq!(at_phase(120_000).evaluation_grid_ms(), Some(60_000)); + assert_eq!(grid(&at_phase(125_000)), Some(5_000)); + assert_eq!(grid(&at_phase(120_000)), Some(60_000)); // A one-step range (span 0) is one evaluation per request: its // step does not narrow the grid (refreshing every 60 s at a 45 s // step would otherwise give a 15 s grid for nothing). @@ -1500,12 +1526,10 @@ mod tests { step_ms: 45_000, span_ms: 0, }), - recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( - RepetitionInterval(60_000), - )), + recurrence: every(60_000, 0), ..entry.clone() }; - assert_eq!(one_step.evaluation_grid_ms(), Some(60_000)); + assert_eq!(grid(&one_step), Some(60_000)); let validate = |range: RangeEvaluation| { let mut workload = base_workload(); diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index c695b7d25..de768a356 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -7,7 +7,9 @@ use asap_types::ir::operator::operator_properties::{Reduction, Source}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{SketchAlgorithm, SummaryUpdate}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, TimeRangeKind}; +use asap_types::ir::{ + ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, TimeRangeKind, +}; const MINUTE: u64 = 60_000; @@ -85,3 +87,213 @@ fn names_before_the_tumbling_window_rename_still_read() { operator ); } + +/// A scan with a `host` label, then the chain, the plain build and the +/// time selections over it by host. +fn scan_with_host() -> Rc { + OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("host", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap() +} + +fn node(operator: Operator) -> Rc { + let node = OperatorNode::new_shared(operator).unwrap(); + node.validate_structure().unwrap(); + node +} + +/// The `SummaryAgg` (`chain` false) or the 1-min chain (`chain` true) of +/// `summary_type` by host over `input`. +fn by_host(input: Rc, summary_type: FieldDataType, chain: bool) -> Operator { + let reduction = Reduction::by(vec![1]); + let update = SummaryUpdate::column(ColumnRef::SampleValue); + Operator::ASAP(if chain { + ASAPOp::SummaryTumblingWindowChain { + child: input, + summary_type, + input: update, + reduction, + filter: None, + tumbling_window_width_ms: MINUTE, + } + } else { + ASAPOp::SummaryAgg { + child: input, + summary_type, + input: update, + reduction, + filter: None, + } + }) +} + +/// `TimeRange(5m, TimeShift(1h, input))`. +fn five_minutes_an_hour_ago(input: Rc) -> Rc { + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: asap_types::ir::operator::operator_properties::TimeShift { + offset_ms: 60 * MINUTE as i64, + at: None, + }, + child: input, + })); + node(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Range, + child: shifted, + })) +} + +fn merge(children: Vec>, group_by: Option>) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by }) +} + +/// A chain's output is a state with a time index, one row per window and +/// key: (window_end, keys, state), unique by (window_end, keys), for a +/// keyed chain and a per-series one alike. `window_end` replaces the +/// input's `ts`. The plain build by host has no time index. +#[test] +fn a_chain_is_a_state_with_a_time_index() { + let names = |node: &Rc| -> Vec { + node.schema.fields.iter().map(|f| f.name.clone()).collect() + }; + let by_host_chain = node(by_host(scan_with_host(), kll(), true)); + let per_series = node(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan_with_host(), + summary_type: kll(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: MINUTE, + })); + for chain in [&by_host_chain, &per_series] { + assert_eq!(chain.result_kind, OperatorResultKind::State); + assert_eq!(chain.schema.time_index, Some(0)); + assert_eq!(names(chain), ["window_end", "host", "state"]); + assert_eq!(chain.schema.unique_keys, [vec![0, 1]]); + } + let build = node(by_host(scan_with_host(), kll(), false)); + assert_eq!(build.schema.time_index, None); + let without_labels = chain(scan()); + assert_eq!(names(&without_labels), ["window_end", "state"]); + assert_eq!(without_labels.schema.unique_keys, [vec![0]]); +} + +/// `TimeRange` and `TimeShift` accept a time-indexed state, a chain or a +/// selection over one, and keep it a state with the chain's schema; over +/// a state without a time index (a plain build) they are rejected. +#[test] +fn time_range_and_time_shift_select_a_chains_windows() { + let windows = node(by_host(scan_with_host(), kll(), true)); + let selected = five_minutes_an_hour_ago(windows.clone()); + assert_eq!(selected.result_kind, OperatorResultKind::State); + assert_eq!(selected.schema, windows.schema); + let build = node(by_host(scan_with_host(), kll(), false)); + let over_build = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Range, + child: build, + })) + .unwrap(); + assert!(over_build.validate_structure().is_err()); +} + +/// A `TimeRange` over a chain selects a range of windows; an instant one +/// is rejected. +#[test] +fn an_instant_time_range_over_a_chain_is_rejected() { + let instant = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(5 * MINUTE), + kind: TimeRangeKind::Instant, + child: node(by_host(scan_with_host(), kll(), true)), + })) + .unwrap(); + assert!(instant.validate_structure().is_err()); +} + +/// Merging the windows `TimeRange(5m, TimeShift(1h, chain(X)))` selects +/// gives the state of `SummaryAgg(TimeRange(5m, TimeShift(1h, X)))`: the +/// same schema and kind, and so the same schema read out, finalized +/// (an exact sum) and rolled up. +#[test] +fn merged_windows_are_the_build_over_the_selected_range() { + let sum = FieldDataType::exact(asap_types::ir::schema::ExactKind::Sum); + for summary_type in [kll(), sum.clone()] { + let merged = node(merge( + vec![five_minutes_an_hour_ago(node(by_host( + scan_with_host(), + summary_type.clone(), + true, + )))], + None, + )); + let build = node(by_host( + five_minutes_an_hour_ago(scan_with_host()), + summary_type.clone(), + false, + )); + assert_eq!(merged.schema, build.schema, "{summary_type:?}"); + assert_eq!(merged.result_kind, build.result_kind); + let read = |state: &Rc| { + node(if summary_type == sum { + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { + child: state.clone(), + }) + } else { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state.clone(), + query: asap_types::ir::schema::SketchStatistic::Quantile { q: 0.95 }, + }) + }) + }; + let (read_merged, read_build) = (read(&merged), read(&build)); + assert_eq!(read_merged.schema, read_build.schema, "{summary_type:?}"); + assert_eq!(read_merged.result_kind, read_build.result_kind); + // Rolled up to one state over every host. + let (rolled_merged, rolled_build) = ( + node(merge(vec![merged], Some(vec![]))), + node(merge(vec![build], Some(vec![]))), + ); + assert_eq!(rolled_merged.schema, rolled_build.schema); + let (read_merged, read_build) = (read(&rolled_merged), read(&rolled_build)); + assert_eq!(read_merged.schema, read_build.schema, "{summary_type:?}"); + } +} + +/// A merge over a time-indexed state reads one `TimeRange` over a chain: +/// not the whole chain, not a `TimeShift` without a range, and not beside +/// another input. +#[test] +fn a_merge_over_a_chain_reads_one_time_range() { + let windows = || node(by_host(scan_with_host(), kll(), true)); + let shifted = node(Operator::NonASAP(NonASAPOp::TimeShift { + shift: asap_types::ir::operator::operator_properties::TimeShift { + offset_ms: MINUTE as i64, + at: None, + }, + child: windows(), + })); + for children in [ + vec![windows()], + vec![shifted], + vec![ + five_minutes_an_hour_ago(windows()), + five_minutes_an_hour_ago(windows()), + ], + ] { + assert!(OperatorNode::new_shared(merge(children, None)).is_err()); + } + assert!( + OperatorNode::new_shared(merge(vec![five_minutes_an_hour_ago(windows())], None)).is_ok() + ); +} From 36312cbafcd7031fee17f9c89f601db354182120 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:34:13 +0000 Subject: [PATCH 26/30] feat(stage0, types): memo hash-consing, group properties (column_sign, depends_on_evaluation_time, aggregate_coverage), @ end()/@ start() lifted above their subtree (#1207 PR 2) - Memo: hash-consing (find_or_add_group by expression hash), candidate storage (add_candidate), add_query(MemoQuery); Memo::new(declared column signs), private declared_column_signs read by a Scan's column_sign. - Stage 0: memo_of_queries(roots, accuracy_targets, data) adds each query with its accuracy target and n_q (values_per_result over series_read), and collects the declared signs of the scanned sources (a counter's value is non-negative without a declaration). - GroupProperties::column_sign derived per operator (concepts doc, "Value sign"); depends_on_evaluation_time: a TimeRange or time()/now() makes a result depend on the evaluation time, @ T pins it; reads_only_time_series: a Scan of a time series, or every input group reading only time series (OperatorInput::reads_only_time_series computes the same rule over node trees; the recursive walks in topk.rs and Pass 1 are deleted). - aggregate_coverage for summary states, chains, TimeRange/TimeShift over a chain, merges and exact Aggregates; timestamp constraints are IntervalConstraintLimit intervals (t - o - W, t - o]; SQL timestamp literals and identity casts fold so absolute filters lift. - PromQL: @ end() / @ start() are lowered as TimeShift { at } above the selector or subquery they anchor, which is lowered without them, so it is the group of the same expression without @; coverage stops at that TimeShift (it is the definition's input group). --- crates/frontend-common/src/canonicalize.rs | 2 +- crates/frontend-common/src/resolve.rs | 20 +- crates/frontend-promql/src/lib.rs | 47 +- crates/frontend-promql/src/promql.rs | 94 ++- .../tests/promql_conformance.rs | 33 +- crates/frontend-sql/src/sql/expr.rs | 21 +- crates/frontend-sql/src/sql/types.rs | 7 + crates/frontend-sql/tests/sql_lowering.rs | 40 +- crates/stage0-memo/src/lib.rs | 637 +++++++++++++++- .../src/per_computation_candidates.rs | 13 +- crates/types/src/ir/operator/input.rs | 31 + crates/types/src/ir/operator/mod.rs | 6 +- .../src/ir/operator/operator_properties.rs | 10 + .../src/ir/properties/summary_coverage.rs | 317 ++++++-- crates/types/src/ir/schema/schema_support.rs | 2 +- crates/types/src/ir/schema/topk.rs | 13 +- crates/types/src/stages/candidates.rs | 543 +++++++++++++- crates/types/src/stages/column_sign.rs | 449 ++++++++++++ crates/types/src/stages/hash_consing.rs | 59 ++ crates/types/src/stages/memo.rs | 37 +- crates/types/src/stages/mod.rs | 2 + .../src/stages/physical/memo_candidates.rs | 4 +- crates/types/tests/main.rs | 2 + crates/types/tests/memo_group_properties.rs | 492 +++++++++++++ crates/types/tests/memo_summary_coverage.rs | 683 ++++++++++++++++++ .../proposals/asap-primitive-schema.md | 244 ++++--- 26 files changed, 3524 insertions(+), 284 deletions(-) create mode 100644 crates/types/src/stages/column_sign.rs create mode 100644 crates/types/src/stages/hash_consing.rs create mode 100644 crates/types/tests/memo_group_properties.rs create mode 100644 crates/types/tests/memo_summary_coverage.rs diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index cfa2dcd4c..ffd5b7e0d 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -1,7 +1,7 @@ //! Post-lowering canonicalization of the operator DAG. //! //! Rewrites equivalent spellings of a query into one shape, so that common -//! sub-DAG sharing (Stage 0 hash-consing, `cse` in `asap_stage0_memo`) can match +//! sub-DAG sharing (hash-consing, `Memo::find_or_add_group`) can match //! them by structural equality, whichever front end produced them. //! //! ## Heavy-hitter promotion diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 003b87680..1a4c230f3 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -487,11 +487,21 @@ fn resolve_expr_in( S::Not(e) => ScalarExpr::Not(bx(e)?), S::IsNull(e) => ScalarExpr::IsNull(bx(e)?), S::IsNotNull(e) => ScalarExpr::IsNotNull(bx(e)?), - S::Cast { expr, to, try_cast } => ScalarExpr::Cast { - expr: bx(expr)?, - to: to.clone(), - try_cast: *try_cast, - }, + // A cast to the type its operand already has is the operand: SQL's + // `ts >= '2026-11-01'` casts the Timestamp column `ts` to DataFusion's + // nanosecond timestamp, which is the same IR type. + S::Cast { expr, to, try_cast } => { + let operand = bx(expr)?; + if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + *operand + } else { + ScalarExpr::Cast { + expr: operand, + to: to.clone(), + try_cast: *try_cast, + } + } + } S::InList { expr, list, diff --git a/crates/frontend-promql/src/lib.rs b/crates/frontend-promql/src/lib.rs index fa90b507d..f760712a7 100644 --- a/crates/frontend-promql/src/lib.rs +++ b/crates/frontend-promql/src/lib.rs @@ -117,11 +117,12 @@ mod tests { use std::rc::Rc; use std::time::Duration; + use asap_types::ir::operator::operator_properties::{AtModifier, TimeShift}; use asap_types::ir::operator::Source; - use asap_types::ir::{NonASAPOp, OperatorNode, TimeRangeKind}; + use asap_types::ir::{NonASAPOp, OperatorNode, QueryRoot, TimeRangeKind}; use asap_types::workload::{ BatchEntry, DataArrival, DataWorkload, PlanningWorkload, Query, QueryRequirements, - QueryWorkload, SourceFacts, WorkloadError, + QueryWorkload, RangeEvaluation, SourceFacts, WorkloadError, }; use super::*; @@ -366,4 +367,46 @@ mod tests { ) .is_empty()); } + + /// `@ end()` and `@ start()` are lifted above the selector they anchor, + /// which is lowered without them (its offset stays): the selector of + /// `sum_over_time(data[5m] @ start() offset 1m)` is + /// `TimeShift(@ start())` over the selector of + /// `sum_over_time(data[5m] offset 1m)`, so the two share it. Instant and + /// range-evaluated queries lower alike. + #[test] + fn at_start_and_at_end_are_lifted_above_the_selector_they_anchor() { + let lowered = |query: &str| { + let roots = lower_promql_query_workload(&workload(query)).unwrap(); + let [QueryRoot::Operator(root)] = &roots[..] else { + panic!("one operator root"); + }; + Rc::clone(root.children()[0]) + }; + let pinned = lowered("sum_over_time(data[5m] @ start() offset 1m)"); + let NonASAPOp::TimeShift { shift, child } = pinned.expect_non_asap() else { + panic!("expected the lifted TimeShift, got {pinned:?}"); + }; + assert_eq!( + *shift, + TimeShift { + offset_ms: 0, + at: Some(AtModifier::Start), + } + ); + assert_eq!(*child, lowered("sum_over_time(data[5m] offset 1m)")); + + let mut range_query = workload("sum_over_time(data[5m] @ start() offset 1m)"); + if let Some(batch) = range_query.query_workload.query_batch.as_mut() { + batch[0].range_evaluation = Some(RangeEvaluation { + step_ms: 60_000, + span_ms: 3_600_000, + }); + } + assert_eq!( + lower_promql_query_workload(&range_query).unwrap(), + lower_promql_query_workload(&workload("sum_over_time(data[5m] @ start() offset 1m)")) + .unwrap() + ); + } } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 66aa42a1b..0313186f8 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -55,7 +55,8 @@ //! | `m{f}` / `m{f}[w]` | `TimeRange{ingestion, Instant, Scan{predicates}}` / `TimeRange{w, Range, Scan}` | //! | `a OP b` | `BinaryOp{vector_match}` | //! | `expr[r:res]` | `PromqlSubquery{r, res}` | -//! | ` offset ` / ` @ `/`start()`/`end()` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | +//! | ` offset ` / ` @ ` | `TimeShift{shift}` over the selector's `Scan` — pass-through schema; a ranged selector shifts under its `TimeRange` (issue #40) | +//! | ` @ start()`/`end()` (also a subquery's) | `TimeShift{at: Start \| End}` above the selector `X`, lowered without the `@` (its offset stays in `X`), so `X` is the group of the same selector without `@` | use std::collections::BTreeMap; use std::rc::Rc; @@ -234,14 +235,16 @@ impl PromqlLowerer<'_> { }; // `offset`/`@` move the whole subquery, including its step grid. let shift = time_shift(sq.offset.as_ref(), sq.at.as_ref())?; - Ok(if shift.is_identity() { - subquery - } else { - Unresolved::TimeShift { - shift, - child: Rc::new(subquery), + Ok(anchored(shift, |shift| { + if shift.is_identity() { + subquery + } else { + Unresolved::TimeShift { + shift, + child: Rc::new(subquery), + } } - }) + })) } Expr::VectorSelector(vs) => { let (metric, matchers, shift) = vs_parts(vs)?; @@ -249,11 +252,13 @@ impl PromqlLowerer<'_> { } Expr::MatrixSelector(ms) => { let (metric, matchers, shift) = vs_parts(&ms.vs)?; - Ok(Unresolved::TimeRange { - range: ms.range, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) + Ok(selector( + metric, + matchers, + shift, + ms.range, + TimeRangeKind::Range, + )) } // Scalar-typed, dispatched above; kept for exhaustiveness. String // literals only appear as function args (`label_replace`, …), so a @@ -1120,11 +1125,13 @@ impl PromqlLowerer<'_> { intent: AggIntent, ) -> Result { let child = match inner.window { - Some(w) => Unresolved::TimeRange { - range: w, - kind: TimeRangeKind::Range, - child: Rc::new(filtered_source(inner.metric, inner.matchers, inner.shift)), - }, + Some(w) => selector( + inner.metric, + inner.matchers, + inner.shift, + w, + TimeRangeKind::Range, + ), None => self.instant_source(inner.metric, inner.matchers, inner.shift)?, }; let reduction = reduction_for(&keys, inner.window.is_some() || intent.is_per_series()); @@ -1150,11 +1157,52 @@ impl PromqlLowerer<'_> { shift: TimeShift, ) -> Result { let interval = self.sample_interval(&metric)?; - Ok(Unresolved::TimeRange { - range: interval, - kind: TimeRangeKind::Instant, - child: Rc::new(filtered_source(metric, matchers, shift)), - }) + Ok(selector( + metric, + matchers, + shift, + interval, + TimeRangeKind::Instant, + )) + } +} + +/// A selector over `metric`: the `TimeRange` of `range` and `kind` over its +/// filtered `Scan`, shifted by `shift` ([`anchored`]). +fn selector( + metric: String, + matchers: Vec, + shift: TimeShift, + range: Duration, + kind: TimeRangeKind, +) -> Unresolved { + anchored(shift, |shift| Unresolved::TimeRange { + range, + kind, + child: Rc::new(filtered_source(metric, matchers, shift)), + }) +} + +/// The subtree `lower` builds with `shift`, its `@ start()` / `@ end()` +/// lifted above it: `lower` gets `shift` without them (only the offset and +/// an `@ ` stay in the subtree), and `TimeShift { at: Start | End }` +/// reads its result. The subtree is then the group of the same expression +/// without `@`, shared with any query over it; in a range query it runs +/// once per request, at the request's end (or start), and in an instant +/// query both are the evaluation time. +fn anchored(shift: TimeShift, lower: impl FnOnce(TimeShift) -> Unresolved) -> Unresolved { + match shift.at { + Some(at @ (AtModifier::Start | AtModifier::End)) => Unresolved::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(at), + }, + child: Rc::new(lower(TimeShift { + offset_ms: shift.offset_ms, + at: None, + })), + }, + _ => lower(shift), } } diff --git a/crates/frontend-promql/tests/promql_conformance.rs b/crates/frontend-promql/tests/promql_conformance.rs index f96ecc8f4..6744d7845 100644 --- a/crates/frontend-promql/tests/promql_conformance.rs +++ b/crates/frontend-promql/tests/promql_conformance.rs @@ -39,7 +39,7 @@ use crate::support; use asap_frontend_promql::PromqlError as LoweringError; use asap_types::ir::operator::{ AggIntent, AtModifier, BinaryOpKind, PromQLVectorSetOpKind, Reduction, SampleKind, Source, - TimeFunc, + TimeFunc, TimeShift, }; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::scalar::{ArithmeticOpKind, CompareOpKind, ScalarValue}; @@ -1039,7 +1039,9 @@ fn offset_modifier_lowers_to_a_time_shift() { fn at_modifier_lowers_to_a_time_shift() { // SEMANTICS (PromQL, issue #40): `@ ` pins the evaluation to an absolute // instant (PromQL seconds → IR milliseconds); `@ start()` / `@ end()` anchor - // to the query range bounds. + // to the query range bounds: a `TimeShift { at }` above the selector, which + // is lowered without the `@`, so it is the selector of the same query + // without `@`. let qe = ok("http_requests_total @ 1609746000"); let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { panic!("expected an ingestion TimeRange"); @@ -1051,24 +1053,27 @@ fn at_modifier_lowers_to_a_time_shift() { assert_eq!(shift.offset_ms, 0); let qe = ok("http_requests_total @ start()"); - let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { - panic!("expected an ingestion TimeRange"); - }; - let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { - panic!("expected a TimeShift for `@ start()`"); + let NonASAPOp::TimeShift { shift, child } = qe.expect_non_asap() else { + panic!("expected a TimeShift for `@ start()`, got {qe:?}"); }; - assert_eq!(shift.at, Some(AtModifier::Start)); + assert_eq!( + *shift, + TimeShift { + offset_ms: 0, + at: Some(AtModifier::Start) + } + ); + assert_eq!(**child, *ok("http_requests_total")); - // Offset and `@` compose: `@ end() offset 5m` carries both. + // Offset and `@` compose: `@ end() offset 5m` is `@ end()` over the + // selector with `offset 5m`. let qe = ok("http_requests_total @ end() offset 5m"); - let NonASAPOp::TimeRange { child, .. } = qe.expect_non_asap() else { - panic!("expected an ingestion TimeRange, got {qe:?}"); - }; - let NonASAPOp::TimeShift { shift, .. } = child.expect_non_asap() else { + let NonASAPOp::TimeShift { shift, child } = qe.expect_non_asap() else { panic!("expected a TimeShift, got {qe:?}"); }; assert_eq!(shift.at, Some(AtModifier::End)); - assert_eq!(shift.offset_ms, 300_000); + assert_eq!(shift.offset_ms, 0); + assert_eq!(**child, *ok("http_requests_total offset 5m")); } #[test] diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 7e706c414..4e51a9dc9 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -1,5 +1,6 @@ use std::rc::Rc; +use datafusion::arrow::datatypes::{DataType as ArrowDataType, TimeUnit}; use datafusion::logical_expr::utils::{split_binary, split_conjunction}; use datafusion::logical_expr::{BinaryExpr, Expr, Operator}; @@ -137,17 +138,27 @@ impl SqlLowerer<'_> { Expr::IsNotNull(inner) => Ok(Unresolved::IsNotNull(bx(inner)?)), - // Fold a numeric cast of a literal, as DataFusion's constant folding - // does: type coercion turns `latency < 100` on a DOUBLE column into - // `latency < CAST(100 AS DOUBLE)`, which lowers as `latency < 100.0`. + // Fold a numeric or timestamp cast of a literal, as DataFusion's + // constant folding does: type coercion turns `latency < 100` on a + // DOUBLE column into `latency < CAST(100 AS DOUBLE)`, which lowers + // as `latency < 100.0`, and `ts >= '2026-11-01'` into a cast of the + // string, which lowers as a `Timestamp` literal (ms since the epoch). Expr::Cast(c) if matches!(c.expr.as_ref(), Expr::Literal(v, _) if !v.is_null()) - && (c.field.data_type().is_integer() || c.field.data_type().is_floating()) => + && (c.field.data_type().is_integer() + || c.field.data_type().is_floating() + || matches!(c.field.data_type(), ArrowDataType::Timestamp(..))) => { let Expr::Literal(sv, _) = c.expr.as_ref() else { unreachable!() }; - match sv.cast_to(c.field.data_type()) { + let to = match c.field.data_type() { + ArrowDataType::Timestamp(..) => { + ArrowDataType::Timestamp(TimeUnit::Millisecond, None) + } + other => other.clone(), + }; + match sv.cast_to(&to) { Ok(folded) => scalar_value_to_asap(&folded).map(Unresolved::Literal), Err(_) => Ok(Unresolved::Cast { expr: bx(&c.expr)?, diff --git a/crates/frontend-sql/src/sql/types.rs b/crates/frontend-sql/src/sql/types.rs index cf18399ad..08cb21fa9 100644 --- a/crates/frontend-sql/src/sql/types.rs +++ b/crates/frontend-sql/src/sql/types.rs @@ -53,6 +53,13 @@ pub(super) fn scalar_value_to_asap(sv: &DfScalarValue) -> Result Ok(ScalarValue::Utf8(s.clone())), DfScalarValue::Boolean(Some(b)) => Ok(ScalarValue::Boolean(*b)), + // A timestamp is ms since the epoch, as the executor's columns. + DfScalarValue::TimestampSecond(Some(s), _) => Ok(ScalarValue::Timestamp(s * 1_000)), + DfScalarValue::TimestampMillisecond(Some(ms), _) => Ok(ScalarValue::Timestamp(*ms)), + DfScalarValue::TimestampMicrosecond(Some(us), _) => Ok(ScalarValue::Timestamp(us / 1_000)), + DfScalarValue::TimestampNanosecond(Some(ns), _) => { + Ok(ScalarValue::Timestamp(ns / 1_000_000)) + } // All three of DataFusion's interval scalars land on one canonical // shape; the narrower two simply leave the fields they do not carry // at zero. diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 84829c04b..2a83de38e 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2210,7 +2210,9 @@ async fn lead_in_frame_lowers_to_its_own_kind_not_lead() { /// PromQL's Float64 Unix-seconds `EvalTimestamp`. #[tokio::test] async fn now_in_predicate_lowers_to_current_timestamp() { - // WHERE folds onto Scan.predicates (no explicit Filter node). + // WHERE folds onto Scan.predicates (no explicit Filter node). The + // coercion cast of `now()` to the column's timestamp type is dropped: + // `now()` is already a Timestamp. let qe = lower("SELECT * FROM metrics WHERE ts < NOW()").await; let NonASAPOp::Project { child, .. } = op(&qe) else { panic!("expected Project at root, got {qe:?}"); @@ -2221,7 +2223,7 @@ async fn now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "NOW() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -2259,7 +2261,7 @@ async fn clickhouse_now_in_predicate_lowers_to_current_timestamp() { assert_eq!(predicates.len(), 1); assert!( matches!(&predicates[0].0, ScalarExpr::Compare { right, .. } - if matches!(right.as_ref(), ScalarExpr::Cast { expr, to: DataType::Timestamp, .. } if matches!(expr.as_ref(), ScalarExpr::CurrentTimestamp))), + if matches!(right.as_ref(), ScalarExpr::CurrentTimestamp)), "now() must lower to CurrentTimestamp, got {:?}", predicates[0].0 ); @@ -3199,3 +3201,35 @@ async fn trailing_window_on_an_aliased_table_and_under_asap_rate() { let (window, _) = trailing_window(below).expect("the WHERE window below it"); assert_eq!(window.as_millis(), 300_000); } + +// An absolute timestamp filter compares the time column with a `Timestamp` +// literal (ms since the epoch), so summary coverage lifts it into an +// interval on the timestamp column: the cast of the string folds, and the +// cast of the Timestamp column to DataFusion's nanosecond unit is dropped. +#[tokio::test] +async fn an_absolute_timestamp_filter_compares_the_column_with_a_timestamp_literal() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + for sql in [ + "SELECT COUNT(*) FROM metrics WHERE ts >= '2026-11-01'", + "SELECT COUNT(*) FROM metrics WHERE ts >= TIMESTAMP '2026-11-01 00:00:00'", + ] { + let qe = lower(sql).await; + let scan = OperatorNode::reachable(&qe) + .into_iter() + .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Scan { .. }))) + .unwrap_or_else(|| panic!("no Scan for {sql}")); + let NonASAPOp::Scan { predicates, .. } = op(&scan) else { + unreachable!() + }; + assert_eq!( + predicates.iter().map(|pred| &pred.0).collect::>(), + [&ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Ge, + right: Box::new(ScalarExpr::Literal(ScalarValue::Timestamp(NOV_1_2026_MS))), + semantics: ExprSemantics::Sql, + }], + "{sql}" + ); + } +} diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index ce293d080..d34901d2b 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -1,30 +1,639 @@ //! `asap-stage0-memo` — #509 Stage 0: the memo groups of the workload's //! query roots (a [`Memo`] without candidates). Each node of the queries' //! DAGs becomes the group of its expression, found or added by -//! [`Memo::find_or_add_group`], so identical sub-expressions are one group. -//! Stage 1 starts from these groups. +//! [`Memo::find_or_add_group`], so identical sub-expressions, within one +//! query or across queries, are one group. Stage 1 starts from these groups. //! -//! It depends only on `asap-types`, where its output (`Memo`) lives; no -//! other stage crate depends on it (the frontends' tests aside, listed in -//! `crate_graph.rs`), and the planner facade calls it first. +//! It depends only on `asap-types`, where its output (`Memo`) lives and +//! where the one expression equality is; no other stage crate depends on +//! it, and the planner facade calls it first. +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::rc::Rc; + +use asap_types::ir::operator::SourceColumn; use asap_types::ir::properties::AccuracyTarget; -use asap_types::ir::{QueryRoot, SchemaDerivationError}; -use asap_types::stages::Memo; -use asap_types::workload::DataWorkload; +use asap_types::ir::{NonASAPOp, Operator, OperatorNode, QueryRoot, SchemaDerivationError}; +use asap_types::stages::{Memo, MemoGroupId, MemoQuery, ValueSign}; +use asap_types::workload::bounds::{series_read, source_name, values_per_result}; +use asap_types::workload::{DataWorkload, PrometheusMetricType}; /// Stage 0: the memo groups of the workload's query roots, query `i` /// being `roots[i]` with accuracy target `accuracy_targets[i]` (`None`: /// none), every group without candidates. Each query is added with its -/// $n_q$, the number of values one evaluation returns, which `data` bounds; -/// a `Scan`'s column signs are `data`'s declared ones. Fails with the first -/// node that breaks the node rules over its children's groups -/// ([`Memo::find_or_add_group`]). +/// $n_q$, the number of values one evaluation returns, which `data` bounds +/// ([`values_per_result`]); a `Scan`'s column signs are the ones `data` +/// declares ([`declared_column_signs`]). Fails with the first node that +/// breaks the node rules over its children's groups +/// ([`Memo::find_or_add_group`]); the facade validates the roots first, so +/// that is a frontend bug. pub fn memo_of_queries( roots: &[QueryRoot], accuracy_targets: &[Option], data: &DataWorkload, ) -> Result { - let _ = (roots, accuracy_targets, data); - todo!("#1207 PR 2: Stage 0 over Memo::find_or_add_group, each query with its target and n_q") + assert_eq!( + roots.len(), + accuracy_targets.len(), + "one accuracy target per query" + ); + let mut memo = Memo::new(declared_column_signs(roots, data)); + let mut group_of_node = GroupOfNode::new(); + for root in roots { + for node in root.operator_refs() { + group_of(&mut memo, &mut group_of_node, node)?; + } + } + let group = |node: &Rc| group_of_node[&Rc::as_ptr(node)]; + for (root, accuracy_target) in roots.iter().zip(accuracy_targets) { + memo.add_query(MemoQuery { + root: match root { + QueryRoot::Operator(node) => QueryRoot::Operator(group(node)), + QueryRoot::Scalar(expr) => { + QueryRoot::Scalar(expr.map_operator_refs(&mut |node| group(node))) + } + }, + accuracy_target: accuracy_target.clone(), + values_per_evaluation: values_per_result(root, series_read(root, data), data), + }); + } + Ok(memo) +} + +/// The sign `data` declares for each column of each source the queries +/// scan ([`SourceFacts::column_signs`](asap_types::workload::SourceFacts::column_signs), +/// a histogram's or summary's for its series, [`DataWorkload::series_facts`]); +/// a declared counter's `value` is non-negative without a declaration. +fn declared_column_signs( + roots: &[QueryRoot], + data: &DataWorkload, +) -> BTreeMap { + let mut signs = BTreeMap::new(); + let mut visited = HashSet::new(); + let mut pending: Vec<&Rc> = + roots.iter().flat_map(QueryRoot::operator_refs).collect(); + while let Some(node) = pending.pop() { + if !visited.insert(Rc::as_ptr(node)) { + continue; + } + pending.extend(node.children()); + let Operator::NonASAP(NonASAPOp::Scan { source, .. }) = &node.operator else { + continue; + }; + let source = source_name(source); + let Some((facts, series_type)) = data.series_facts(source) else { + continue; + }; + let column = |name: &str| SourceColumn { + source: source.to_owned(), + column: name.to_owned(), + }; + for (name, sign) in &facts.column_signs { + signs.insert(column(name), *sign); + } + if series_type == Some(PrometheusMetricType::Counter) { + signs + .entry(column("value")) + .or_insert(ValueSign::NonNegative); + } + } + signs +} + +/// Each node of the queries' DAGs already visited, by address, to its +/// group: a node reached through several paths is visited once. The roots +/// outlive the map, so no address is reused. +type GroupOfNode = HashMap<*const OperatorNode, MemoGroupId>; + +/// The group of `node`: its operator over its children's groups (each +/// found or added first), found or added by [`Memo::find_or_add_group`]. +fn group_of( + memo: &mut Memo, + group_of_node: &mut GroupOfNode, + node: &Rc, +) -> Result { + if let Some(&group) = group_of_node.get(&Rc::as_ptr(node)) { + return Ok(group); + } + for child in node.children() { + group_of(memo, group_of_node, child)?; + } + let expression = node + .operator + .map_children(|child| group_of_node[&Rc::as_ptr(child)]); + let group = memo.find_or_add_group(expression)?; + group_of_node.insert(Rc::as_ptr(node), group); + Ok(group) +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::operator::agg_intent::AggIntent; + use asap_types::ir::operator::asap::ASAPOp; + use asap_types::ir::operator::operator_properties::{ + BinaryOpKind, GroupKeys, Reduction, Source, + }; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::ir::scalar::expr_ir::{ColumnRef, CompareOpKind}; + use asap_types::ir::schema::sketch::{SketchAlgorithm, SketchStatistic, SummaryUpdate}; + use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; + use asap_types::ir::{BinaryOperator, NonASAPOp, Operator, ScalarExpr}; + + fn node(op: NonASAPOp) -> Rc { + OperatorNode::new_shared(Operator::NonASAP(op)).unwrap() + } + + /// `[ts, service, value, latency]`, no unique key. + fn scan() -> Rc { + node(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("service", DataType::Utf8, false), + Field::plain("value", DataType::Float64, false), + Field::plain("latency", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + fn quantile_agg(by: Vec, col: Option, q: f64) -> Rc { + node(NonASAPOp::Aggregate { + reduction: Reduction::by(by), + measures: vec![AggIntent::Quantile { col, q }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }) + } + + fn compare(lhs: Rc, rhs: Rc) -> Rc { + node(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Compare(CompareOpKind::Eq), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }) + } + + /// The memo of `roots`, one query per root, without accuracy targets + /// or data facts. + fn memo_of_roots(roots: &[QueryRoot]) -> Result { + memo_of_queries(roots, &vec![None; roots.len()], &DataWorkload::default()) + } + + /// [`memo_of_roots`] of operator roots. + fn memo_of(roots: Vec>) -> Memo { + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + memo_of_roots(&roots).unwrap() + } + + /// Query `query`'s root group. + fn root(memo: &Memo, query: usize) -> MemoGroupId { + match &memo.queries[query].root { + QueryRoot::Operator(group) => *group, + QueryRoot::Scalar(_) => panic!("expected an operator root"), + } + } + + /// Whether queries 0 and 1 of `[a, b]` are one group. + fn one_group(a: Rc, b: Rc) -> bool { + let memo = memo_of(vec![a, b]); + root(&memo, 0) == root(&memo, 1) + } + + /// Two queries sharing a sub-expression read one group for it, and + /// every group lists the queries that reach it and the groups that + /// read it. + #[test] + fn two_queries_sharing_a_sub_expression_get_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let p50_vs_p90 = compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.9), + ); + let memo = memo_of(vec![p50, p50_vs_p90]); + // scan, p50, p90, compare. + assert_eq!(memo.groups().len(), 4); + let (scan, p50, compare) = (MemoGroupId(0), root(&memo, 0), root(&memo, 1)); + let p90 = MemoGroupId(2); + assert_eq!( + memo.groups()[compare.0 as usize].expression.children(), + vec![&p50, &p90] + ); + assert_eq!(memo.groups()[scan.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p50.0 as usize].reached_by_queries(), &[0, 1]); + assert_eq!(memo.groups()[p90.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[compare.0 as usize].reached_by_queries(), &[1]); + assert_eq!(memo.groups()[scan.0 as usize].parent_groups(), &[p50, p90]); + assert_eq!(memo.groups()[p50.0 as usize].parent_groups(), &[compare]); + assert!(memo + .groups() + .iter() + .all(|group| group.candidates.is_empty())); + } + + /// One query's repeated sub-expression is one group, read twice. + #[test] + fn a_query_s_repeated_sub_expression_is_one_group() { + let memo = memo_of(vec![compare( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(2), 0.5), + )]); + assert_eq!(memo.groups().len(), 3); + let p50 = MemoGroupId(1); + assert_eq!( + memo.groups()[root(&memo, 0).0 as usize] + .expression + .children(), + vec![&p50, &p50] + ); + assert_eq!( + memo.groups()[p50.0 as usize].parent_groups(), + &[root(&memo, 0)] + ); + } + + /// A node reached through two paths is one group, visited once. + #[test] + fn an_already_shared_node_is_one_group() { + let p50 = quantile_agg(vec![1], Some(2), 0.5); + let memo = memo_of(vec![compare(Rc::clone(&p50), p50)]); + assert_eq!(memo.groups().len(), 3); + } + + /// Aggregates that differ in a column, a quantile or a filter are + /// different groups. + #[test] + fn aggregates_that_differ_in_an_argument_are_different_groups() { + use asap_types::ir::scalar::ScalarValue; + use asap_types::ir::Predicate; + assert!(!one_group( + quantile_agg(vec![1], Some(2), 0.5), + quantile_agg(vec![1], Some(3), 0.5) + )); + assert!(!one_group( + quantile_agg(vec![], Some(2), 0.5), + quantile_agg(vec![], Some(2), 0.9) + )); + let filtered = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Quantile { + col: Some(2), + q: 0.5, + }], + output_names: vec![], + filters: vec![Some(Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(3)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: asap_types::ir::ExprSemantics::Sql, + }))], + having: None, + child: scan(), + }); + assert!(!one_group(quantile_agg(vec![1], Some(2), 0.5), filtered)); + } + + /// Equal nodes without a unique key are one group when they return the + /// same rows each time: a keyless scan, a global aggregate, a `without` + /// aggregate. + #[test] + fn equal_keyless_nodes_that_return_the_same_rows_are_one_group() { + let table_scan = || { + node(NonASAPOp::Scan { + source: Source::Table { + table_ref: "metrics".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, true), + ], + 0, + vec![], + ), + }) + }; + assert!(one_group(table_scan(), table_scan())); + assert!(one_group( + quantile_agg(vec![], Some(2), 0.9), + quantile_agg(vec![], Some(2), 0.9) + )); + let without = || { + node(NonASAPOp::Aggregate { + reduction: Reduction::Reduce(GroupKeys::without(vec![0])), + measures: vec![AggIntent::Count], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }) + }; + assert!(one_group(without(), without())); + } + + /// Queries that differ only in their accuracy target are one group: the + /// target is the query's ([`MemoQuery::accuracy_target`]), not an + /// argument of its operators. + #[test] + fn queries_that_differ_only_in_their_accuracy_target_are_one_group() { + let roots = [ + QueryRoot::Operator(quantile_agg(vec![1], Some(2), 0.99)), + QueryRoot::Operator(quantile_agg(vec![1], Some(2), 0.99)), + ]; + let targets = [ + Some(AccuracyTarget::max_error(0.01)), + Some(AccuracyTarget::max_error(0.05)), + ]; + let memo = memo_of_queries(&roots, &targets, &DataWorkload::default()).unwrap(); + assert_eq!(root(&memo, 0), root(&memo, 1)); + assert_eq!(memo.groups().len(), 2); + assert_eq!(memo.queries[0].accuracy_target, targets[0]); + assert_eq!(memo.queries[1].accuracy_target, targets[1]); + } + + /// `vector(scalar(sum by (service) (m)))`. + fn scalar_of_vector() -> Rc { + let sum_up = node(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![1]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec![], + filters: vec![], + having: None, + child: scan(), + }); + node(NonASAPOp::PromqlVectorFromScalar( + ScalarExpr::PromqlScalarFromVector(sum_up), + )) + } + + /// A vector read through `scalar(v)` is a child group like any input, + /// one group across queries, and a scalar query's root reads it. + #[test] + fn a_scalar_referenced_vector_is_one_group_across_queries() { + let roots = vec![ + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Operator(scalar_of_vector()), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let memo = memo_of_roots(&roots).unwrap(); + // scan, sum, bridge. + assert_eq!(memo.groups().len(), 3); + assert_eq!(root(&memo, 0), root(&memo, 1)); + let sum = MemoGroupId(1); + assert_eq!(memo.queries[2].root.operator_refs(), vec![&sum]); + assert_eq!( + memo.groups()[sum.0 as usize].reached_by_queries(), + &[0, 1, 2] + ); + } + + /// `SummaryAgg[DDSketch](scan)`. + fn summary_agg() -> Rc { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child: scan(), + summary_type: FieldDataType::Sketch(SketchAlgorithm::DDSketch), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + })) + .unwrap() + } + + /// Two quantile readouts of one summary are two groups over one state + /// group. + #[test] + fn readouts_share_their_state_group_but_not_each_other() { + let readout = |q| { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: summary_agg(), + query: SketchStatistic::Quantile { q }, + })) + .unwrap() + }; + let memo = memo_of(vec![readout(0.95), readout(0.99)]); + let (p95, p99) = (root(&memo, 0), root(&memo, 1)); + assert_ne!(p95, p99); + assert_eq!( + memo.groups()[p95.0 as usize].expression.children(), + memo.groups()[p99.0 as usize].expression.children() + ); + assert_eq!(memo.groups().len(), 4); + } + + /// `DISTINCT` on every column returns the same rows each time and is one + /// group; a `Dedup` on some columns keeps an unspecified row per key. + #[test] + fn dedup_is_one_group_only_on_every_column() { + let dedup = |cols: Vec| { + node(NonASAPOp::Dedup { + cols, + child: scan(), + }) + }; + assert!(one_group(dedup(vec![]), dedup(vec![]))); + assert!(!one_group(dedup(vec![1]), dedup(vec![1]))); + } + + fn unordered_limit(child: Rc) -> Rc { + node(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child, + }) + } + + /// Two unordered `LIMIT 10`s may return different rows: two groups, and + /// so are equal parents over them. + #[test] + fn unordered_limits_and_their_parents_are_different_groups() { + assert!(!one_group(unordered_limit(scan()), unordered_limit(scan()))); + let distinct_over_limit = || { + node(NonASAPOp::Dedup { + cols: vec![], + child: unordered_limit(scan()), + }) + }; + assert!(!one_group(distinct_over_limit(), distinct_over_limit())); + } + + /// A `LIMIT` over a `Sort` with the same partitions keeps the first rows + /// of a fixed order (one group); over a `Sort` partitioned differently, + /// which rows come first depends on input order (two groups). + #[test] + fn limits_over_a_sort_with_their_partitions_are_one_group() { + let limit = |sort_partitions: GroupKeys| { + node(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: node(NonASAPOp::Sort { + keys: vec![asap_types::ir::scalar::SortKey { + expr: ScalarExpr::Column(2), + ascending: false, + nulls_first: false, + }], + partition_by: sort_partitions, + child: scan(), + }), + }) + }; + assert!(one_group( + limit(GroupKeys::default()), + limit(GroupKeys::default()) + )); + assert!(!one_group( + limit(GroupKeys::by(vec![1])), + limit(GroupKeys::by(vec![1])) + )); + } + + /// A projection emitting the literal `value`. + fn literal(value: f64) -> Rc { + node(NonASAPOp::Project { + cols: vec![asap_types::ir::ProjectItem { + alias: Some("v".into()), + expr: ScalarExpr::literal_f64(value), + }], + qualifier: None, + child: scan(), + }) + } + + /// Signed zeros and different non-finite literals are different groups; + /// JSON's `null` for non-finite floats never decides equality. + #[test] + fn signed_zero_and_non_finite_literals_are_different_groups() { + for (a, b) in [ + (0.0, -0.0), + (-0.0, 0.0), + (f64::INFINITY, f64::NEG_INFINITY), + (f64::NAN, f64::NAN), + ] { + assert!(!one_group(literal(a), literal(b)), "{a} and {b}"); + } + assert!(one_group(literal(f64::INFINITY), literal(f64::INFINITY))); + } + + /// A frontend node that breaks the node rules (a readout of raw rows + /// instead of summary state) makes Stage 0 return the error. + #[test] + fn memo_of_queries_returns_the_error_of_a_node_that_breaks_the_node_rules() { + let readout_of_rows = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: scan(), + query: SketchStatistic::Quantile { q: 0.5 }, + })) + .unwrap(); + assert!(memo_of_roots(&[QueryRoot::Operator(readout_of_rows)]).is_err()); + } + + /// `m{…}` at the evaluation time: the latest sample of each series. + fn instant_selector(source: Rc) -> Rc { + node(NonASAPOp::TimeRange { + range: std::time::Duration::from_secs(60), + kind: asap_types::ir::TimeRangeKind::Instant, + child: source, + }) + } + + /// Each query enters the memo with $n_q$, the values one evaluation + /// returns, as the data workload bounds it: `m`'s 1000 series for the + /// instant vector `m`, one for the scalar `scalar(sum(m))`, unknown for + /// `m` when no series count is declared. + #[test] + fn each_query_enters_with_the_values_one_evaluation_returns() { + let roots = [ + QueryRoot::Operator(instant_selector(scan())), + QueryRoot::Scalar(ScalarExpr::PromqlScalarFromVector(Rc::clone( + scalar_of_vector().children()[0], + ))), + ]; + let declared = DataWorkload { + sources: BTreeMap::from([( + "m".to_owned(), + asap_types::workload::SourceFacts { + series_count: Some(1000), + ..Default::default() + }, + )]), + ..Default::default() + }; + let memo = memo_of_queries(&roots, &[None, None], &declared).unwrap(); + assert_eq!(memo.queries[0].values_per_evaluation, Some(1000)); + assert_eq!(memo.queries[1].values_per_evaluation, Some(1)); + let undeclared = memo_of_roots(&roots).unwrap(); + assert_eq!(undeclared.queries[0].values_per_evaluation, None); + } + + /// A `Scan`'s column signs are the data workload's declared ones: `m` + /// declares `latency` non-negative, the counter `requests_total`'s + /// `value` is non-negative without a declaration, and every other + /// column is `Unknown`. + #[test] + fn a_scan_reads_the_declared_column_signs() { + let counter = node(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "requests_total".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }); + let data = DataWorkload { + sources: BTreeMap::from([ + ( + "m".to_owned(), + asap_types::workload::SourceFacts { + column_signs: BTreeMap::from([( + "latency".to_owned(), + ValueSign::NonNegative, + )]), + ..Default::default() + }, + ), + ( + "requests_total".to_owned(), + asap_types::workload::SourceFacts { + metric_type: Some(PrometheusMetricType::Counter), + ..Default::default() + }, + ), + ]), + ..Default::default() + }; + let roots = [QueryRoot::Operator(scan()), QueryRoot::Operator(counter)]; + let memo = memo_of_queries(&roots, &[None, None], &data).unwrap(); + use ValueSign::{NonNegative, Unknown}; + let signs = |query| { + &memo.groups()[root(&memo, query).0 as usize] + .group_properties + .column_sign + }; + // [ts, service, value, latency] + assert_eq!(signs(0), &[Unknown, Unknown, Unknown, NonNegative]); + // [ts, value] + assert_eq!(signs(1), &[Unknown, NonNegative]); + } } diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs index 3cec2afed..52d9424a5 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs @@ -42,7 +42,8 @@ use std::collections::{HashMap, HashSet}; use std::rc::Rc; -use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::input::OperatorInput; +use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; use asap_types::ir::schema::{ @@ -735,7 +736,7 @@ fn prune_topk_heaps_over_one_row_per_item( } // A PromQL top-k ranks an instant vector: one row per series or // group. (A SQL top-k keeps its heaps.) - if !reads_only_time_series(child) { + if !child.reads_only_time_series() { continue; } prune_own(&mut target.ways_to_compute, |candidate| { @@ -745,14 +746,6 @@ fn prune_topk_heaps_over_one_row_per_item( } } -/// Whether every scan below `node` reads a time series (PromQL rows). -fn reads_only_time_series(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => matches!(source, Source::TimeSeries { .. }), - _ => node.children().iter().all(|c| reads_only_time_series(c)), - } -} - /// The input, update and partitions of a whole-expression top-k over /// `target`'s inner aggregate, by the keyed-additive rule, or `None` when it /// does not apply. Rows that carry the full series identity rank it as a diff --git a/crates/types/src/ir/operator/input.rs b/crates/types/src/ir/operator/input.rs index b5d0ac29d..e34e0f9b5 100644 --- a/crates/types/src/ir/operator/input.rs +++ b/crates/types/src/ir/operator/input.rs @@ -8,6 +8,7 @@ use std::rc::Rc; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::non_asap::NonASAPOp; +use super::operator_properties::Source; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; @@ -21,6 +22,25 @@ pub trait OperatorInput: Sized + Clone { /// What the input covers when it is an aggregate group in the memo; /// `None` for any other input. fn aggregate_coverage(&self) -> Option<&SummaryCoverage>; + /// Whether every `Scan` below it reads a time series (PromQL rows): + /// [`reads_only_time_series`] of its operator, a memo group's stored + /// property. + fn reads_only_time_series(&self) -> bool; +} + +/// Whether every `Scan` at or below `operator` reads a time series: a +/// `Scan` of a `Source::TimeSeries`, or any other operator all of whose +/// inputs do. +pub(crate) fn reads_only_time_series(operator: &Operator) -> bool { + match operator { + Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { + matches!(source, Source::TimeSeries { .. }) + } + operator => operator + .children() + .into_iter() + .all(OperatorInput::reads_only_time_series), + } } /// A front-end node tree. Node trees carry no summary coverage. @@ -40,6 +60,10 @@ impl OperatorInput for Rc { fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { None } + + fn reads_only_time_series(&self) -> bool { + reads_only_time_series(&self.operator) + } } /// An input of a rule: a real input `I`, or a build the rule constructs @@ -115,4 +139,11 @@ impl OperatorInput for InputOrTemporaryBuild { Self::TemporaryBuild(_) => None, } } + + fn reads_only_time_series(&self) -> bool { + match self { + Self::Input(input) => input.reads_only_time_series(), + Self::TemporaryBuild(build) => reads_only_time_series(&build.operator), + } + } } diff --git a/crates/types/src/ir/operator/mod.rs b/crates/types/src/ir/operator/mod.rs index 8b0d5ce2e..33af36096 100644 --- a/crates/types/src/ir/operator/mod.rs +++ b/crates/types/src/ir/operator/mod.rs @@ -28,7 +28,7 @@ pub use agg_intent::{ }; pub use operator_properties::{ AtModifier, BinaryOpKind, ColState, GroupKeys, GroupSide, InfoMatcher, JoinKind, - PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, TimeShift, - VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, WindowFrameOffset, - WindowFrameUnits, WindowFuncKind, + PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, Source, SourceColumn, + TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, + WindowFrameOffset, WindowFrameUnits, WindowFuncKind, }; diff --git a/crates/types/src/ir/operator/operator_properties.rs b/crates/types/src/ir/operator/operator_properties.rs index e2a6ef1da..603f4a4aa 100644 --- a/crates/types/src/ir/operator/operator_properties.rs +++ b/crates/types/src/ir/operator/operator_properties.rs @@ -178,6 +178,16 @@ pub enum Source { Table { table_ref: String }, } +/// A column of a source (a `Scan`'s metric or table), by name: what the data +/// workload declares facts about, such as a column's sign +/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceColumn { + pub source: String, + pub column: String, +} + /// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's /// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison /// operator has exactly one representation (and one `Display`) across the IR. diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 4813d510b..498aa07fa 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -3,15 +3,17 @@ //! `docs/design_docs/proposals/asap-primitive-schema.md` §4.2, after //! Goldstein & Larson's view matching. //! -//! Coverage is derived once for each summary-state memo group, from its -//! input groups, never declared, and stored in its `GroupProperties`. -//! Walking down from a `SummaryAgg` through the input groups' expressions, a +//! Coverage is derived once for each aggregate memo group (a summary state +//! or an exact `Aggregate`), from its input groups, never declared, and +//! stored in its `GroupProperties::aggregate_coverage`. Walking down from a +//! `SummaryAgg` or `Aggregate` through the input groups' expressions, a //! predicate conjunct moves into `selection` when it can be lifted to the -//! `SummaryAgg` (through `Filter`, a range `TimeRange`, a `TimeShift` without -//! `@` and direct-column `Project` items) and it is a value set or an -//! interval on one column. Everything else stays in `definition` as a -//! residual, so two states are merged only when they compute the same thing -//! over disjoint rows. Node trees carry no coverage. +//! top (through `Filter`, a range `TimeRange`, a `TimeShift` and +//! direct-column `Project` items) and it is a value set or an interval on +//! one column. The `TimeRange` and `TimeShift`s become one interval on the +//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in +//! `definition` as a residual, so two states are merged only when they +//! compute the same thing over disjoint rows. Node trees carry no coverage. use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; @@ -23,7 +25,7 @@ use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::operator::operator_properties::Reduction; +use crate::ir::operator::operator_properties::{AtModifier, Reduction}; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::summary_type_capability::{MergeInputRule, SummaryTypeCapability}; @@ -234,12 +236,12 @@ mod limit_bound { #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum CoverageError { - /// The node is not a summary state (`SummaryAgg`, `SummaryTumblingWindowChain` or - /// `SummaryMerge`): a caller bug, or bad input when it is a merge's - /// input. - #[error( - "coverage is derived only for SummaryAgg, SummaryTumblingWindowChain and SummaryMerge" - )] + /// The node has no coverage: it is no aggregate (`SummaryAgg`, + /// `Aggregate`, `SummaryTumblingWindowChain`, `SummaryMerge`), or a + /// `TimeRange`/`TimeShift` over a chain whose windows are not one + /// interval on the timestamp column. A caller bug, or bad input when it + /// is a merge's input. + #[error("coverage is derived only for aggregates and one time interval over a chain")] NotSummary, /// A `SummaryMerge` without inputs: bad input. #[error("summary merge requires at least one input")] @@ -280,10 +282,15 @@ pub enum CoverageError { type GroupInput<'m> = InputOrTemporaryBuild>; impl SummaryCoverage { - /// Coverage of a summary state a memo group computes with `operator` - /// over its input groups: a `SummaryAgg`, a `SummaryTumblingWindowChain`, + /// Coverage of what a memo group computes with `operator` over its + /// input groups: a `SummaryAgg`, an exact `Aggregate`, a + /// `SummaryTumblingWindowChain` (a build over its input, the timestamp + /// column unconstrained), a `TimeRange` or `TimeShift` over a chain (the + /// chain's coverage with the windows' interval on the timestamp column), /// a `SummaryMerge`, or a `Filter` that keeps whole states of a grouped - /// `SummaryAgg`. A merge reads its inputs' stored coverage and fails + /// `SummaryAgg`. So `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` + /// covers what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does. A merge + /// reads its inputs' stored coverage and fails /// unless every input has the same definition and their selections are /// pairwise disjoint; a family whose merge is idempotent /// ([`MergeInputRule::MayOverlap`] in the family-capability table) may @@ -296,14 +303,41 @@ impl SummaryCoverage { op.map_children(|input| InputOrTemporaryBuild::Input(*input)) }; match operator { + Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) => { + let top = + unlifted(&operator.map_children(|input| InputOrTemporaryBuild::Input(*input))); + of_aggregate(&InputOrTemporaryBuild::Input(*child), None, |_| top) + } + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if operator.output_kind() == OperatorResultKind::State => + { + of_window_selection(operator) + } Operator::NonASAP(_) => match selected_build(operator)? { Some(build) => of_summary_agg(&build), None => Err(CoverageError::NotSummary), }, Operator::ASAP(op @ ASAPOp::SummaryAgg { .. }) => of_summary_agg(&as_built(op)), - Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => todo!( - "#1207 PR 2: a chain's coverage is a build's over its input, with the timestamp column unconstrained" - ), + Operator::ASAP(op @ ASAPOp::SummaryTumblingWindowChain { .. }) => { + let ASAPOp::SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + filter, + .. + } = as_built(op) + else { + unreachable!("matched a chain"); + }; + of_summary_agg(&ASAPOp::SummaryAgg { + child, + summary_type, + input, + reduction, + filter, + }) + } Operator::ASAP(ASAPOp::SummaryMerge { children, group_by, .. }) => { @@ -316,7 +350,7 @@ impl SummaryCoverage { .ok_or(CoverageError::NotSummary) }) .collect::, _>>()?; - let merged = of_merge(&inputs)?; + let merged = of_merge(&inputs, time_column_of_build_input(&children[0]).as_ref())?; match group_by { None => Ok(merged), Some(keys) => regrouped(merged, &children[0], keys), @@ -407,8 +441,12 @@ fn selected_build<'m>( })) } -/// The coverage of a merge of states covering `inputs`. -fn of_merge(inputs: &[SummaryCoverage]) -> Result { +/// The coverage of a merge of states covering `inputs`, whose builds read +/// rows with timestamp column `time_column`. +fn of_merge( + inputs: &[SummaryCoverage], + time_column: Option<&ColumnIdentity>, +) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs .iter() @@ -426,12 +464,16 @@ fn of_merge(inputs: &[SummaryCoverage]) -> Result None, } .unwrap_or(MergeInputRule::Disjoint); - let separated: fn(&SelectionBox, &SelectionBox) -> bool = match required { - MergeInputRule::MayOverlap => |_, _| true, - MergeInputRule::DisjointInTime => todo!( - "#1207 PR 2: disjoint intervals on the timestamp column, with IntervalConstraintLimit ends" - ), - MergeInputRule::Disjoint => SelectionBox::disjoint, + let separated = |a: &SelectionBox, b: &SelectionBox| match required { + MergeInputRule::MayOverlap => true, + // Consecutive windows of one series: disjoint on the timestamp column. + MergeInputRule::DisjointInTime => time_column.is_some_and(|column| { + matches!( + (a.columns.get(column), b.columns.get(column)), + (Some(a), Some(b)) if a.disjoint(b) + ) + }), + MergeInputRule::Disjoint => a.disjoint(b), }; // Every pair of boxes of different inputs. if required != MergeInputRule::MayOverlap @@ -542,9 +584,15 @@ fn build_input_schema(state: &I) -> Option { } } +/// The timestamp column of the rows the build of summary state `state` +/// reads ([`build_input_schema`]), as a selection names it. +fn time_column_of_build_input(state: &I) -> Option { + let rows = build_input_schema(state)?; + column_identity(&rows, rows.time_index?) +} + /// The coverage of `build`, a `SummaryAgg` over memo groups (or over an -/// input a rule built over them): its operators walked down through the -/// liftable operators of its input groups' expressions. +/// input a rule built over them). fn of_summary_agg(build: &ASAPOp>) -> Result { let ASAPOp::SummaryAgg { child, @@ -556,6 +604,26 @@ fn of_summary_agg(build: &ASAPOp>) -> Result, + filter: Option<&Predicate>>, + unlifted_top: impl FnOnce(Option>>) -> Operator>, +) -> Result { // The operators a predicate can be lifted through, top to bottom, and // the first input below them. A subquery's group stays named in the // residual of the operator that reads it. @@ -566,7 +634,11 @@ fn of_summary_agg(build: &ASAPOp>) -> Result true, Operator::NonASAP(NonASAPOp::TimeRange { kind, .. }) => *kind == TimeRangeKind::Range, - Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => shift.at.is_none(), + // `@ end()` / `@ start()` pin their subtree to one end of the + // request: not lifted, so their group is the input group. + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + !matches!(shift.at, Some(AtModifier::Start | AtModifier::End)) + } _ => false, }; if !passes { @@ -583,9 +655,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>>> = chain .iter() .enumerate() @@ -619,13 +689,26 @@ fn of_summary_agg(build: &ASAPOp>) -> Result None, }; - let mut unlifted_operators = vec![Operator::ASAP(ASAPOp::SummaryAgg { - child: None, - summary_type: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - filter: agg_filter, - })]; + // The `TimeRange` and `TimeShift`s: one interval on the timestamp column + // of the `TimeRange`'s input, when they make one and it combines with + // what is lifted on that column already; else they stay unlifted. + let time = time_constraint(&chain).and_then(|constraint| { + let depth = chain + .iter() + .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; + let column = chain[depth].children()[0].schema().time_index?; + let column = column_at_top(column, &chain[..=depth])?; + column_identity(top, column)?; + let combined = match lifted.get(&column) { + Some(existing) => existing.intersect(&constraint)?, + None => constraint, + }; + Some((column, combined)) + }); + let lift_time = time.is_some(); + lifted.extend(time); + + let mut unlifted_operators = vec![unlifted_top(agg_filter)]; for (depth, link) in chain.iter().enumerate() { match link { Operator::NonASAP(NonASAPOp::Filter { .. }) => { @@ -636,9 +719,8 @@ fn of_summary_agg(build: &ASAPOp>) -> Result todo!( - "#1207 PR 2: lift the range and the offset into an interval on the timestamp column, with IntervalConstraintLimit ends" - ), + Operator::NonASAP(NonASAPOp::TimeRange { .. } | NonASAPOp::TimeShift { .. }) + if lift_time => {} other => unlifted_operators.push(unlifted(other)), } } @@ -653,14 +735,7 @@ fn of_summary_agg(build: &ASAPOp>) -> Result>) -> Result>, top: &Schema, @@ -688,15 +762,7 @@ fn residual( for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; - let field = &top.fields[column]; - let namesakes = top - .fields - .iter() - .filter(|f| f.name == field.name && f.table == field.table) - .count(); - if namesakes != 1 { - return None; - } + column_identity(top, column)?; let combined = match lifted.get(&column) { Some(existing) => existing.intersect(&constraint)?, None => constraint, @@ -725,8 +791,8 @@ fn group_id(input: &GroupInput<'_>) -> Option { } } -/// `link`, a liftable operator kept whole in a definition: its main input -/// `None`, every other input the group it is. +/// `link`, a liftable operator or an exact `Aggregate` kept whole in a +/// definition: its main input `None`, every other input the group it is. fn unlifted(link: &Operator>) -> Operator> { let mut operator = link.map_children(group_id); match &mut operator { @@ -734,9 +800,10 @@ fn unlifted(link: &Operator>) -> Operator> { NonASAPOp::Filter { child, .. } | NonASAPOp::Project { child, .. } | NonASAPOp::TimeRange { child, .. } - | NonASAPOp::TimeShift { child, .. }, + | NonASAPOp::TimeShift { child, .. } + | NonASAPOp::Aggregate { child, .. }, ) => *child = None, - _ => unreachable!("only liftable operators are walked"), + _ => unreachable!("only liftable operators and aggregates are unlifted"), } operator } @@ -754,6 +821,118 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option Option { + let field = schema.fields.get(column)?; + let namesakes = schema + .fields + .iter() + .filter(|f| f.name == field.name && f.table == field.table) + .count(); + (namesakes == 1).then(|| ColumnIdentity { + table: field.table.clone(), + name: field.name.clone(), + }) +} + +/// The interval on the timestamp column that `links` (top down, the +/// `TimeRange` and `TimeShift`s between an aggregate and its input) select: +/// `(anchor − o − W, anchor − o]` for the one range `TimeRange` of width W +/// and the shifts' offsets summing to o, left-open and right-closed like a +/// PromQL range. The anchor is the evaluation time t +/// (`EvaluationTimePlusMs`), or T for `@ T` (`FixedValue(Timestamp)`). +/// `None` when they are not one interval: no or several range `TimeRange`s, +/// an instant one, several `@`s, or an `@ end()` or `@ start()`, which pins +/// its subtree to one end of the request and is never lifted (the front +/// ends put it above the subtree it anchors). +fn time_constraint(links: &[Operator]) -> Option { + let (mut width, mut offset_ms, mut at) = (None, 0i64, None::); + for link in links { + match link { + Operator::NonASAP(NonASAPOp::TimeRange { range, kind, .. }) => { + if *kind != TimeRangeKind::Range || width.replace(*range).is_some() { + return None; + } + } + Operator::NonASAP(NonASAPOp::TimeShift { shift, .. }) => { + offset_ms = offset_ms.checked_add(shift.offset_ms)?; + if let Some(anchor) = shift.at { + if at.replace(anchor).is_some() { + return None; + } + } + } + _ => {} + } + } + let width_ms = i64::try_from(width?.as_millis()).ok()?; + let fixed_anchor = match at { + Some(AtModifier::Timestamp(ms)) => Some(ms), + Some(AtModifier::Start | AtModifier::End) => return None, + None => None, + }; + let before_anchor = |ms: i64| -> Option { + Some(match fixed_anchor { + Some(anchor) => { + IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(anchor.checked_sub(ms)?)) + } + None => IntervalConstraintLimit::EvaluationTimePlusMs(ms.checked_neg()?), + }) + }; + Some(Constraint::Interval { + lower: Bound::Excluded(before_anchor(offset_ms.checked_add(width_ms)?)?), + upper: Bound::Included(before_anchor(offset_ms)?), + }) +} + +/// The coverage of `selection`, a `TimeRange` or `TimeShift` over a chain +/// (through more of them): the chain's coverage with the interval they +/// select on the timestamp column of the chain's input, in every box. +fn of_window_selection( + selection: &Operator>, +) -> Result { + let mut links = vec![selection.clone()]; + let mut below = *selection.children()[0]; + loop { + match below.operator() { + link @ Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => { + below = child; + links.push(link); + } + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) => break, + _ => return Err(CoverageError::NotSummary), + } + } + let chain = below + .aggregate_coverage() + .ok_or(CoverageError::NotSummary)?; + let time_column = time_column_of_build_input(&below).ok_or(CoverageError::NotSummary)?; + let constraint = time_constraint(&links).ok_or(CoverageError::NotSummary)?; + let selection = chain + .selection + .iter() + .map(|selection_box| { + let mut columns = selection_box.columns.clone(); + let combined = match columns.get(&time_column) { + Some(existing) => existing.intersect(&constraint), + None => Some(constraint.clone()), + }; + columns.insert( + time_column.clone(), + combined.ok_or(CoverageError::NotSummary)?, + ); + Ok(SelectionBox { columns }) + }) + .collect::>()?; + Ok(SummaryCoverage { + definition: chain.definition.clone(), + selection, + }) +} + /// `column = v`, `!=`, `<`, `<=`, `>`, `>=`, `[NOT] IN (...)` and `OR` of /// equalities on one column, with non-null literals. fn constraint_of(expr: &ScalarExpr) -> Option<(ColumnId, Constraint)> { diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index 94e997390..61bc53e69 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -64,7 +64,7 @@ pub fn with_promql_series_identity( // A series has at most one sample per timestamp, so the full // identity and the timestamp key each row. Pass 2's // identical-expression rule reads this key: only a keyed - // node is legal to share (Stage 0 hash-consing, `cse` in `asap_stage0_memo`). + // node is legal to share (hash-consing, `Memo::find_or_add_group`). if let Some(time) = schema.time_index { schema.add_unique_key(vec![schema.fields.len() - 1, time]); } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 6600fd1ca..adc4b41d5 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -1,6 +1,6 @@ //! Ranked-row output contract shared by logical TopK and its implementations. use crate::ir::operator::input::OperatorInput; -use crate::ir::operator::{Reduction, Source}; +use crate::ir::operator::Reduction; use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; @@ -85,7 +85,7 @@ pub(crate) fn ranked_row_projection( alias: Some(output_name.into()), expr: ScalarExpr::Column(value), }); - let promql = reads_only_time_series(child); + let promql = child.reads_only_time_series(); for (index, field) in child.schema().fields.iter().enumerate() { let names_item = !promql || field.dtype == FieldDataType::Plain(DataType::Utf8); if names_item @@ -142,12 +142,3 @@ pub fn series_identity_item( SummaryInputExpr::Tuple(items) } } - -fn reads_only_time_series(node: &I) -> bool { - match node.operator() { - Operator::NonASAP(NonASAPOp::Scan { source, .. }) => { - matches!(source, Source::TimeSeries { .. }) - } - operator => operator.children().into_iter().all(reads_only_time_series), - } -} diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index d47f785f6..4f530d8d8 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -3,10 +3,14 @@ //! $r^{\log}_{c,i}$ (concepts doc `memo-groups-properties-requirements.md`, //! section 3; #1203). -use std::collections::HashSet; +use crate::ir::operator::input::reads_only_time_series; +use std::collections::{BTreeMap, HashMap, HashSet}; -use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId}; +use super::column_sign::output_column_sign; +use super::hash_consing::{expression_hash, same_expression, same_value}; +use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId, ValueSign}; use crate::ir::operator::asap::summary_type_of_operator; +use crate::ir::operator::operator_properties::{AtModifier, SourceColumn, TimeShift}; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::properties::{AccuracyMetric, AccuracyTarget}; use crate::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; @@ -14,6 +18,7 @@ use crate::ir::ASAPOp; use crate::ir::Operator; use crate::ir::QueryRoot; use crate::ir::SchemaDerivationError; +use crate::ir::{NonASAPOp, ScalarExpr}; /// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`]). @@ -33,6 +38,13 @@ pub struct Memo { groups: Vec, /// The queries, by index in the workload. pub queries: Vec, + /// The groups [`Self::find_or_add_group`] may return, by + /// [`expression_hash`] of their expression: every group whose sharing + /// is legal ([`GroupProperties::shareable_when_equal`]). + groups_by_expression_hash: HashMap>, + /// The sign the data workload declares for each source column it + /// declares one for: a `Scan`'s [`GroupProperties::column_sign`]. + declared_column_signs: BTreeMap, } /// One query of the memo: its root over groups, its accuracy target and @@ -50,6 +62,15 @@ pub struct MemoQuery { } impl Memo { + /// A memo without groups whose `Scan`s read `declared_column_signs` + /// (an undeclared source column's sign is `Unknown`). + pub fn new(declared_column_signs: BTreeMap) -> Self { + Self { + declared_column_signs, + ..Self::default() + } + } + /// The groups, by id. pub fn groups(&self) -> &[MemoGroup] { &self.groups @@ -59,34 +80,149 @@ impl Memo { /// group below them list it ([`MemoGroup::reached_by_queries`]). Stage 0 /// adds each query once. pub fn add_query(&mut self, query: MemoQuery) -> usize { - let _ = query; - todo!("#1207 PR 2") + let index = self.queries.len(); + let groups = query.root.operator_refs().into_iter().copied().collect(); + self.queries.push(query); + self.add_queries_below(groups, &[index]); + index } - /// Hash-consing: the group with this expression, added if new, with the - /// `GroupProperties` [`derive_group_properties`] derives. Fails when the - /// expression breaks the node rules (schema derivation, input - /// validation, merge check). + /// Hash-consing: the group whose expression is `expression` + /// (`same_expression`: operator, arguments and input groups), else a + /// new group for it, with the `GroupProperties` [`derive_group_properties`] + /// derives from its input groups, listed as a parent of each input + /// group. A group whose copies may return different rows (an unordered + /// `LIMIT`, [`GroupProperties::shareable_when_equal`]) is never returned + /// for another expression, so such an expression always gets a new + /// group. Fails, leaving the memo unchanged, when the expression breaks + /// the node rules (schema derivation, input validation, merge check). pub fn find_or_add_group( &mut self, expression: Operator, ) -> Result { - let _group_properties = derive_group_properties(self, &expression)?; - todo!("#1207 PR 2: hash-consing over groups") + let group_properties = derive_group_properties(self, &expression)?; + let hash = expression_hash(&expression); + let shareable = group_properties.shareable_when_equal; + if shareable { + if let Some(&group) = self + .groups_by_expression_hash + .get(&hash) + .and_then(|groups| { + groups.iter().find(|&&group| { + same_expression(&self.groups[group.0 as usize].expression, &expression) + }) + }) + { + return Ok(group); + } + } + let id = MemoGroupId( + u32::try_from(self.groups.len()).expect("a memo has at most u32::MAX groups"), + ); + for &input in expression.children() { + add_parent(&mut self.groups[input.0 as usize], id); + } + self.groups.push(MemoGroup { + expression, + group_properties, + parents: Vec::new(), + queries: Vec::new(), + candidates: Vec::new(), + }); + if shareable { + self.groups_by_expression_hash + .entry(hash) + .or_default() + .push(id); + } + Ok(id) } - /// Adds `candidate` to `group`'s candidates. Fails when its expression - /// breaks the node rules ([`derive_group_properties`]) or its output - /// schema or kind differs from the group's. + /// Adds `candidate` to `group`'s candidates and returns its id, unless + /// the group already has a candidate with the same expression + /// (`same_expression`) and logical properties: then that candidate's + /// id, and `candidate` is dropped. A new candidate makes `group` a parent + /// of each of its input groups, and `group`'s queries reach each input + /// group and every group below it. + /// + /// Fails, leaving the memo unchanged, when the candidate's expression + /// breaks the node rules a group's does ([`derive_group_properties`]), + /// or when its output schema or kind differs from the group's: then it + /// does not compute the group's result. pub fn add_candidate( &mut self, group: MemoGroupId, candidate: CandidateForMemoGroup, ) -> Result { - let _output = derive_group_properties(self, &candidate.expression.operator) + let output = derive_group_properties(self, &candidate.expression.operator) .map_err(CandidateError::Schema)?; - let _ = group; - todo!("#1207 PR 2") + let group_properties = &self.groups[group.0 as usize].group_properties; + if output.schema != group_properties.schema + || output.result_kind != group_properties.result_kind + { + return Err(CandidateError::OutputDiffersFromGroup { group }); + } + let candidates = &mut self.groups[group.0 as usize].candidates; + if let Some(index_in_group) = candidates + .iter() + .position(|known| same_expression_and_logical_properties(known, &candidate)) + { + return Ok(CandidateForMemoGroupId { + group, + index_in_group, + }); + } + let input_groups: Vec = candidate + .expression + .operator + .children() + .into_iter() + .copied() + .collect(); + candidates.push(candidate); + let index_in_group = candidates.len() - 1; + for &input in &input_groups { + add_parent(&mut self.groups[input.0 as usize], group); + } + let queries = self.groups[group.0 as usize].queries.clone(); + self.add_queries_below(input_groups, &queries); + Ok(CandidateForMemoGroupId { + group, + index_in_group, + }) + } + + /// Adds `queries` to each of `groups` and every group below them, through + /// the groups' expressions and their candidates' input groups. A group + /// that has every query already is not walked: every group below it has + /// them too. + fn add_queries_below(&mut self, groups: Vec, queries: &[usize]) { + let mut to_visit = groups; + while let Some(group) = to_visit.pop() { + let group = &mut self.groups[group.0 as usize]; + let mut added = false; + for &query in queries { + if let Err(position) = group.queries.binary_search(&query) { + group.queries.insert(position, query); + added = true; + } + } + if added { + to_visit.extend( + group + .expression + .children() + .into_iter() + .chain( + group + .candidates + .iter() + .flat_map(|candidate| candidate.expression.operator.children()), + ) + .copied(), + ); + } + } } /// The candidate `id`. @@ -160,9 +296,6 @@ impl Memo { /// input group's `group_properties` and `expression` /// ([`MemoGroupAsInput`]), never a node tree. Fails when the expression /// breaks them (schema derivation, input validation, merge check). -// `column_sign` and `depends_on_evaluation_time` are derived in #1207 PR 2; -// until then the result is unreachable. -#[allow(unreachable_code)] fn derive_group_properties( memo: &Memo, expression: &Operator, @@ -179,15 +312,55 @@ fn derive_group_properties( Err(_) => None, }; Ok(GroupProperties { + column_sign: output_column_sign(&operator, &schema, &memo.declared_column_signs), + depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), aggregate_coverage, shareable_when_equal: operator.is_shareable_when_equal(), - column_sign: todo!("#1207 PR 2: each output column's sign, per operator, from the input groups' and the sources' declared signs"), - depends_on_evaluation_time: todo!("#1207 PR 2: whether the result changes with the evaluation time, from the operator and the input groups"), + reads_only_time_series: reads_only_time_series(&operator), }) } +/// Whether `operator`'s result changes with the evaluation time: a +/// `TimeRange` reads the window before it, and `time()` or `now()` read it +/// directly; `@ T` pins its subtree to T. Any other operator depends on it +/// when an input group does. +fn depends_on_evaluation_time(operator: &Operator>) -> bool { + fn reads_evaluation_time(expr: &ScalarExpr) -> bool { + matches!( + expr, + ScalarExpr::EvalTimestamp | ScalarExpr::CurrentTimestamp + ) || expr.children().into_iter().any(reads_evaluation_time) + } + match operator { + Operator::NonASAP(NonASAPOp::TimeRange { .. }) => true, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: + TimeShift { + at: Some(AtModifier::Timestamp(_)), + .. + }, + .. + }) => false, + Operator::NonASAP(op) => { + op.scalar_exprs().into_iter().any(reads_evaluation_time) + || operator.children().into_iter().any(depends_on_it) + } + Operator::ASAP( + ASAPOp::SummaryAgg { filter, .. } | ASAPOp::SummaryTumblingWindowChain { filter, .. }, + ) => { + filter.iter().any(|pred| reads_evaluation_time(&pred.0)) + || operator.children().into_iter().any(depends_on_it) + } + Operator::ASAP(_) => operator.children().into_iter().any(depends_on_it), + } +} + +fn depends_on_it(input: &MemoGroupAsInput<'_>) -> bool { + input.group_properties().depends_on_evaluation_time +} + /// A candidate [`Memo::add_candidate`] rejects; the memo is unchanged. #[derive(Debug, thiserror::Error)] pub enum CandidateError { @@ -201,6 +374,27 @@ pub enum CandidateError { OutputDiffersFromGroup { group: MemoGroupId }, } +/// Lists `parent` among `group`'s parents, once. +fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { + if !group.parents.contains(&parent) { + group.parents.push(parent); + } +} + +/// Whether `left` and `right` are one candidate of a group: the same +/// expression ([`same_expression`]) and logical properties (parameters by +/// [`same_value`]). +fn same_expression_and_logical_properties( + left: &CandidateForMemoGroup, + right: &CandidateForMemoGroup, +) -> bool { + same_expression(&left.expression.operator, &right.expression.operator) + && same_value( + &left.logical_properties.parameters, + &right.logical_properties.parameters, + ) +} + /// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. #[derive(Debug, Clone)] pub struct CandidateForMemoGroup { @@ -789,3 +983,310 @@ mod tests { assert_eq!(parameter_list(SketchAlgorithm::UnivMon).len(), 40); } } + +/// The `Memo` methods: hash-consing groups ([`Memo::find_or_add_group`]) and +/// storing candidates ([`Memo::add_candidate`]), on the concepts doc's KLL +/// example (section 1: G1 the latency scan, G2 its KLL state). +#[cfg(test)] +mod memo_tests { + use super::*; + use crate::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; + use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchStatistic, SummaryUpdate, + }; + use crate::ir::NonASAPOp; + + /// `Scan metric[1m]`: `[ts, value]`. + fn scan(metric: &str) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + }) + } + + /// `SummaryAgg[KLL](input)`: no θ in the expression. + fn kll_state(input: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child: input, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) + } + + /// `SummaryEstimate[q](state)`. + fn quantile_readout(state: MemoGroupId, q: f64) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q }, + }) + } + + /// An unordered `LIMIT 10` over `input`, whose copies may return + /// different rows. + fn unordered_limit(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Limit { + n: Some(10), + offset: 0, + partition_by: GroupKeys::default(), + child: input, + }) + } + + /// `Project(input)` keeping its first column. + fn project_first_column(input: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Project { + cols: vec![crate::ir::ProjectItem { + alias: None, + expr: crate::ir::ScalarExpr::Column(0), + }], + qualifier: None, + child: input, + }) + } + + /// A candidate computing `operator` with `parameters`. + fn candidate( + operator: Operator, + parameters: Option, + ) -> CandidateForMemoGroup { + let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; + CandidateForMemoGroup { + // Built directly: the doc example's k = 200 and k = 345 are not + // in today's KLL list, which `LogicalProperties::new` checks. + logical_properties: LogicalProperties { parameters }, + expression: CandidateExpression { operator }, + input_requirements, + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + } + } + + /// A query rooted at `group`, without an accuracy target or $n_q$. + fn query(group: MemoGroupId) -> MemoQuery { + MemoQuery { + root: QueryRoot::Operator(group), + accuracy_target: None, + values_per_evaluation: None, + } + } + + /// KLL `k`. + fn kll_k(k: u32) -> Option { + Some(SketchParams::Kll { k }) + } + + /// An equal expression over the same input groups is found, not added. + #[test] + fn find_or_add_group_returns_the_group_of_an_equal_expression() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + assert_eq!(memo.find_or_add_group(scan("latency")).unwrap(), g1); + assert_eq!(memo.find_or_add_group(kll_state(g1)).unwrap(), g2); + assert_ne!(memo.find_or_add_group(scan("errors")).unwrap(), g1); + assert_eq!(memo.groups().len(), 3); + assert_eq!( + memo.groups()[g2.0 as usize].expression.children(), + vec![&g1] + ); + } + + /// The same operator over different input groups is a different group. + #[test] + fn find_or_add_group_separates_equal_operators_over_different_inputs() { + let mut memo = Memo::default(); + let latency = memo.find_or_add_group(scan("latency")).unwrap(); + let errors = memo.find_or_add_group(scan("errors")).unwrap(); + assert_ne!( + memo.find_or_add_group(kll_state(latency)).unwrap(), + memo.find_or_add_group(kll_state(errors)).unwrap() + ); + } + + /// An expression whose copies may return different rows (an unordered + /// `LIMIT`) gets a new group every time. + #[test] + fn find_or_add_group_never_returns_an_unshareable_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + assert_ne!( + memo.find_or_add_group(unordered_limit(g1)).unwrap(), + memo.find_or_add_group(unordered_limit(g1)).unwrap() + ); + } + + /// A new group is a parent of each of its input groups, listed once. + #[test] + fn find_or_add_group_lists_the_new_group_among_its_inputs_parents() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + memo.find_or_add_group(kll_state(g1)).unwrap(); + assert_eq!(memo.groups()[g1.0 as usize].parent_groups(), &[g2]); + assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); + } + + /// KLL builds with k = 200 and k = 345 are two candidates of one group + /// G2: θ is a logical property, not part of the expression. + #[test] + fn kll_builds_with_k_200_and_k_345_share_one_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let k200_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k200 = memo + .add_candidate(k200_group, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + let k345_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k345 = memo + .add_candidate(k345_group, candidate(kll_state(g1), kll_k(345))) + .unwrap(); + assert_eq!(k200_group, k345_group); + assert_ne!(k200, k345); + assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); + assert_eq!( + memo.candidate(k345).logical_properties.parameters(), + kll_k(345).as_ref() + ); + } + + /// A candidate with the same expression and logical properties as one + /// in the group is not added again; its id is returned. + #[test] + fn add_candidate_returns_the_existing_candidate_for_a_duplicate() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let first = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + let again = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .unwrap(); + assert_eq!(first, again); + assert_eq!( + first, + CandidateForMemoGroupId { + group: g2, + index_in_group: 0 + } + ); + let other_size = candidate(kll_state(g1), kll_k(345)); + assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); + assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); + } + + /// A new candidate makes its group a parent of each group it reads, and + /// the group's queries reach those groups and every group below them. + #[test] + fn add_candidate_maintains_parents_and_queries() { + // Query 0 is G3 = SummaryEstimate[q=0.95](G2) over G2 = KLL(G1). + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + assert_eq!(memo.add_query(query(g3)), 0); + // A rule adds a state group G5 over another scan G4: no query + // reaches them yet. + let g4 = memo.find_or_add_group(scan("latency_copy")).unwrap(); + let g5 = memo.find_or_add_group(kll_state(g4)).unwrap(); + assert!(memo.groups()[g5.0 as usize].reached_by_queries().is_empty()); + // G3 gains a candidate reading G5: G5 and G4 now serve query 0. + memo.add_candidate(g3, candidate(quantile_readout(g5, 0.95), None)) + .unwrap(); + assert_eq!(memo.groups()[g5.0 as usize].parent_groups(), &[g3]); + assert_eq!(memo.groups()[g4.0 as usize].parent_groups(), &[g5]); + assert_eq!(memo.groups()[g5.0 as usize].reached_by_queries(), &[0]); + assert_eq!(memo.groups()[g4.0 as usize].reached_by_queries(), &[0]); + // Query 1 is G6 = Project(G3): it reaches G3 and everything below + // it, through expressions and candidates, in ascending order. + let g6 = memo.find_or_add_group(project_first_column(g3)).unwrap(); + assert_eq!(memo.add_query(query(g6)), 1); + for group in [g1, g2, g3, g4, g5] { + assert_eq!( + memo.groups()[group.0 as usize].reached_by_queries(), + &[0, 1], + "group {group:?}" + ); + } + assert_eq!(memo.groups()[g3.0 as usize].parent_groups(), &[g6]); + } + + /// A candidate that computes another result than its group's (a + /// readout offered for a state group) is rejected and changes nothing. + #[test] + fn add_candidate_rejects_a_candidate_whose_output_differs_from_its_group() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g2, candidate(quantile_readout(g2, 0.5), None)), + Err(CandidateError::OutputDiffersFromGroup { group }) if group == g2 + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); + } + + /// A candidate whose expression fails input validation (a readout of + /// raw rows instead of summary state) is rejected and changes nothing. + #[test] + fn add_candidate_rejects_a_candidate_whose_inputs_fail_validation() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.95)).unwrap(); + let over_raw_rows = quantile_readout(g1, 0.95).map_children(|&group_id| MemoGroupAsInput { + memo: &memo, + group_id, + }); + assert!( + over_raw_rows.output_schema().is_ok(), + "fixture: the schema derives; only input validation rejects it" + ); + let before = format!("{memo:?}"); + assert!(matches!( + memo.add_candidate(g3, candidate(quantile_readout(g1, 0.95), None)), + Err(CandidateError::Schema(_)) + )); + assert_eq!( + format!("{memo:?}"), + before, + "a rejected candidate changes nothing" + ); + } + + /// An expression with no output schema (a group-by column out of range) + /// is an error, and no group is added. + #[test] + fn find_or_add_group_returns_an_error_for_an_expression_with_no_schema() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let out_of_range = Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![9]), + measures: vec![crate::ir::operator::agg_intent::AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child: g1, + }); + assert!(memo.find_or_add_group(out_of_range).is_err()); + assert_eq!(memo.groups().len(), 1); + assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); + } +} diff --git a/crates/types/src/stages/column_sign.rs b/crates/types/src/stages/column_sign.rs new file mode 100644 index 000000000..ffef920aa --- /dev/null +++ b/crates/types/src/stages/column_sign.rs @@ -0,0 +1,449 @@ +//! Value sign: the sign of each output column of a memo group (concepts +//! doc, section 3, "Value sign"), derived once from the input groups' signs +//! and, for a `Scan`, the signs the data workload declares for its source's +//! columns. Error contracts that hold only for values of one sign read it. + +use std::collections::BTreeMap; + +use super::{MemoGroupAsInput, ValueSign}; +use crate::ir::operator::asap::{summary_agg_schema, summary_build}; +use crate::ir::operator::input::OperatorInput; +use crate::ir::operator::operator_properties::{ + BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, +}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; +use crate::ir::schema::sketch::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; +use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::{ASAPOp, NonASAPOp, Operator}; +use crate::workload::bounds::source_name; + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +type Input<'m> = MemoGroupAsInput<'m>; + +/// The sign of each column of `output`, the output schema of `operator`, +/// from its input groups' `column_sign` and, for a `Scan`, `declared`. +pub(crate) fn output_column_sign( + operator: &Operator>, + output: &Schema, + declared: &BTreeMap, +) -> Vec { + let signs = match operator { + Operator::NonASAP(op) => non_asap_signs(op, output, declared), + Operator::ASAP(op) => asap_signs(op, output), + }; + debug_assert_eq!( + signs.len(), + output.fields.len(), + "one sign per output column of {}", + operator.kind_name() + ); + signs +} + +fn signs_of(input: &Input<'_>) -> Vec { + input.group_properties().column_sign.clone() +} + +fn non_asap_signs( + op: &NonASAPOp>, + output: &Schema, + declared: &BTreeMap, +) -> Vec { + use NonASAPOp::*; + let unknown = || vec![Unknown; output.fields.len()]; + match op { + Scan { source, schema, .. } => schema + .fields + .iter() + .map(|field| { + let column = SourceColumn { + source: source_name(source).to_owned(), + column: field.name.clone(), + }; + declared.get(&column).copied().unwrap_or(Unknown) + }) + .collect(), + Values { .. } | PromqlVectorFromScalar(_) => unknown(), + Filter { child, .. } + | Sort { child, .. } + | Limit { child, .. } + | Dedup { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | PromqlSubquery { child, .. } + | PromqlSeriesSample { child, .. } + | PromqlInfoEnrich { child, .. } => signs_of(child), + // The relabeled column is written; every other column is copied. + PromqlRelabel { dst, child, .. } => { + let mut signs = signs_of(child); + match child.schema().column_id(dst) { + Some(written) => signs[written] = Unknown, + None => signs.push(Unknown), + } + signs + } + Project { cols, child, .. } => project_signs(cols, &signs_of(child)), + Aggregate { + reduction, + measures, + child, + .. + } => { + if let [AggIntent::TopK { .. }] = measures.as_slice() { + // The ranked rows, projected: each column copied. + return match crate::ir::schema::topk::ranked_row_projection(child, reduction, "") { + Ok(cols) => project_signs(&cols, &signs_of(child)), + Err(_) => unknown(), + }; + } + aggregate_signs(child, reduction, measures, output) + } + Join { + kind, left, right, .. + } => { + let mut signs = signs_of(left); + if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { + signs.extend(signs_of(right)); + } + signs + } + SetOp { left, right, .. } => agreed(&[signs_of(left), signs_of(right)]), + Concat { children } => agreed(&children.iter().map(signs_of).collect::>()), + SQLWindowFunc { child, .. } => { + let mut signs = signs_of(child); + signs.push(Unknown); + signs + } + // The left side's columns, its value per the operator, then the + // labels the match adds (unknown). + BinaryOp { + operator, lhs, rhs, .. + } => { + let mut signs = signs_of(lhs); + if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { + let rhs_value = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) + .ok() + .map_or(Unknown, |column| signs_of(rhs)[column]); + signs[value] = match &operator.kind { + // A filter keeps the left side's values. + BinaryOpKind::Compare(_) + | BinaryOpKind::Set( + PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, + ) => signs[value], + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), + BinaryOpKind::CompareBool(_) => NonNegative, + BinaryOpKind::Arithmetic(_) => Unknown, + }; + } + signs.resize(output.fields.len(), Unknown); + signs + } + } +} + +fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { + use ASAPOp::*; + match op { + SummaryAgg { + child, + input, + reduction, + .. + } => state_signs(child, input, reduction, output), + // The build's layout without its own time column, after the + // computed `window_end` (column 0). + SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + .. + } => match summary_agg_schema(child.schema(), summary_type, reduction) { + Ok(build) => { + let mut signs = state_signs(child, input, reduction, &build); + if let Some(time) = build.time_index { + signs.remove(time); + } + signs.insert(0, Unknown); + signs + } + Err(_) => vec![Unknown; output.fields.len()], + }, + SummaryMerge { children, group_by } => { + // The windows a `TimeRange` over a chain selects merge into the + // chain's build over that range. + if let (None, [child]) = (group_by, children.as_slice()) { + if let Some(SummaryTumblingWindowChain { + child, + input, + reduction, + .. + }) = summary_build(child) + { + return state_signs(&child, &input, &reduction, output); + } + } + let merged = agreed(&children.iter().map(signs_of).collect::>()); + match group_by { + None => merged, + Some(keys) => { + let state = children[0] + .schema() + .fields + .iter() + .position(|field| !field.is_plain()); + keys.iter() + .chain(state.as_ref()) + .map(|&column| merged.get(column).copied().unwrap_or(Unknown)) + .collect() + } + } + } + SummaryEstimate { + summary_input, + query, + } => readout_signs(summary_input, output, |state| match query { + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy => NonNegative, + SketchStatistic::Quantile { .. } + | SketchStatistic::PointCount { .. } + | SketchStatistic::TopK { .. } => state, + }), + FinalizeExactAccumulator { child } => { + let counts = child.schema().fields.iter().any(|field| { + matches!( + field.dtype, + FieldDataType::ExactAggregate(ExactKind::Count, _) + ) + }); + readout_signs( + child, + output, + |state| if counts { NonNegative } else { state }, + ) + } + MaintainPopulation { child, .. } => signs_of(child), + EvaluatePopulation { child, .. } if output == child.schema() => signs_of(child), + EvaluatePopulation { .. } + | SummarySubtract { .. } + | SummaryDelete { .. } + | SummaryJoin { .. } + | Extension { .. } => vec![Unknown; output.fields.len()], + } +} + +/// A column item keeps its input column's sign; an expression item's sign +/// is [`expression_sign`]'s. +fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec { + cols.iter() + .map(|item| expression_sign(&item.expr, input)) + .collect() +} + +/// `abs` is non-negative, a negation flips its operand's sign, a numeric +/// cast keeps it and a numeric literal has its own; any other expression is +/// `Unknown`. +fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { + match expr { + ScalarExpr::Column(column) => input.get(*column).copied().unwrap_or(Unknown), + ScalarExpr::Literal(ScalarValue::Float64(value)) => sign_of_number(*value), + ScalarExpr::Literal(ScalarValue::Int64(value)) => sign_of_number(*value as f64), + ScalarExpr::Negative { expr, .. } => flipped(expression_sign(expr, input)), + ScalarExpr::Cast { expr, .. } => expression_sign(expr, input), + // PromQL's `abs(v)` lowers to `promql_abs`, SQL's to `abs`. + ScalarExpr::FunctionCall { name, .. } + if name.eq_ignore_ascii_case("abs") || name == "promql_abs" => + { + NonNegative + } + _ => Unknown, + } +} + +/// An aggregate's, in `aggregate_output_schema`'s layout: a per-entity +/// reduction keeps its input's columns in place, its value replaced by the +/// measure's; a `by` reduction lists its keys, then its measures; a +/// `without` reduction the input's other label columns, then its measures. +fn aggregate_signs( + child: &Input<'_>, + reduction: &Reduction, + measures: &[AggIntent], + output: &Schema, +) -> Vec { + let input = signs_of(child); + let schema = child.schema(); + let sample_value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); + let value_of = |measure: &AggIntent| measure.input_cols().first().copied().or(sample_value); + let keys: Vec = match reduction { + Reduction::PerEntity => { + let mut signs = input.clone(); + if let Some(measure) = measures.first() { + if let Some(value) = value_of(measure).filter(|&value| value < signs.len()) { + signs[value] = measure_sign(measure, input[value]); + } + } + return signs; + } + Reduction::Reduce(by) if by.is_without() => (0..schema.fields.len()) + .filter(|&column| { + schema.time_index != Some(column) + && sample_value != Some(column) + && !by.keys().contains(&column) + }) + .map(|column| input[column]) + .collect(), + Reduction::Reduce(by) => by + .keys() + .iter() + .map(|&key| input.get(key).copied().unwrap_or(Unknown)) + .collect(), + }; + let measured: Vec = measures + .iter() + .flat_map(|measure| { + let sign = measure_sign( + measure, + value_of(measure) + .and_then(|value| input.get(value).copied()) + .unwrap_or(Unknown), + ); + // `count_values` and `absent` add label columns before their value. + let labels = match measure { + AggIntent::CountValues { label } => { + usize::from(!output.fields[..keys.len()].iter().any(|f| f.name == *label)) + } + AggIntent::Absent { labels } => labels.len(), + _ => 0, + }; + std::iter::repeat_n(Unknown, labels).chain([sign]) + }) + .collect(); + keys.into_iter().chain(measured).collect() +} + +/// The sign of `measure`'s value over an input value of sign `input`. +fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { + use AggIntent::*; + match measure { + Count + | Cardinality { .. } + | CountValues { .. } + | Changes + | Resets + | Group + | Absent { .. } + | AbsentOverTime + | PresentOverTime + | StdDev { .. } + | Variance { .. } + | FrequencyL2 { .. } + | FrequencyEntropy { .. } + | Math(crate::ir::operator::MathFunc::Abs) => NonNegative, + Sum { .. } + | Avg { .. } + | Min { .. } + | Max { .. } + | Quantile { .. } + | LastOverTime + | FirstOverTime => input, + // A counter's rate and increase are non-negative; over a value of + // another sign a decrease reads as a reset, so no sign is known. + Rate | IRate | Increase if input == NonNegative => NonNegative, + _ => Unknown, + } +} + +/// A summary state's, in `state_schema`'s layout (an aggregate of one +/// measure): its key columns keep their input's sign and its state column +/// records the sign of the values it is updated with. +fn state_signs( + child: &Input<'_>, + update: &SummaryUpdate, + reduction: &Reduction, + state_schema: &Schema, +) -> Vec { + let mut signs = aggregate_signs( + child, + reduction, + &[AggIntent::Sum { col: None }], + state_schema, + ); + let updated_with = match &update.weight { + SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) + .ok() + .and_then(|column| signs_of(child).get(column).copied()) + .unwrap_or(Unknown), + SummaryInputExpr::Constant(value) => sign_of_number(*value), + SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, + }; + if let Some(state) = state_schema.fields.iter().position(|f| !f.is_plain()) { + signs[state] = updated_with; + } + signs +} + +/// A readout of summary state: the state's plain columns keep their sign +/// in place, and the value read out has `value(the state's recorded sign)`. +/// `Unknown` everywhere when the output does not keep the state's layout (a +/// top-k's rows). +fn readout_signs( + state: &Input<'_>, + output: &Schema, + value: impl Fn(ValueSign) -> ValueSign, +) -> Vec { + let input = state.schema(); + if input.fields.len() != output.fields.len() { + return vec![Unknown; output.fields.len()]; + } + signs_of(state) + .into_iter() + .zip(&input.fields) + .map(|(sign, field)| if field.is_plain() { sign } else { value(sign) }) + .collect() +} + +/// Per column, the inputs' sign if they all agree, else `Unknown`. +fn agreed(inputs: &[Vec]) -> Vec { + let Some((first, rest)) = inputs.split_first() else { + return Vec::new(); + }; + first + .iter() + .enumerate() + .map(|(column, &sign)| { + rest.iter().fold(sign, |agreed, other| { + agree(agreed, other.get(column).copied().unwrap_or(Unknown)) + }) + }) + .collect() +} + +fn agree(left: ValueSign, right: ValueSign) -> ValueSign { + if left == right { + left + } else { + Unknown + } +} + +fn flipped(sign: ValueSign) -> ValueSign { + match sign { + NonNegative => NonPositive, + NonPositive => NonNegative, + Unknown => Unknown, + } +} + +fn sign_of_number(value: f64) -> ValueSign { + if value >= 0.0 { + NonNegative + } else if value < 0.0 { + NonPositive + } else { + Unknown + } +} diff --git a/crates/types/src/stages/hash_consing.rs b/crates/types/src/stages/hash_consing.rs new file mode 100644 index 000000000..3a482dda0 --- /dev/null +++ b/crates/types/src/stages/hash_consing.rs @@ -0,0 +1,59 @@ +//! The one equality that makes two expressions one memo group, and two +//! candidates of a group one candidate: [`same_expression`] (operator, +//! arguments and input groups). [`Memo::find_or_add_group`] looks a group up +//! by it, Stage 0 included, and [`Memo::add_candidate`] finds a duplicate by +//! it. Parameters θ are not an argument of any operator (they are a +//! candidate's `LogicalProperties`), so builds of one family over one input +//! that differ only in θ are one group. +//! +//! [`Memo::find_or_add_group`]: super::Memo::find_or_add_group +//! [`Memo::add_candidate`]: super::Memo::add_candidate + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use super::MemoGroupId; +use crate::ir::Operator; + +/// `expression` with every input group replaced by `()`: the operator and +/// its arguments. +fn operator_and_arguments(expression: &Operator) -> Operator<()> { + expression.map_children(|_| ()) +} + +/// Whether `left` and `right` are one expression: equal operators and +/// arguments ([`same_value`]) over the same input groups at the same +/// positions. +pub(super) fn same_expression(left: &Operator, right: &Operator) -> bool { + left.children() == right.children() + && same_value( + &operator_and_arguments(left), + &operator_and_arguments(right), + ) +} + +/// A hash consistent with [`same_expression`], used only to narrow the +/// groups compared: SipHash over the canonical JSON of the operator and its +/// arguments, then the input groups. A non-finite `f64` serializes as +/// `null`, which only widens one bucket; [`same_expression`] decides. +pub(super) fn expression_hash(expression: &Operator) -> u64 { + let mut hasher = DefaultHasher::new(); + serde_json::to_string(&operator_and_arguments(expression)) + .unwrap_or_default() + .hash(&mut hasher); + expression.children().hash(&mut hasher); + hasher.finish() +} + +/// Typed equality, confirmed by equal serializations: numeric `PartialEq` +/// alone conflates signed zeros, and `SketchParams`' equality omits the +/// recorded error and failure probability. The serialized check is +/// additional evidence, never a replacement for typed equality (JSON maps +/// non-finite floats to `null`). +pub(super) fn same_value(left: &T, right: &T) -> bool { + left == right + && match (serde_json::to_string(left), serde_json::to_string(right)) { + (Ok(left), Ok(right)) => left == right, + _ => false, + } +} diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index cce07008d..1995a62a8 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -25,10 +25,14 @@ pub struct MemoGroup { /// changes. pub group_properties: GroupProperties, /// The groups that read it ([`Self::parent_groups`]) through their - /// expression or a candidate's. + /// expression or a candidate's: kept up to date by + /// [`Memo::find_or_add_group`](super::Memo::find_or_add_group) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). pub parents: Vec, /// The queries whose roots reach it ([`Self::reached_by_queries`]), - /// ascending. + /// ascending: kept up to date by + /// [`Memo::add_query`](super::Memo::add_query) and + /// [`Memo::add_candidate`](super::Memo::add_candidate). pub queries: Vec, /// Its candidates: empty after Stage 0, filled by Stage 1. pub candidates: Vec, @@ -41,22 +45,27 @@ pub struct MemoGroup { pub struct GroupProperties { pub schema: Schema, pub result_kind: OperatorResultKind, - /// Per output column, in schema order: the sign of its values. + /// Per output column, in schema order: the sign of its values, from the + /// input groups' and, for a `Scan`, the declared ones. pub column_sign: Vec, /// Whether the result changes with the evaluation time: true for a /// group under a `TimeRange` relative to the evaluation time - /// (`sum_over_time(m[5m])`), false for one under a fixed `@ T` or - /// reading no time. A result that does not depend on it is the same at - /// every evaluation, so a stored copy delivers on every grid. + /// (`sum_over_time(m[5m])`) or reading it (`time()`, `now()`), false + /// for one under a fixed `@ T` or reading no time. A result that does + /// not depend on it is the same at every evaluation, so a stored copy + /// delivers on every grid. pub depends_on_evaluation_time: bool, - /// What an aggregate group covers: a summary state, a chain, a merge - /// over a chain's windows, or an exact `Aggregate` result; `None` for - /// any other group. Derived today for summary states only; exact - /// `Aggregate` groups follow in #1207 PR 2. + /// What an aggregate group covers: a summary state, a chain, a + /// `TimeRange`/`TimeShift` over a chain or a merge over one, or an exact + /// `Aggregate` result; `None` for any other group. pub aggregate_coverage: Option, /// Whether two equal expressions return the same rows for the same /// input rows, so they may be one group (not an unordered `LIMIT`). pub shareable_when_equal: bool, + /// Whether every `Scan` below reads a time series (PromQL rows): a + /// `Scan` of a `Source::TimeSeries`, or any other operator all of whose + /// input groups do. A PromQL top-k ranks one row per series. + pub reads_only_time_series: bool, } /// The sign of one column's values, a group property derived from the @@ -83,6 +92,10 @@ impl<'m> MemoGroupAsInput<'m> { fn group(&self) -> &'m MemoGroup { &self.memo.groups()[self.group_id.0 as usize] } + + pub(crate) fn group_properties(&self) -> &'m GroupProperties { + &self.group().group_properties + } } impl OperatorInput for MemoGroupAsInput<'_> { @@ -104,6 +117,10 @@ impl OperatorInput for MemoGroupAsInput<'_> { fn aggregate_coverage(&self) -> Option<&SummaryCoverage> { self.group().group_properties.aggregate_coverage.as_ref() } + + fn reads_only_time_series(&self) -> bool { + self.group().group_properties.reads_only_time_series + } } impl MemoGroup { diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 728fab01b..66bb0da50 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -8,6 +8,8 @@ //! live here. mod candidates; +mod column_sign; +mod hash_consing; mod logical_memo_candidates; mod memo; mod per_computation_logical_memo_candidates; diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index dcef7d916..11357f3f3 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -264,7 +264,7 @@ mod tests { /// Group 1, KLL states over the scan group 0, groups 2 and 3, chains /// of KLL states over it 5 s and 10 s wide, and group 4, the scan's last /// 5 minutes, each with its own expression as its one candidate. Built - /// through `find_or_add_group` and `add_candidate` (#1207 PR 2). + /// through `find_or_add_group` and `add_candidate`. fn memo() -> Memo { let mut memo = Memo::default(); let scan = memo @@ -413,7 +413,6 @@ mod tests { /// width) and phase. With phases 0 and 5 s every 10 s, only w = 5 s /// serves both. #[test] - #[ignore = "filled in by #1207 PR 2 (#1230)"] fn delivers_on_a_chain_when_its_width_divides_the_context_times() { let candidates = candidates(memo(), vec![None; 5]); let across_5s = candidate(WINDOWS_5S, QueryTime, Lifetime::AcrossEvaluations(MEMORY)); @@ -452,7 +451,6 @@ mod tests { /// evaluated on 60 s @ 0, so it serves both but not a 30 s grid or /// another phase. Without a known grid it serves none. #[test] - #[ignore = "filled in by #1207 PR 2 (#1230)"] fn delivers_on_a_stored_result_on_its_group_grid() { let grid_60s = Some(EvaluationGrid { period_ms: 60_000, diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index bf56559c0..e4f12481b 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,8 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod memo_group_properties; +mod memo_summary_coverage; mod physical_export; mod planner_vocabulary; mod schema_rebuilding; diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs new file mode 100644 index 000000000..9989fe250 --- /dev/null +++ b/crates/types/tests/memo_group_properties.rs @@ -0,0 +1,492 @@ +//! `column_sign` and `depends_on_evaluation_time`, derived once per memo +//! group from its input groups (concepts doc, section 3, "Value sign" and +//! `GroupProperties`). Every example reads one of three sources: +//! +//! ```text +//! latency [ts, value, host] value declared non-negative +//! requests_total [ts, value, host] a counter: value non-negative +//! gauge [ts, value, host] nothing declared +//! ``` +use std::collections::BTreeMap; +use std::time::Duration; + +use asap_types::ir::operator::operator_properties::{ + AtModifier, GroupKeys, JoinKind, Reduction, Source, SourceColumn, TimeShift, +}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::{ColumnRef, ProjectItem}; +use asap_types::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchStatistic, SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, TimeRangeKind, +}; +use asap_types::stages::{Memo, MemoGroupId, ValueSign}; + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +const TS: usize = 0; +const VALUE: usize = 1; +const HOST: usize = 2; +const MINUTE_MS: u64 = 60_000; + +/// The memo of every example: `latency.value` declared non-negative, and +/// `requests_total.value` as Stage 0 declares a counter's. +fn memo() -> Memo { + let non_negative = |source: &str| { + ( + SourceColumn { + source: source.into(), + column: "value".into(), + }, + NonNegative, + ) + }; + Memo::new(BTreeMap::from([ + non_negative("latency"), + non_negative("requests_total"), + ])) +} + +fn add(memo: &mut Memo, expression: Operator) -> MemoGroupId { + memo.find_or_add_group(expression).unwrap() +} + +fn signs(memo: &Memo, group: MemoGroupId) -> &[ValueSign] { + &memo.groups()[group.0 as usize].group_properties.column_sign +} + +fn depends(memo: &Memo, group: MemoGroupId) -> bool { + memo.groups()[group.0 as usize] + .group_properties + .depends_on_evaluation_time +} + +/// `Scan metric`: `[ts, value, host]`. +fn scan(metric: &str) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("host", DataType::Utf8, false), + ], + 0, + vec![], + ), + }) +} + +fn range(child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(MINUTE_MS), + kind: TimeRangeKind::Range, + child, + }) +} + +fn shift(at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 60_000, + at, + }, + child, + }) +} + +fn compare( + left: ScalarExpr, + op: CompareOpKind, + right: ScalarExpr, +) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(left), + op, + right: Box::new(right), + semantics: ExprSemantics::Sql, + } +} + +fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +/// `Project(ts, expr AS value, host)`. +fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator { + let item = |expr, alias: &str| ProjectItem { + alias: Some(alias.into()), + expr, + }; + Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + item(ScalarExpr::Column(TS), "ts"), + item(expr, "value"), + item(ScalarExpr::Column(HOST), "host"), + ], + qualifier: None, + child, + }) +} + +fn aggregate( + reduction: Reduction, + measure: AggIntent, + child: MemoGroupId, +) -> Operator { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + measures: vec![measure], + output_names: vec![], + filters: vec![], + having: None, + child, + }) +} + +fn by_host() -> Reduction { + Reduction::by(vec![HOST]) +} + +fn state(algorithm: SketchAlgorithm, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: FieldDataType::Sketch(algorithm), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +fn readout(query: SketchStatistic, state: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query, + }) +} + +/// Rows of the per-operator table that keep or set a column's sign: +/// +/// | Group | Expression | `column_sign` | +/// |---|---|---| +/// | G1 | `Scan latency` | `[Unknown, NonNegative, Unknown]` (declared) | +/// | G2 | `Scan gauge` | `[Unknown, Unknown, Unknown]` (nothing declared) | +/// | G3 | `Filter(value > 1)(G1)` | G1's | +/// | G4 | `TimeRange[1m](G3)` | G1's | +/// | G5 | `TimeShift(1m)(G4)` | G1's | +/// | G6 | `Project(abs(value))(G2)` | value `NonNegative` | +/// | G7 | `Project(-value)(G1)` | value `NonPositive` (flipped) | +/// | G8 | `Project(-value)(G7)` | value `NonNegative` (flipped back) | +/// | G9 | `Project(value * 2)(G1)` | value `Unknown` (another expression) | +#[test] +fn scans_read_declared_signs_and_row_operators_keep_or_set_them() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let g2 = add(&mut memo, scan("gauge")); + let g3 = add( + &mut memo, + filter( + compare( + ScalarExpr::Column(VALUE), + CompareOpKind::Gt, + ScalarExpr::Literal(ScalarValue::Float64(1.0)), + ), + g1, + ), + ); + let g4 = add(&mut memo, range(g3)); + let g5 = add(&mut memo, shift(None, g4)); + let value = || Box::new(ScalarExpr::Column(VALUE)); + let negated = || ScalarExpr::Negative { + expr: value(), + semantics: ExprSemantics::Promql, + }; + let g6 = add( + &mut memo, + map_value( + ScalarExpr::FunctionCall { + name: "promql_abs".into(), + args: vec![ScalarExpr::Column(VALUE)], + }, + g2, + ), + ); + let g7 = add(&mut memo, map_value(negated(), g1)); + let g8 = add(&mut memo, map_value(negated(), g7)); + let g9 = add( + &mut memo, + map_value( + ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: value(), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }, + g1, + ), + ); + let declared = [Unknown, NonNegative, Unknown]; + assert_eq!(signs(&memo, g1), declared); + assert_eq!(signs(&memo, g2), [Unknown; 3]); + for group in [g3, g4, g5] { + assert_eq!(signs(&memo, group), declared, "{group:?}"); + } + for (group, value) in [ + (g6, NonNegative), + (g7, NonPositive), + (g8, NonNegative), + (g9, Unknown), + ] { + assert_eq!(signs(&memo, group), [Unknown, value, Unknown], "{group:?}"); + } +} + +/// Aggregates: keys keep their sign, sum / max keep the value's, counts are +/// non-negative, and a rate is non-negative over a non-negative value only: +/// +/// | Group | Expression | `column_sign` | +/// |---|---|---| +/// | G2 | `sum by (host)(G1)`, G1 = `Scan latency` | `[Unknown, NonNegative]` | +/// | G3 | `max by (host)(G1)` | `[Unknown, NonNegative]` | +/// | G5 | `sum by (host)(G4)`, G4 = `Scan gauge` | `[Unknown, Unknown]` | +/// | G6 | `count by (host)(G4)` | `[Unknown, NonNegative]` | +/// | G7 | `count(distinct value) by (host)(G4)` | `[Unknown, NonNegative]` | +/// | G9 | `rate(G8)`, G8 = `requests_total[1m]` | value `NonNegative` | +/// | G11 | `rate(G10)`, G10 = `gauge[1m]` | value `Unknown` | +#[test] +fn aggregates_keep_their_keys_and_derive_their_values() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let sum = || AggIntent::Sum { col: Some(VALUE) }; + let g2 = add(&mut memo, aggregate(by_host(), sum(), g1)); + let g3 = add( + &mut memo, + aggregate(by_host(), AggIntent::Max { col: Some(VALUE) }, g1), + ); + let g4 = add(&mut memo, scan("gauge")); + let g5 = add(&mut memo, aggregate(by_host(), sum(), g4)); + let g6 = add(&mut memo, aggregate(by_host(), AggIntent::Count, g4)); + let g7 = add( + &mut memo, + aggregate(by_host(), AggIntent::Cardinality { cols: vec![VALUE] }, g4), + ); + let requests = add(&mut memo, scan("requests_total")); + let g8 = add(&mut memo, range(requests)); + let g9 = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, g8), + ); + let g10 = add(&mut memo, range(g4)); + let g11 = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, g10), + ); + for (group, value) in [ + (g2, NonNegative), + (g3, NonNegative), + (g5, Unknown), + (g6, NonNegative), + (g7, NonNegative), + ] { + assert_eq!(signs(&memo, group), [Unknown, value], "{group:?}"); + } + assert_eq!(signs(&memo, g9), [Unknown, NonNegative, Unknown]); + assert_eq!(signs(&memo, g11), [Unknown; 3]); +} + +/// A state records the sign of the values it is updated with, a merge +/// keeps the sign its inputs agree on, and a readout reads the state's, a +/// count-like statistic being non-negative whatever the values: +/// +/// | Group | Expression | `column_sign` | +/// |---|---|---| +/// | G3 | `SummaryAgg[KLL](G2)`, G2 = `latency[1m]` | state `NonNegative` | +/// | G6 | `SummaryAgg[KLL](G5)`, G5 = `(latency offset 1m)[1m]` | state `NonNegative` | +/// | G7 | `SummaryMerge(G3, G6)` | state `NonNegative` (agreed) | +/// | G8 | `SummaryEstimate[p99](G7)` | quantile `NonNegative` | +/// | G11 | `SummaryAgg[HLL](G10)`, G10 = `gauge[1m]` | state `Unknown` | +/// | G12 | `SummaryEstimate[cardinality](G11)` | cardinality `NonNegative` | +#[test] +fn states_record_their_input_sign_and_readouts_read_it() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let g2 = add(&mut memo, range(g1)); + let g3 = add(&mut memo, state(SketchAlgorithm::Kll, g2)); + let g4 = add(&mut memo, shift(None, g1)); + let g5 = add(&mut memo, range(g4)); + let g6 = add(&mut memo, state(SketchAlgorithm::Kll, g5)); + let g7 = add( + &mut memo, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![g3, g6], + group_by: None, + }), + ); + let g8 = add( + &mut memo, + readout(SketchStatistic::Quantile { q: 0.99 }, g7), + ); + let g9 = add(&mut memo, scan("gauge")); + let g10 = add(&mut memo, range(g9)); + let g11 = add(&mut memo, state(SketchAlgorithm::Hll, g10)); + let g12 = add(&mut memo, readout(SketchStatistic::Cardinality, g11)); + // [ts, state, host] + for group in [g3, g6, g7] { + assert_eq!( + signs(&memo, group), + [Unknown, NonNegative, Unknown], + "{group:?}" + ); + } + assert_eq!(signs(&memo, g8), [Unknown, NonNegative, Unknown]); + assert_eq!(signs(&memo, g11), [Unknown; 3]); + assert_eq!(signs(&memo, g12), [Unknown, NonNegative, Unknown]); +} + +/// The concepts doc's example: `topk(5, sum by (host)(gauge))` ranks values +/// of `Unknown` sign (Stage 2 then prunes the Count-Min heap builds), while +/// `topk(5, sum by (host)(rate(requests_total[1m])))` ranks `NonNegative` +/// ones. +#[test] +fn a_top_k_ranks_values_of_its_inputs_sign() { + let mut memo = memo(); + let top_5 = |memo: &mut Memo, child| { + add( + memo, + aggregate( + Reduction::Reduce(GroupKeys::by(vec![])), + AggIntent::TopK { k: 5 }, + child, + ), + ) + }; + let sum = AggIntent::Sum { col: Some(VALUE) }; + let gauge = add(&mut memo, scan("gauge")); + let gauge_sum = add(&mut memo, aggregate(by_host(), sum.clone(), gauge)); + let gauge_top = top_5(&mut memo, gauge_sum); + let requests = add(&mut memo, scan("requests_total")); + let ranged = add(&mut memo, range(requests)); + let rate = add( + &mut memo, + aggregate(Reduction::PerEntity, AggIntent::Rate, ranged), + ); + let rate_sum = add(&mut memo, aggregate(by_host(), sum, rate)); + let rate_top = top_5(&mut memo, rate_sum); + // [host, sum]: the ranked value is column 1 of the ranked rows. + assert_eq!(signs(&memo, gauge_sum), [Unknown, Unknown]); + assert_eq!(signs(&memo, rate_sum), [Unknown, NonNegative]); + let ranked = |group| { + let schema = &memo.groups()[group as usize].group_properties.schema; + let value = schema.column_id("topk_5").unwrap(); + signs(&memo, MemoGroupId(group))[value] + }; + assert_eq!(ranked(gauge_top.0), Unknown); + assert_eq!(ranked(rate_top.0), NonNegative); +} + +/// `depends_on_evaluation_time`: a `TimeRange` reads the window before the +/// evaluation time, `now()` reads it directly, and everything above them +/// depends on it, unless `@ T` pins the subtree; `@ end()` does not pin it +/// to a fixed time. +/// +/// | Group | Expression | Depends | +/// |---|---|---| +/// | G1 | `Scan latency` | no | +/// | G2 | `sum by (host)(G1)` | no | +/// | G3 | `latency[1m]` | yes | +/// | G4 | `SummaryAgg[KLL](G3)` | yes | +/// | G5 | `TimeShift(@ T, offset 1m)(G3)` | no | +/// | G6 | `SummaryAgg[KLL](G5)` | no | +/// | G7 | `TimeShift(@ end(), offset 1m)(G3)` | yes | +/// | G8 | `Filter(ts > now())(G1)` | yes | +#[test] +fn a_result_depends_on_the_evaluation_time_under_a_time_range_or_now() { + let mut memo = memo(); + let g1 = add(&mut memo, scan("latency")); + let g2 = add( + &mut memo, + aggregate(by_host(), AggIntent::Sum { col: Some(VALUE) }, g1), + ); + let g3 = add(&mut memo, range(g1)); + let g4 = add(&mut memo, state(SketchAlgorithm::Kll, g3)); + let g5 = add( + &mut memo, + shift(Some(AtModifier::Timestamp(1_700_000_000_000)), g3), + ); + let g6 = add(&mut memo, state(SketchAlgorithm::Kll, g5)); + let g7 = add(&mut memo, shift(Some(AtModifier::End), g3)); + let g8 = add( + &mut memo, + filter( + compare( + ScalarExpr::Column(TS), + CompareOpKind::Gt, + ScalarExpr::CurrentTimestamp, + ), + g1, + ), + ); + for (group, expected) in [ + (g1, false), + (g2, false), + (g3, true), + (g4, true), + (g5, false), + (g6, false), + (g7, true), + (g8, true), + ] { + assert_eq!(depends(&memo, group), expected, "{group:?}"); + } +} + +/// `reads_only_time_series`, derived once per group: a PromQL `Scan` of a +/// time series reads one, and so does a `TimeRange` over it; a SQL table +/// `Scan` does not, nor a `Join` reading it (a SQL join reads relations +/// only, so a time series cannot be one of its inputs). +#[test] +fn a_group_reads_only_time_series_when_every_scan_below_reads_one() { + let mut memo = memo(); + let series = add(&mut memo, scan("latency")); + let window = add(&mut memo, range(series)); + let table = |name: &str| { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: name.into(), + }, + predicates: vec![], + schema: Schema::new(vec![ + Field::plain("host", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }) + }; + let requests = add(&mut memo, table("requests")); + let hosts = add(&mut memo, table("hosts")); + let join = add( + &mut memo, + Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), + left: requests, + right: hosts, + }), + ); + let reads = |group: MemoGroupId| { + memo.groups()[group.0 as usize] + .group_properties + .reads_only_time_series + }; + assert!(reads(series)); + assert!(reads(window)); + assert!(!reads(requests)); + assert!(!reads(join)); +} diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_summary_coverage.rs new file mode 100644 index 000000000..adade1a62 --- /dev/null +++ b/crates/types/tests/memo_summary_coverage.rs @@ -0,0 +1,683 @@ +//! Group properties and aggregate coverage, derived once per memo group +//! from its input groups (`docs/design_docs/proposals/asap-primitive-schema.md` +//! §4.2.2). The memo of the worked example: +//! +//! ```text +//! G1 Scan latency +//! G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) +//! G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) +//! G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) +//! ``` +use std::ops::Bound; +use std::time::Duration; + +use asap_types::ir::operator::operator_properties::{ + AtModifier, GroupKeys, Reduction, Source, TimeShift, +}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::summary_coverage::{ + ColumnIdentity, Constraint, CoverageError, IntervalConstraintLimit, SummaryCoverage, +}; +use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{ + DataType, ExactKind, Field, FieldDataType, MergeInputRule, Schema, SketchAlgorithm, + SketchStatistic, SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, + SchemaDerivationError, TimeRangeKind, +}; +use asap_types::stages::{ + CandidateExpression, CandidateForMemoGroup, LogicalProperties, LogicalRequirements, Memo, + MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::test_support::definition_id; + +const TS: usize = 0; +const JOB: usize = 2; +const A: usize = 3; +const B: usize = 4; +const MINUTE_MS: i64 = 60_000; +const HOUR_MS: i64 = 60 * MINUTE_MS; + +/// `Scan latency{predicates}`: `[ts, value, job, a, b]`. +fn scan(predicates: Vec>) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: predicates.into_iter().map(Predicate).collect(), + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, false), + Field::plain("a", DataType::Float64, false), + Field::plain("b", DataType::Float64, false), + ], + 0, + vec![], + ), + }) +} + +fn job_is(job: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(JOB)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(job.into()))), + semantics: ExprSemantics::Sql, + } +} + +/// `a + b > 10`: it spans two columns, so it is no selection. +fn a_plus_b_above_10() -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Add, + left: Box::new(ScalarExpr::Column(A)), + right: Box::new(ScalarExpr::Column(B)), + semantics: ExprSemantics::Sql, + }), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(10.0))), + semantics: ExprSemantics::Sql, + } +} + +fn filter(pred: ScalarExpr, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(pred), + child, + }) +} + +/// `TimeShift(offset_ms, at)`: PromQL `offset` and `@`. +fn shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { offset_ms, at }, + child, + }) +} + +fn range(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(width_ms as u64), + kind: TimeRangeKind::Range, + child, + }) +} + +/// `SummaryAgg[summary_type]` of the sample value per series. +fn state(summary_type: FieldDataType, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +fn kll_summary_type() -> FieldDataType { + FieldDataType::Sketch(SketchAlgorithm::Kll) +} + +/// `SummaryAgg[KLL]` of the sample value per series. +fn kll(child: MemoGroupId) -> Operator { + state(kll_summary_type(), child) +} + +/// `SummaryTumblingWindowChain[KLL, width_ms]` of the sample value per series. +fn kll_chain(width_ms: i64, child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + summary_type: kll_summary_type(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: width_ms as u64, + }) +} + +fn merge(children: Vec) -> Operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + }) +} + +/// KLL over the last minute of `G1` filtered by `pred`. +fn kll_of_last_minute_where( + memo: &mut Memo, + g1: MemoGroupId, + pred: ScalarExpr, +) -> MemoGroupId { + let filtered = memo.find_or_add_group(filter(pred, g1)).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, filtered)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() +} + +fn coverage(memo: &Memo, group: MemoGroupId) -> &SummaryCoverage { + memo.groups()[group.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state") +} + +/// `KLL` as an unlifted operator: its main input is `None`. +fn kll_unlifted() -> Operator> { + kll(MemoGroupId(0)).map_children(|_| None) +} + +fn jobs(values: &[&str]) -> Constraint { + Constraint::In( + values + .iter() + .map(|value| ScalarValue::Utf8((*value).into())) + .collect(), + ) +} + +/// `(t + lower_ms, t + upper_ms]` on the timestamp column, t the evaluation +/// time: left-open, right-closed like a PromQL range. +fn relative(lower_ms: i64, upper_ms: i64) -> Constraint { + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::EvaluationTimePlusMs(lower_ms)), + upper: Bound::Included(IntervalConstraintLimit::EvaluationTimePlusMs(upper_ms)), + } +} + +fn last_minute() -> Constraint { + relative(-MINUTE_MS, 0) +} + +fn ts() -> ColumnIdentity { + ColumnIdentity::new(None, "ts") +} + +/// The constraint on the timestamp column of `group`'s one selection box. +fn time_of(memo: &Memo, group: MemoGroupId) -> Constraint { + let [selection] = coverage(memo, group).selection() else { + panic!("one box: {:?}", coverage(memo, group)) + }; + selection.columns()[&ts()].clone() +} + +/// A group's properties come from its operator and its input groups' +/// properties: the scan's schema, the state's kind and state column, the +/// readout's value column, and whether equal copies may be one group. +#[test] +fn group_properties_come_from_input_groups() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let readout = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Quantile { q: 0.99 }, + })) + .unwrap(); + let limit = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Limit { + n: Some(1), + offset: 0, + partition_by: GroupKeys::default(), + child: g1, + })) + .unwrap(); + let properties = |group: MemoGroupId| &memo.groups()[group.0 as usize].group_properties; + assert_eq!(properties(g1).schema.fields.len(), 5); + assert_eq!(properties(g1).aggregate_coverage, None); + assert_eq!(properties(g4).result_kind, OperatorResultKind::State); + assert!(properties(g4) + .schema + .fields + .iter() + .any(|field| field.name == "state" && !field.is_plain())); + assert!(properties(g4).aggregate_coverage.is_some()); + assert_eq!( + properties(readout).result_kind, + OperatorResultKind::InstantVector + ); + assert!(properties(readout) + .schema + .fields + .iter() + .any(|field| field.name == "quantile" + && field.dtype == FieldDataType::Plain(DataType::Float64))); + assert!(properties(g4).shareable_when_equal); + assert!(!properties(limit).shareable_when_equal); +} + +/// G4: `job = "api"` and the minute are lifted; the definition is the KLL +/// over G1. +#[test] +fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![kll_unlifted()] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.columns()[&ts()], last_minute()); +} + +/// G7 has G4's definition and id and a disjoint selection, so +/// `SummaryMerge(G4, G7)` is a valid group covering job ∈ {api, web}. +#[test] +fn states_with_equal_definitions_and_disjoint_selections_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g7 = kll_of_last_minute_where(&mut memo, g1, job_is("web")); + assert_eq!( + coverage(&memo, g4).definition(), + coverage(&memo, g7).definition() + ); + assert_eq!( + definition_id(coverage(&memo, g4)), + definition_id(coverage(&memo, g7)) + ); + let merged = memo.find_or_add_group(merge(vec![g4, g7])).unwrap(); + let coverage = coverage(&memo, merged); + assert_eq!(coverage.definition().input_group, Some(g1)); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// Two states that select the same rows do not merge: KLL counts a row +/// once per input. +#[test] +fn states_with_overlapping_selections_do_not_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let groups = memo.groups().len(); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g4])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::Disjoint + } + )) + )); + assert_eq!(memo.groups().len(), groups); +} + +/// G10: `a + b > 10` spans two columns, so it stays as a `Filter` in the +/// definition; G10 does not merge with G4 although both select job = api. +#[test] +fn a_filter_that_is_no_selection_stays_unlifted() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g4 = kll_of_last_minute_where(&mut memo, g1, job_is("api")); + let g10 = kll_of_last_minute_where( + &mut memo, + g1, + ScalarExpr::BoolAnd(vec![job_is("api"), a_plus_b_above_10()]), + ); + let coverage_of_g10 = coverage(&memo, g10); + assert_eq!(coverage_of_g10.definition().input_group, Some(g1)); + assert_eq!( + coverage_of_g10.definition().unlifted_operators, + vec![ + kll_unlifted(), + filter(a_plus_b_above_10(), MemoGroupId(0)).map_children(|_| None), + ] + ); + assert_eq!( + coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert!(matches!( + memo.find_or_add_group(merge(vec![g4, g10])), + Err(SchemaDerivationError::Coverage( + CoverageError::DefinitionMismatch + )) + )); +} + +/// `a > (SELECT threshold)`: a scalar subquery reading group `threshold`. +fn a_above(threshold: MemoGroupId) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(A)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::ScalarSubquery(threshold)), + semantics: ExprSemantics::Sql, + } +} + +/// A residual `Filter` that reads a scalar subquery names the subquery's +/// group (`Some`) next to its main input (`None`). Two states with that +/// equal definition and disjoint selections merge. +#[test] +fn states_whose_definitions_read_the_same_subquery_merge() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let threshold = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Values { + rows: vec![vec![ScalarExpr::Literal(ScalarValue::Float64(10.0))]], + schema: Schema::new(vec![Field::plain("threshold", DataType::Float64, false)]), + })) + .unwrap(); + let state_of_job = |memo: &mut Memo, job: &str| { + let pred = ScalarExpr::BoolAnd(vec![job_is(job), a_above(threshold)]); + kll_of_last_minute_where(memo, g1, pred) + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input_group, Some(g1)); + assert_eq!( + definition.unlifted_operators, + vec![ + kll_unlifted(), + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(a_above(threshold).map_operator_refs(&mut |group| Some(*group))), + child: None, + }), + ] + ); + assert_eq!(definition, coverage(&memo, web).definition()); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// PromQL label matchers are the scan's predicates: they are lifted too, so +/// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, +/// without them, is the last unlifted operator. +#[test] +fn scan_predicates_lift_and_their_states_merge() { + let mut memo = Memo::default(); + let state_of_job = |memo: &mut Memo, job: &str| { + let series = memo.find_or_add_group(scan(vec![job_is(job)])).unwrap(); + let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); + memo.find_or_add_group(kll(last_minute)).unwrap() + }; + let api = state_of_job(&mut memo, "api"); + let web = state_of_job(&mut memo, "web"); + let definition = coverage(&memo, api).definition(); + assert_eq!(definition.input_group, None); + assert_eq!( + definition.unlifted_operators, + vec![kll_unlifted(), scan(vec![]).map_children(|_| None)] + ); + assert_eq!( + coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); + assert_eq!( + coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api", "web"]) + ); +} + +/// A chain has the definition of a build over its input and leaves the +/// timestamp column unconstrained. `TimeRange` and `TimeShift` over it +/// select windows as over raw rows, so the merge over that selection covers +/// what the direct build does, and `add_candidate` accepts the merge in the +/// direct build's group: +/// +/// ```text +/// G1 Scan latency +/// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) +/// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) +/// G4 candidate: SummaryMerge(G7) +/// ``` +#[test] +fn a_merge_over_a_chains_windows_covers_the_direct_build() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(shift(HOUR_MS, None, g1)).unwrap(); + let g3 = memo.find_or_add_group(range(5 * MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(kll(g3)).unwrap(); + let g5 = memo.find_or_add_group(kll_chain(MINUTE_MS, g1)).unwrap(); + let g6 = memo.find_or_add_group(shift(HOUR_MS, None, g5)).unwrap(); + let g7 = memo.find_or_add_group(range(5 * MINUTE_MS, g6)).unwrap(); + + let whole_chain = coverage(&memo, g5); + assert_eq!(whole_chain.definition(), coverage(&memo, g4).definition()); + assert!(whole_chain.selection()[0].columns().is_empty()); + let five_minutes_an_hour_ago = relative(-HOUR_MS - 5 * MINUTE_MS, -HOUR_MS); + assert_eq!(time_of(&memo, g7), five_minutes_an_hour_ago); + assert_eq!(coverage(&memo, g7), coverage(&memo, g4)); + + let expression = CandidateExpression { + operator: merge(vec![g7]), + }; + let merge_of_windows = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + memo.add_candidate(g4, merge_of_windows).unwrap(); + let merged = memo.find_or_add_group(merge(vec![g7])).unwrap(); + assert_eq!(coverage(&memo, merged), coverage(&memo, g4)); +} + +/// The range and every offset lift into one interval relative to the +/// evaluation time t; `@ T` fixes the interval in absolute time: +/// +/// | Group | Expression | Timestamp column | +/// |---|---|---| +/// | G2 | `TimeRange[5m](G1)` (`m[5m]`) | `(t − 5m, t]` | +/// | G4 | `TimeRange[5m](TimeShift(1h)(G1))` (`m[5m] offset 1h`) | `(t − 65m, t − 60m]` | +/// | G6 | `TimeRange[5m](TimeShift(@ 1700000000)(G1))` | `(1699999700000, 1700000000000]` | +/// +/// each under a `SummaryAgg[KLL]`. +#[test] +fn time_range_and_shifts_lift_into_one_interval_on_the_timestamp_column() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let plain = kll_over_last_5m(&mut memo, g1); + let offset = kll_over_last_5m_of(&mut memo, shift(HOUR_MS, None, g1)); + let at_t = kll_over_last_5m_of( + &mut memo, + shift(0, Some(AtModifier::Timestamp(1_700_000_000_000)), g1), + ); + assert_eq!(time_of(&memo, plain), relative(-5 * MINUTE_MS, 0)); + assert_eq!(time_of(&memo, offset), relative(-3_900_000, -3_600_000)); + let timestamp = |ms| IntervalConstraintLimit::FixedValue(ScalarValue::Timestamp(ms)); + assert_eq!( + time_of(&memo, at_t), + Constraint::Interval { + lower: Bound::Excluded(timestamp(1_699_999_700_000)), + upper: Bound::Included(timestamp(1_700_000_000_000)), + } + ); + for state in [offset, at_t] { + assert_eq!( + coverage(&memo, state).definition(), + coverage(&memo, plain).definition() + ); + } +} + +/// `SummaryAgg[KLL](TimeRange[5m](G1))`: `m[5m]`. +fn kll_over_last_5m(memo: &mut Memo, input: MemoGroupId) -> MemoGroupId { + let ranged = memo.find_or_add_group(range(5 * MINUTE_MS, input)).unwrap(); + memo.find_or_add_group(kll(ranged)).unwrap() +} + +/// [`kll_over_last_5m`] over the group of `expression`. +fn kll_over_last_5m_of(memo: &mut Memo, expression: Operator) -> MemoGroupId { + let input = memo.find_or_add_group(expression).unwrap(); + kll_over_last_5m(memo, input) +} + +/// `@ end()` and `@ start()` are lifted above the subtree they anchor: +/// the front end lowers `m[5m] @ end()` as `TimeShift{@ end()}(G2)` over +/// G2 = `m[5m]`, the group an instant query's `m[5m]` reads too. The +/// `TimeShift` pins G2 to one end of the request, so it is not lifted: it +/// is the input group of the state over it, whose selection is empty. +/// +/// ```text +/// G1 Scan latency; G2 TimeRange[5m](G1); G3 SummaryAgg[KLL](G2) (instant query) +/// G4 TimeShift(@ end())(G2); G5 SummaryAgg[KLL](G4) (range query) +/// ``` +#[test] +fn at_end_and_at_start_share_the_subtree_they_anchor_and_stay_unlifted() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + assert_eq!(time_of(&memo, g3), relative(-5 * MINUTE_MS, 0)); + for at in [AtModifier::End, AtModifier::Start] { + let pinned = memo.find_or_add_group(shift(0, Some(at), g2)).unwrap(); + assert_eq!( + memo.groups()[pinned.0 as usize].expression.children(), + [&g2] + ); + let state = memo.find_or_add_group(kll(pinned)).unwrap(); + let coverage = coverage(&memo, state); + assert_eq!(coverage.definition().input_group, Some(pinned)); + assert_eq!(coverage.definition().unlifted_operators, [kll_unlifted()]); + assert!(coverage.selection()[0].columns().is_empty()); + } +} + +/// `ts >= ` (a SQL timestamp filter) and `value > 10` lift as +/// intervals between fixed values: `G3 SummaryAgg[KLL](G2)`, `G2 +/// Filter(ts >= 2026-11-01 ∧ value > 10)(G1)`. +#[test] +fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { + const NOV_1_2026_MS: i64 = 1_793_491_200_000; + let compare = |column, op, value| ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Sql, + }; + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo + .find_or_add_group(filter( + ScalarExpr::BoolAnd(vec![ + compare(TS, CompareOpKind::Ge, ScalarValue::Timestamp(NOV_1_2026_MS)), + compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), + ]), + g1, + )) + .unwrap(); + let g3 = memo.find_or_add_group(kll(g2)).unwrap(); + let fixed = |value| Bound::Included(IntervalConstraintLimit::FixedValue(value)); + assert_eq!( + time_of(&memo, g3), + Constraint::Interval { + lower: fixed(ScalarValue::Timestamp(NOV_1_2026_MS)), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + Constraint::Interval { + lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( + 10.0 + ))), + upper: Bound::Unbounded, + } + ); + assert_eq!( + coverage(&memo, g3).definition().unlifted_operators, + vec![kll_unlifted()] + ); +} + +/// Counter states (`Increase`) merge only over windows disjoint in time: +/// the last 5 minutes and the 5 before them merge into the last 10; two +/// copies of the last 5 minutes do not. A KLL over the last minute and one +/// over the minute before it are disjoint too. +/// +/// ```text +/// G2 TimeRange[5m](G1); G3 SummaryAgg[Increase](G2) +/// G4 TimeShift(5m)(G1); G5 TimeRange[5m](G4); G6 SummaryAgg[Increase](G5) +/// G7 SummaryMerge(G3, G6) +/// ``` +#[test] +fn counter_states_merge_over_windows_disjoint_in_time() { + let increase = || FieldDataType::exact(ExactKind::Increase); + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(range(5 * MINUTE_MS, g1)).unwrap(); + let g3 = memo.find_or_add_group(state(increase(), g2)).unwrap(); + let g4 = memo + .find_or_add_group(shift(5 * MINUTE_MS, None, g1)) + .unwrap(); + let g5 = memo.find_or_add_group(range(5 * MINUTE_MS, g4)).unwrap(); + let g6 = memo.find_or_add_group(state(increase(), g5)).unwrap(); + let g7 = memo.find_or_add_group(merge(vec![g3, g6])).unwrap(); + assert_eq!(time_of(&memo, g7), relative(-10 * MINUTE_MS, 0)); + assert!(matches!( + memo.find_or_add_group(merge(vec![g3, g3])), + Err(SchemaDerivationError::Coverage( + CoverageError::InputsOverlap { + required: MergeInputRule::DisjointInTime + } + )) + )); + let g8 = memo.find_or_add_group(kll(g2)).unwrap(); + let g9 = memo.find_or_add_group(kll(g5)).unwrap(); + let g10 = memo.find_or_add_group(merge(vec![g8, g9])).unwrap(); + assert_eq!(time_of(&memo, g10), relative(-10 * MINUTE_MS, 0)); +} + +/// An exact `Aggregate` gets coverage like a summary state: `G4 +/// Aggregate[sum(value) by job](G3)` over the last minute of `job = "api"` +/// has the `Aggregate` as its definition and the job and minute as its +/// selection. +#[test] +fn an_exact_aggregate_has_coverage() { + let sum_by_job = |child| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![JOB]), + measures: vec![AggIntent::Sum { col: Some(1) }], + output_names: vec![], + filters: vec![], + having: None, + child, + }) + }; + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan(vec![])).unwrap(); + let g2 = memo.find_or_add_group(filter(job_is("api"), g1)).unwrap(); + let g3 = memo.find_or_add_group(range(MINUTE_MS, g2)).unwrap(); + let g4 = memo.find_or_add_group(sum_by_job(g3)).unwrap(); + let coverage = coverage(&memo, g4); + assert_eq!(coverage.definition().input_group, Some(g1)); + assert_eq!( + coverage.definition().unlifted_operators, + vec![sum_by_job(MemoGroupId(0)).map_children(|_| None)] + ); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + assert_eq!( + selection.columns()[&ColumnIdentity::new(None, "job")], + jobs(&["api"]) + ); + assert_eq!(selection.columns()[&ts()], last_minute()); +} diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 4a0b267e0..6326e7a3a 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -114,10 +114,10 @@ A node in the physical data will represent the data or summary instance, so a no | Why not put it in the schema? | States worth merging cover different data but must have the same schema | below | | What is it based on? | Goldstein & Larson view matching (SIGMOD 2001), explained with an example | §4.1 | | What does it store? | `definition` (what is computed) + `selection` (which rows were taken) | §4.2.1 | -| How is it computed? | by the planner, from the sub-DAG the node covers, in one walk that yields both the definition and the selection | §4.2.2 | -| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; cached per node | §4.2.3 | +| How is it computed? | by the planner, from the memo group's input groups, in one walk that yields both the definition and the selection | §4.2.2 | +| How expensive is it? | proportional to the few operators directly under the `SummaryAgg`, not to the whole sub-DAG; derived once per memo group | §4.2.3 | | What uses it? | merge (in #646), rollup; slice, reuse (planned); subtract (reserved) | §5.6, §5.7 | -| Where is it in the code? | `OperatorNode::coverage()`, `SummaryCoverage::derive` | §6.1, §6.6 | +| Where is it in the code? | `GroupProperties::aggregate_coverage`, `SummaryCoverage::derive_from_inputs` | §6.6 | | What is left out? | the deployment and runtime implementation, e.g. SDS | §4.3 | **Why coverage is not part of the schema.** @@ -237,70 +237,119 @@ Formally, a state means `family(input(σ(C)))` for each group of `G`, where `C` #### 4.2.2 Deriving the definition and the selection -The planner computes the coverage of a node from the sub-DAG the node covers, not from a declaration. One walk down the sub-DAG produces both parts: every filter condition either moves into `selection` or stays in `definition`. The definition is described first, then how the planner decides which conditions move into the selection. +Coverage is derived for memo groups, never declared and never computed on operator-node trees. When `Memo::find_or_add_group` creates an aggregate group (a summary state or an exact `Aggregate`), `SummaryCoverage::derive_from_inputs` walks down from the group's `SummaryAgg` or `Aggregate` through its input groups' `expression`s in the memo and stores the result once in the group's `GroupProperties::aggregate_coverage`. One walk produces both parts: every filter condition either moves into the `selection` or stays in the `definition`. A `SummaryMerge` group reads its input groups' stored coverage; it is a valid group only when their definitions are equal and their selections are disjoint (disjoint in time, for counter states). -**Worked example.** p99 latency per job over table `t` (columns `job`, `region`, `latency`): +```rust +/// What a summary state computes, with every lifted selection removed. +pub struct SummaryStateDefinition { + /// The group the unlifted operators read: the first input under the + /// SummaryAgg or Aggregate that is not Filter / range TimeRange / + /// TimeShift / direct-column Project. `None` when that input is a `Scan` with a + /// liftable predicate (a PromQL label matcher): it is lifted too, and the + /// `Scan` with the rest of its predicates is the last unlifted operator. + pub input_group: Option, + /// The operators not lifted into the selection, listed top down from the + /// SummaryAgg, each with only its residual arguments. `None` marks an + /// operator's main input (the next unlifted operator, or input_group); + /// `Some(g)` is another input group it reads, such as the group a scalar + /// subquery in a Filter reads. + pub unlifted_operators: Vec>>, +} -```sql -SELECT job, - approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) -FROM (SELECT job, region AS r, latency - FROM t - WHERE region = 'us' AND latency * 2 > 10) -GROUP BY job; +pub struct SummaryCoverage { + pub(crate) definition: SummaryStateDefinition, + /// Union of boxes over the output rows of the unlifted operators' input. + pub(crate) selection: Vec, +} + +impl SummaryCoverage { + pub(crate) fn derive_from_inputs( + operator: &Operator>, + ) -> Result; + /// A hash of the definition: equal definitions of one memo get equal ids. + pub(crate) fn definition_id(&self) -> String; +} ``` -The SQL frontend lowers it as follows: `FILTER (WHERE …)` becomes the aggregate's filter, `region AS r` becomes a `Project`, and the `WHERE` conditions are folded into the `Scan`'s predicates. With an error target, the planner answers the quantile from a KLL `SummaryAgg`, read out by a `SummaryEstimate`. The sub-DAG under that `SummaryEstimate`: +**Worked example.** Three queries over the series `latency` (labels `job`, `a`, `b`), each a KLL state over the last minute: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job, filter: latency < 100 - ▲ - │ - [ Project ] job, region AS r, latency - ▲ - │ - [ Scan t ] predicates: region = 'us' AND latency * 2 > 10 +G1 Scan latency +G2 Filter(job = "api")(G1); G3 TimeRange[1m](G2); G4 SummaryAgg[KLL](G3) ← Q1 +G5 Filter(job = "web")(G1); G6 TimeRange[1m](G5); G7 SummaryAgg[KLL](G6) ← Q2 +G8 Filter(job = "api" ∧ a + b > 10)(G1); G9 TimeRange[1m](G8); G10 SummaryAgg[KLL](G9) ← Q3 ``` -##### The definition +**Case 1, G4: everything is lifted.** The walk starts at G4's `SummaryAgg` and reads each input group's `expression`: + +| Step | Group | Expression | What the walk does | +|---|---|---|---| +| 1 | G4 | `SummaryAgg[KLL](G3)` | the first unlifted operator, with no filter of its own | +| 2 | G3 | `TimeRange[1m](G2)` | the one range window: lifted as `ts ∈ (t − 60 s, t]` (t the evaluation time) | +| 3 | G2 | `Filter(job = "api")(G1)` | `job = "api"` is a value set on one column: lifted, nothing of the `Filter` remains | +| 4 | G1 | `Scan latency` | not liftable, no predicate to lift: the walk stops; G1 is the `input_group` | + +```text +G4.aggregate_coverage = SummaryCoverage { + definition: SummaryStateDefinition { + input_group: Some(G1), + unlifted_operators: [SummaryAgg[KLL]], + }, + selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], +} +``` + +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. + +**Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: +```text +G10.aggregate_coverage = SummaryCoverage { + definition: SummaryStateDefinition { + input_group: Some(G1), + unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], + }, + selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], +} +``` -The `definition` is the `SummaryAgg` together with its sub-DAG, with every condition that moves into `selection` (below) taken out. Everything else stays exactly as it is: +Its definition differs from G4's, so G10 cannot merge with G4, even though both select `job = api`. + +**The fields.** + +- `input_group`: the group the definition reads, below everything the walk passed (`None` when the walk lifted a predicate out of a `Scan`). +- `unlifted_operators`: what the state computes over `input_group`, top down from the `SummaryAgg`, each operator with only the conditions that stayed. Each operator's main input is `None`; any other group it reads, such as the group of a scalar subquery in a `Filter`'s condition, is `Some(group)`. +- `selection`: which output rows of that computation the state took, as a union of boxes. +- `definition_id()`: a hash of the definition, the key a summary store files states under together with a concrete selection. + +##### The definition -| In the sub-DAG | In the `definition` | +The `definition` is what remains of the operators the walk passed, with every condition that moves into `selection` (below) taken out: + +| On the walk | In `unlifted_operators` | |---|---| | a `Filter` whose conditions all move into `selection` | removed | -| a `Filter` with some conditions that stay | kept, with only the conditions that stay | -| `Scan.predicates` and `SummaryAgg.filter` | trimmed the same way | -| the one range `TimeRange`, and any `TimeShift`s, when they become relative time | removed | -| a `TimeShift` with no range `TimeRange`, or two or more range `TimeRange`s | unchanged: no time is taken out | -| any other operator | unchanged | - -So the `definition` holds what the state computes: the computation `C` with its remaining conditions, the summary family and its parameters, the input column, and the grouping `G`. +| a `Filter` with some conditions that stay | kept, with only the conditions that stay; a scalar subquery among them names its group | +| `SummaryAgg.filter` | trimmed the same way | +| the one range `TimeRange`, and every `TimeShift` (with or without `@`), when they become one interval on the timestamp column | removed | +| a `TimeShift` with no range `TimeRange`, two or more range `TimeRange`s, two `@`s, or an interval that does not combine with a timestamp condition already lifted | unchanged: no time is taken out | +| a direct-column `Project` | unchanged | +| a `Scan` with a predicate that moves into `selection` (a PromQL label matcher) | the last operator, with only the predicates that stay; `input_group` is `None` | +| any other operator (the first group the walk cannot pass) | not in the list: it is the `input_group` | -In the worked example, `latency < 100` and `region = 'us'` move into `selection` (below), and `latency * 2 > 10` stays: +An exact `Aggregate` is the first unlifted operator, kept whole (its per-measure filters and `HAVING` read its input or output, not the rows it reads). A `SummaryTumblingWindowChain[w](X)` has the definition of a build over X, and its selection leaves the timestamp column unconstrained: a chain has no range. A `TimeRange` or `TimeShift` over a chain (a group whose output is the chain's state with a time index) has the chain's coverage with the interval they select on the timestamp column of X, computed as over raw rows. A `SummaryMerge` over that group reads it, so `SummaryMerge(TimeRange(W, TimeShift(o, Chain[w](X))))` covers exactly what `SummaryAgg(TimeRange(W, TimeShift(o, X)))` does, and `Memo::add_candidate` accepts the merge as a candidate of the direct build's group: ```text - ( SummaryEstimate p99 ) - ▲ - │ - [[ SummaryAgg ]] KLL(latency) by job ← filter removed - ▲ - │ - [ Project ] job, region AS r, latency ← unchanged - ▲ - │ - [ Scan t ] predicates: latency * 2 > 10 ← region = 'us' removed +G1 Scan latency +G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) ts ∈ (t − 65m, t − 60m] +G5 Chain[KLL, 1m](G1) ts unconstrained +G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) ts ∈ (t − 65m, t − 60m] +G4 candidate: SummaryMerge(G7) the same coverage as G4 ``` -**When two definitions are equal.** Merging (§5.6) requires equal definitions. +A rollup (`SummaryMerge` with `group_by`) has its input's definition with the `SummaryAgg` grouped by the rollup's keys. -- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. -- `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. -- States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. +**When two definitions are equal.** Merging (§5.6) requires equal definitions: the same `input_group` and the same `unlifted_operators`, compared as values. A definition with a scalar subquery compares like any other: two states whose `Filter(a > (SELECT threshold))` read the same group are equal there. Timing is not part of a group, so a tumbling window built at ingestion time can merge with one built at query time. States over different tables never merge: a KLL over `m1` and one over `m2` read different groups. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. ##### The selection @@ -309,7 +358,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **Steps.** -1. **Collect the conditions.** Go down the sub-DAG from the `SummaryAgg` and collect every filter condition: from `Filter` nodes, from `Scan.predicates`, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. +1. **Collect the conditions.** Go down the input groups from the `SummaryAgg` and collect every filter condition: from `Filter` groups, from the `Scan.predicates` where the walk stops, and from the `SummaryAgg`'s own `filter`. A condition `A AND B` counts as two conditions, `A` and `B`. 2. **Ask two questions about each condition:** - **Rule 1: would it pick the same rows if it were moved to just below the `SummaryAgg`?** `region = 'us'` below a `Project` that only renames columns: yes. `value > 5` below `rate`: no, because it filters the raw samples that `rate` reads, which changes the rate values. - **Rule 2: is it a simple condition on one column?** That is, a value set such as `region IN ('us', 'eu')`, or a range such as `latency < 100`. `latency * 2 > 10` is not: it is on an expression. @@ -318,7 +367,16 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` - If either answer is no: leave it in the sub-DAG, so it is part of `definition`. The paper calls such conditions *residuals*. - One more case stays in `definition`: a condition on a column that already has a lifted condition it cannot be combined with (a forbidden-value set and a range, or values of different types). Conditions are taken in walk order: the `SummaryAgg.filter`, then the `Filter`s from top to bottom, then `Scan.predicates`. -In the worked example: +A SQL example, p99 latency per job over table `t` (columns `job`, `region`, `latency`), whose `WHERE` conditions the SQL front end folds into the `Scan`'s predicates and whose `region AS r` is a `Project`: + +```sql +SELECT job, + approx_percentile_cont(latency, 0.99) FILTER (WHERE latency < 100) +FROM (SELECT job, region AS r, latency + FROM t + WHERE region = 'us' AND latency * 2 > 10) +GROUP BY job; +``` | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| @@ -333,11 +391,11 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl | Operator it must pass | Can it pass? | Why | Example | |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | -| `TimeRange` (range) or `TimeShift` (without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | +| `TimeRange` (range) or `TimeShift` (with or without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | | `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | -| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit`, a `TimeShift` with `@` | no | | | +| any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: @@ -377,18 +435,21 @@ Any other shape stays in `definition`: - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. -**Time.** Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`. Its KLL state (one per series) has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = time `(−3m, −2m]`, the minute that ended 2 minutes before evaluation. There are two kinds of time: +**Time.** The timestamp column is a column like any other: a selection constrains it with an interval whose ends are `IntervalConstraintLimit`s, `EvaluationTimePlusMs(ms)` (relative to the evaluation time t) or `FixedValue(Timestamp(ms since the epoch))` (absolute). The walk turns the one range `TimeRange(W)` and the `TimeShift`s on its path, whose offsets sum to o, into `ts ∈ (anchor − o − W, anchor − o]` on the timestamp column of the `TimeRange`'s input. Like a PromQL range, which `m[r]` evaluated at t reads as (t − r, t], the interval is left-open and right-closed. The anchor is t, or T for `@ T`. `@ end()` and `@ start()` pin a subtree to one end of the request: the PromQL front end lowers the subtree X without them and puts `TimeShift { at: End | Start }(X)` above it, so X is the group of the same expression without `@`. That `TimeShift` is not lifted: the walk stops at it, and it is the definition's `input_group`. Example: `quantile_over_time(0.99, m[1m] offset 2m)` lowers to `TimeRange(1m)` over `TimeShift(2m)` over `Scan m`; its KLL state has `definition` = the `SummaryAgg` over `Scan m`, and `selection` = `ts ∈ (t − 3m, t − 2m]`. -| Kind | Comes from | Example | +| Query fragment | `lower` | `upper` | |---|---|---| -| **Relative** to the evaluation time | a range `TimeRange(w)` over a `TimeShift(s)` → `(−(s+w), −s]`; a SQL trailing window `ts >= now() - INTERVAL 'w'` lowers to `TimeRange(w)` | `TimeRange(1m)` alone → `(−1m, 0]`; over `TimeShift(1m)` → `(−2m, −1m]` | -| **Absolute** (later) | an interval on the timestamp column | `ts >= t0 AND ts < t1` → `ts ∈ [t0, t1)` | +| PromQL `m[5m]` | `Excluded(EvaluationTimePlusMs(-300_000))` | `Included(EvaluationTimePlusMs(0))` | +| PromQL `m[5m] offset 1h` | `Excluded(EvaluationTimePlusMs(-3_900_000))` | `Included(EvaluationTimePlusMs(-3_600_000))` | +| PromQL `m[5m] @ end()` or `@ start()` | the subtree `m[5m]` is the group of `m[5m]`; the `TimeShift { at: End \| Start }` above it is not lifted (it is `input_group`) | | +| PromQL `m[5m] @ 1700000000` | `Excluded(FixedValue(Timestamp(1_699_999_700_000)))` | `Included(FixedValue(Timestamp(1_700_000_000_000)))` | +| SQL `WHERE ts >= now() - INTERVAL '5 minutes'` | lowers to `TimeRange(5m)`: as `m[5m]` | | +| SQL `WHERE ts >= '2026-11-01'` | `Included(FixedValue(Timestamp(1_793_491_200_000)))` | `Unbounded` | -- PromQL windows exclude their start, so relative windows are open on the left. A SQL trailing window takes the same timing, `>=` included. -- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) builds its tumbling windows this way: a 3-minute window as tumbling windows `(−1m, 0]`, `(−2m, −1m]`, `(−3m, −2m]`, so tumbling window times are derived, not declared (#601). +- Window composition ([Pass 2](planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) of logical optimization) reads a longer range from a chain: `TimeRange(3m)` over a 1-minute chain selects the windows `(t − 1m, t]`, `(t − 2m, t − 1m]`, `(t − 3m, t − 2m]`, and the merge over them covers `(t − 3m, t]`. +- Counter states (`Rate`, `Increase`) merge only when their intervals on the timestamp column are disjoint (`MergeInputRule::DisjointInTime`); other families need any column disjoint. - An instant `TimeRange` (latest sample per series) does not pick rows by time, so it stays in `definition`. The walk also stops there, so for now the label matchers of an instant selector stay in `definition` too; lifting them is planned together with `rate` (Rule 1). -- The IR cannot yet write a timestamp constant, so absolute SQL time filters stay in `definition` for now. -- Absolute and relative time are never compared: a state over `(−1m, 0]` and one over `ts ∈ [t0, t1)` are treated as possibly overlapping. +- Limits of different variants never compare: a state over `ts ∈ (t − 1m, t]` and one over `ts ∈ (t0, t1]` are treated as possibly overlapping, and a relative interval and an absolute timestamp condition on one path do not combine (time then stays in `definition`). **More examples** of what goes where. Each query's quantile is answered from a KLL `SummaryAgg`: @@ -396,7 +457,7 @@ Any other shape stays in `definition`: |---|---|---|---| | SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | | SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | the last 3 to 2 minutes before evaluation, `(−3m, −2m]` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ (t − 3m, t − 2m]` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | | PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | @@ -405,7 +466,7 @@ Any other shape stays in `definition`: #### 4.2.3 Cost of deriving coverage -**When it runs.** `coverage()` derives a node's coverage the first time it is called and caches it on the node, so each node pays once. Building a `SummaryMerge` (`OperatorNode::new`) derives its coverage to reject an invalid merge and caches the result, so `coverage()` and `validate_structure` do not derive it again. `validate_structure` derives it only for a merge that was not built through `new`. +**When it runs.** `find_or_add_group` derives an aggregate group's coverage once, when it creates the group, and stores it in `GroupProperties::aggregate_coverage`; `add_candidate` derives a candidate's the same way. A `SummaryMerge` reads its inputs' stored coverage, so each state is derived once. Operator-node trees carry no coverage. **Sizes used below.** @@ -688,7 +749,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ State: series labels, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: time (−2m, 0] + ▲ selection: ts ∈ (t − 2m, t] │ ┌─────────────────┴─────────────────┐ │ State: series labels, │ State: series labels, @@ -696,7 +757,7 @@ Window composition (Pass 2) can answer it from two one-minute tumbling windows, │ │ [[ SummaryAgg ]] tumbling window 0 [[ SummaryAgg ]] tumbling window 1 KLL k=200, per series KLL k=200, per series - selection: time (−1m, 0] selection: time (−2m, −1m] + selection: ts ∈ (t − 1m, t] selection: ts ∈ (t − 2m, t − 1m] ▲ ▲ │ │ [ TimeRange 1m ] [ TimeRange 1m ] @@ -798,26 +859,26 @@ input groups output groups | State | Query | Selection | |---|---|---| -| `A` | `quantile_over_time(0.99, m[1m])` | time `(−1m, 0]` | -| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | time `(−2m, −1m]` | -| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | time `(−90s, −30s]` | -| `D` | `quantile_over_time(0.99, n[1m])` | time `(−1m, 0]`, but over metric `n`: definition `KLL(value) per series over Scan n` | +| `A` | `quantile_over_time(0.99, m[1m])` | ts ∈ `(t − 1m, t]` | +| `B` | `quantile_over_time(0.99, m[1m] offset 1m)` | ts ∈ `(t − 2m, t − 1m]` | +| `C` | `quantile_over_time(0.99, m[1m] offset 30s)` | ts ∈ `(t − 90s, t − 30s]` | +| `D` | `quantile_over_time(0.99, n[1m])` | ts ∈ `(t − 1m, t]`, but over metric `n`: definition `KLL(value) per series over Scan n` | | Merge | Allowed? | Why | Result's selection | |---|---|---|---| -| `A + B` | ✓ | same definition, no overlap | `(−2m, 0]` (touching time windows join) | -| `A + C` | ✗ | `(−60s, −30s]` is in both, so those rows would be counted twice | | +| `A + B` | ✓ | same definition, no overlap | `(t − 2m, t]` (touching time windows join) | +| `A + C` | ✗ | `(t − 60s, t − 30s]` is in both, so those rows would be counted twice | | | `A + A` | ✗ | every row is in both | | | `A + D` | ✗ | different definitions: `D` reads metric `n`, not `m` | | | `A` + a KLL with `k = 400` | ✗ | different schema: `k` is part of the state type, so the merge is rejected before coverage is checked | | -| `(A + B)` + the state of `m[1m] offset 2m`, time `(−3m, −2m]` | ✓ | a merge has coverage like any state, so merges nest | `(−3m, 0]` | +| `(A + B)` + the state of `m[1m] offset 2m`, ts ∈ `(t − 3m, t − 2m]` | ✓ | a merge has coverage like any state, so merges nest | `(t − 3m, t]` | **Whether inputs may overlap** depends on the summary type (family and algorithm, §3). This is #592 (open), which adds `FieldDataType::family_merges` and `merge_relation`; #646 alone requires every merge to be overlap-free: | Rule | Summary types | Example | |---|---|---| -| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(−60s, −30s]` would be counted twice | -| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(−1m, 0]` and `(−90s, −30s]`, which together cover `(−90s, 0]` | +| **must not overlap** | KLL, DDSketch, Count-Min, Count Sketch, UnivMon, exact `Sum`/`Count` | KLL `A + C` ✗: the rows in `(t − 60s, t − 30s]` would be counted twice | +| **may overlap** | HLL, exact `Min`/`Max` | HLL over `A`'s and `C`'s rows ✓: a value seen twice is still one distinct value. The result keeps both boxes, `(t − 1m, t]` and `(t − 90s, t − 30s]`, which together cover `(t − 90s, t]` | | **cannot merge (yet)** | `CmsWithHeap`, `CountSketchWithHeap`, Theta, KMV, exact `Rate`/`IRate`/`Increase`, samples, wavelets, models | no sound merge is modeled yet, so #592 rejects them | | **right inside left** | subtraction | 5.7 | @@ -825,7 +886,7 @@ input groups output groups |---|---| | What comes out? | the children's schema; with `group_by`, its columns in order, then the state column. A `group_by` of no columns over a SQL relation's states stands for a global SQL aggregate, which has one row over no input rows: it outputs one state, empty when there is no input state | | When is it rejected? | no children; a child is not state; the schemas differ (e.g. different sketch parameters); the definitions differ (different column, filters or source); the selections may overlap (#646; with #592, only where the family does not allow it); the family cannot merge at all (#592); a `group_by` column is not a group key of the children's build, appears twice, or is the state or time column; `group_by` over a family that cannot merge different groups (counter states, `IRate`, Theta, Hydra) | -| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(−2m, −1m]` and `(−1m, 0]` give `(−2m, 0]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | +| What is its coverage? | the shared `definition` (with `group_by`, the same build grouped by the columns `group_by` names), and the union of the children's selections. Two boxes join when they differ in only one dimension and their union is again one constraint, as in the paper's one range per column: touching time windows (`(t − 2m, t − 1m]` and `(t − 1m, t]` give `(t − 2m, t]`), touching value ranges (`latency < 100` and `latency >= 100` give `latency ∈ (−∞, ∞)`, which still excludes NULL), or value sets of one column (`{us}` and `{eu}` give `{us, eu}`). Gaps stay as separate boxes | | State or value? | state in, state out | **Guarantee after a rollup.** The groups of one build partition its rows, so the states a rollup merges into one output group are disjoint, and the family-capability table's merged error applies. A rolled-up state of these families keeps the guarantee one build of the same size would have over the coarse group's rows; most are exactly that build's state: @@ -847,7 +908,7 @@ Rejected: counter states (`Rate`, `Increase`), which merge consecutive windows o **Other uses of coverage.** The planner also uses coverage to read or reuse a state without merging: - **Slice:** from a `by[region, job]` state, a query for `region = 'us'` by job reads only the `us` groups ✓. A query for `value < 50` ✗: `value` is not a grouping column, and a sketch cannot be filtered after it is built. -- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(−5m, 0]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. +- **Reuse:** a stored state answers a query when the definitions are equal and the query's rows are all in the state, with any difference covered by a slice. A stored `by[region, job]` state over `(t − 5m, t]` answers p99 by job over the last 5 minutes for `region = 'us'` ✓, but not over the last 1 minute ✗: time is not a grouping column, so minutes 2–5 cannot be taken out. ### 5.7 Reserved operators (not implemented) @@ -888,9 +949,10 @@ OperatorNode §6.1 │ └── fields[i].dtype: FieldDataType ◀───────┘ §6.3 (the same type: the output │ └── Sketch(SketchAlgorithm) `state` field has type `summary_type`) │ (θ, `SketchParams`, is on the candidate) §6.4 -├── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) -└── coverage() → SummaryCoverage §6.6 - └── definition: OperatorNode (again a node, so the same structure) +└── guarantee: AccuracyGuarantee §6.5 (accuracy of an evaluation) + +MemoGroup.group_properties.aggregate_coverage: SummaryCoverage §6.6 + └── definition: SummaryStateDefinition { input_group, unlifted_operators } ``` | Subsection | Defines | Used by | @@ -900,7 +962,7 @@ OperatorNode §6.1 | §6.3 Schema and field types | `Schema`, `Field`, `FieldDataType` | every node's `schema`; `SummaryAgg.summary_type` | | §6.4 Summary algorithms and parameters | `SketchAlgorithm`, `SketchParams`, `HydraParams`, exact/sample/wavelet/model kinds and params | the payload of each non-plain `FieldDataType` variant; θ on a candidate's `LogicalProperties` | | §6.5 Update input and evaluations | `SummaryUpdate`, `SketchStatistic`, `MaintainedPopulation`, `PopulationStatistic` | fields of the `ASAPOp` variants in §6.2 | -| §6.6 Summary coverage | `SummaryCoverage`, `SelectionBox`, `Constraint` | `OperatorNode::coverage()` on summary nodes | +| §6.6 Summary coverage | `SummaryCoverage`, `SummaryStateDefinition`, `SelectionBox`, `Constraint` | `GroupProperties::aggregate_coverage` of summary-state memo groups | **Example: the p99 latency state of §5.1.** The `SummaryAgg` node (§6.1) holds `ASAPOp::SummaryAgg` (§6.2) with `summary_type = Sketch(Kll)` (§6.3, §6.4) and `input = SummaryUpdate::column(latency)` (§6.5); its candidate's θ is `Kll { k: 200 }`. Its `schema` (§6.3) is `(job Utf8, state Sketch(Kll))`: the `state` field has the same type as `summary_type`. A `SummaryEstimate` above it holds `query = Quantile { q: 0.99 }` (§6.5), and its node carries the `guarantee` of the evaluation. `coverage()` on the `SummaryAgg` node returns a `SummaryCoverage` (§6.6). @@ -1313,16 +1375,16 @@ A finalized value's accuracy statement is `AccuracyGuarantee` (`ir/properties/gu ### 6.6 Summary coverage (`crates/types/src/ir/properties/summary_coverage.rs`) -The code for §4, as of #646. On `main` today `SummaryCoverage` is still the older declared form (`source` plus `regions`). +The code for §4. `SummaryStateDefinition` is shown in §4.2.2. ```rust pub struct SummaryCoverage { - /// What the state computes: the `SummaryAgg` and its sub-DAG, with the - /// conditions that went into `selection` taken out (§4.2.2). - pub definition: Rc, + /// What the state computes: its input group and the operators the walk + /// did not lift (§4.2.2). + pub(crate) definition: SummaryStateDefinition, /// Which rows went in: a union of boxes. A row is in the state if it is /// in at least one box. - pub selection: Vec, + pub(crate) selection: Vec, } /// One box: a row is in it when it meets every constraint. @@ -1376,9 +1438,10 @@ A PromQL range `m[r]` evaluated at t reads the samples in (t − r, t]: left-ope ```rust impl SummaryCoverage { - /// Computes the coverage of a summary node from its sub-DAG (§4.2.2). - /// Errors: `NotSummary` for a node that is not a `SummaryAgg`, - /// `SummaryTumblingWindowChain` or `SummaryMerge`; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` + /// The coverage of a summary-state group computed by `operator` over its + /// input groups (§4.2.2). Errors: `NotSummary` for an operator that is + /// not a `SummaryAgg`, `SummaryTumblingWindowChain`, `SummaryMerge` or a + /// `Filter` keeping whole states; `TumblingWindowChainExpansion` for a `SummaryTumblingWindowChain` /// that does not expand to one build over its whole window; for a /// merge, an input's own error, `EmptyMerge` (no inputs), /// `DefinitionMismatch` (inputs compute different things) or @@ -1388,9 +1451,12 @@ impl SummaryCoverage { /// not a group key of the inputs' build), `FamilyCannotRegroup` (counter /// states, a family without a merge, or a Hydra) or `RegroupedBuild` /// (the coarse build it stands for does not derive). - pub fn derive(node: &OperatorNode) -> Result; + pub(crate) fn derive_from_inputs( + operator: &Operator>, + ) -> Result; + pub(crate) fn definition_id(&self) -> String; } ``` -- `OperatorNode::new` and `validate_structure` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap; #592 relaxes the overlap check per family). The derived coverage is cached on the node. +- `find_or_add_group` and `add_candidate` reject an invalid `SummaryMerge` (different definitions, or selections that may overlap, per the family's `MergeInputRule`). Operator-node trees have no coverage and no merge check. - A `SummaryAgg` always has coverage: what cannot go into `selection` stays in `definition`. From 50686ae0d775475d781e46830eb4adaab85f9b82 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:51:58 +0000 Subject: [PATCH 27/30] refactor(types, frontends): column lineage replaces table qualifiers (#1207 PR 2b) GroupProperties gains column_lineage (one ColumnLineage per output column, derived once from the input groups'). Lineage and column_sign are derived together by one per-operator column layout (stages/column_properties.rs, which replaces column_sign.rs): a copied column keeps its lineage and sign, a computed one is Computed with the operator's sign rule. Chains compute window_end and their state and keep their keys' lineage; a TimeRange or TimeShift over a chain, and TimeShift { at: End | Start } above a selector, pass lineage through; a merge of a chain's selected windows has the chain build's layout. Summary coverage names a selection's column by its SourceColumn lineage (the one SourceColumn type, beside Source); ColumnIdentity is deleted. A predicate on a Computed column, or on one of two columns of one lineage, is not lifted; the regrouping key check compares lineage. Schema fields and the IR Project lose their table qualifier; the front ends bind qualified names with their own scope while lowering. The requalification patches (with_aggregate_qualifiers, node.rs) are deleted. --- crates/devtools/src/planner_fuzz/relations.rs | 8 +- crates/devtools/tests/viewer_contract.rs | 1 - crates/executor/src/expressions/columnar.rs | 1 - .../src/operators/aggregate/temporal.rs | 2 - crates/executor/src/operators/common.rs | 1 - crates/executor/src/operators/group_table.rs | 2 - .../executor/src/operators/series_labels.rs | 1 - crates/executor/src/operators/summary/mod.rs | 2 - crates/executor/src/values/batch.rs | 8 - .../executor/tests/aggregate_memory_limit.rs | 1 - crates/executor/tests/blocking_resources.rs | 3 - .../tests/cheap_measures_memory_limit.rs | 1 - crates/executor/tests/current_series_heap.rs | 1 - crates/executor/tests/deployment.rs | 1 - crates/executor/tests/exact_state_columns.rs | 1 - .../tests/grouped_limit_memory_limit.rs | 1 - crates/executor/tests/hydra_cms_execution.rs | 1 - .../tests/keyed_summary_row_weight.rs | 1 - crates/executor/tests/physical_dag.rs | 3 - crates/executor/tests/physical_semantics.rs | 2 - .../executor/tests/precompute_population.rs | 1 - crates/executor/tests/promql_binary.rs | 4 - crates/executor/tests/raw_scan.rs | 2 - .../tests/series_window_memory_limit.rs | 1 - crates/executor/tests/state_column_updater.rs | 1 - .../stored_tumbling_window_memory_limit.rs | 2 - .../tests/summary_build_memory_limit.rs | 1 - crates/executor/tests/summary_projection.rs | 4 - crates/executor/tests/text_extremum_memory.rs | 1 - crates/executor/tests/univmon_execution.rs | 1 - crates/frontend-common/src/canonicalize.rs | 7 - crates/frontend-common/src/resolve.rs | 574 ++++++++++++------ crates/frontend-common/src/schema_resolver.rs | 1 + crates/frontend-common/src/unresolved.rs | 7 + crates/frontend-metricsql/src/lib.rs | 1 + crates/frontend-promql/src/promql.rs | 1 + crates/frontend-sql/src/frequency.rs | 3 +- crates/frontend-sql/src/sql/mod.rs | 23 +- crates/frontend-sql/tests/sql_lowering.rs | 30 +- crates/planner/src/lib.rs | 3 +- crates/stage0-memo/src/lib.rs | 1 - crates/types/src/ir/flat.rs | 1 - crates/types/src/ir/operator/agg_intent.rs | 8 +- crates/types/src/ir/operator/node.rs | 25 +- crates/types/src/ir/operator/non_asap.rs | 39 +- .../src/ir/operator/operator_properties.rs | 12 +- crates/types/src/ir/properties/accuracy.rs | 1 - .../src/ir/properties/summary_coverage.rs | 185 +++--- .../types/src/ir/scalar/column_resolution.rs | 33 +- crates/types/src/ir/scalar/expr_ir.rs | 7 +- crates/types/src/ir/scalar/mod.rs | 1 - crates/types/src/ir/schema/mod.rs | 54 +- crates/types/src/ir/schema/topk.rs | 10 + crates/types/src/stages/candidates.rs | 15 +- crates/types/src/stages/column_properties.rs | 569 +++++++++++++++++ crates/types/src/stages/column_sign.rs | 449 -------------- crates/types/src/stages/memo.rs | 18 + crates/types/src/stages/mod.rs | 5 +- crates/types/src/test_support.rs | 8 +- crates/types/tests/column_lineage.rs | 427 +++++++++++++ crates/types/tests/main.rs | 1 + crates/types/tests/memo_group_properties.rs | 1 - crates/types/tests/memo_summary_coverage.rs | 55 +- crates/types/tests/physical_export.rs | 1 - crates/types/tests/schema_rebuilding.rs | 2 - crates/types/tests/structure_contract.rs | 2 - .../proposals/asap-primitive-schema.md | 79 ++- 67 files changed, 1707 insertions(+), 1012 deletions(-) create mode 100644 crates/types/src/stages/column_properties.rs delete mode 100644 crates/types/src/stages/column_sign.rs create mode 100644 crates/types/tests/column_lineage.rs diff --git a/crates/devtools/src/planner_fuzz/relations.rs b/crates/devtools/src/planner_fuzz/relations.rs index ddc2b2aaf..755d9e443 100644 --- a/crates/devtools/src/planner_fuzz/relations.rs +++ b/crates/devtools/src/planner_fuzz/relations.rs @@ -954,12 +954,7 @@ fn with_root_estimate( estimate: Rc, dropped: &[String], ) -> Result, String> { - let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - else { + let Some(NonASAPOp::Project { cols, child }) = root.non_asap() else { return Ok(estimate); }; let mut kept = Vec::new(); @@ -980,7 +975,6 @@ fn with_root_estimate( let cols = kept; OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { cols, - qualifier: qualifier.clone(), child: estimate, })) .map_err(|e| format!("projection over the new estimate: {e}")) diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs index ece3eee51..89f91ea3a 100644 --- a/crates/devtools/tests/viewer_contract.rs +++ b/crates/devtools/tests/viewer_contract.rs @@ -42,7 +42,6 @@ fn non_asap_samples() -> Vec> { }, NonASAPOp::Project { cols: vec![], - qualifier: None, child: (), }, NonASAPOp::Aggregate { diff --git a/crates/executor/src/expressions/columnar.rs b/crates/executor/src/expressions/columnar.rs index dbb37b317..d718588d2 100644 --- a/crates/executor/src/expressions/columnar.rs +++ b/crates/executor/src/expressions/columnar.rs @@ -813,7 +813,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/aggregate/temporal.rs b/crates/executor/src/operators/aggregate/temporal.rs index b913fad41..12771677e 100644 --- a/crates/executor/src/operators/aggregate/temporal.rs +++ b/crates/executor/src/operators/aggregate/temporal.rs @@ -341,13 +341,11 @@ mod tests { name: "time".into(), dtype: SummaryFamilyType::Plain(DataType::Timestamp), nullable: false, - table: None, }, SummaryField { name: "value".into(), dtype: SummaryFamilyType::Plain(DataType::Float64), nullable: false, - table: None, }, ], time_index: Some(0), diff --git a/crates/executor/src/operators/common.rs b/crates/executor/src/operators/common.rs index f0182757d..0a9732dc5 100644 --- a/crates/executor/src/operators/common.rs +++ b/crates/executor/src/operators/common.rs @@ -15,7 +15,6 @@ pub(super) fn result_field(name: &str, dtype: DataType, nullable: bool) -> Summa name: name.into(), dtype: SummaryFamilyType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/group_table.rs b/crates/executor/src/operators/group_table.rs index 64a173c97..53e6a702d 100644 --- a/crates/executor/src/operators/group_table.rs +++ b/crates/executor/src/operators/group_table.rs @@ -476,7 +476,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: true, - table: None, }; let schema = Arc::new(Schema { fields: vec![ @@ -548,7 +547,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let schema = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/src/operators/series_labels.rs b/crates/executor/src/operators/series_labels.rs index f44a87933..fdf8b8b73 100644 --- a/crates/executor/src/operators/series_labels.rs +++ b/crates/executor/src/operators/series_labels.rs @@ -1130,7 +1130,6 @@ mod columnar_tests { name: name.into(), dtype: SummaryFamilyType::Plain(dtype), nullable, - table: None, } } diff --git a/crates/executor/src/operators/summary/mod.rs b/crates/executor/src/operators/summary/mod.rs index 38a77dd6b..aa6d939f1 100644 --- a/crates/executor/src/operators/summary/mod.rs +++ b/crates/executor/src/operators/summary/mod.rs @@ -92,7 +92,6 @@ impl Operator { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }); Ok(Self { kind: Kind::AllGroupsSummaryBuild { @@ -176,7 +175,6 @@ impl Operator { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }); Ok(Self { kind: Kind::PerGroupSummaryBuild { diff --git a/crates/executor/src/values/batch.rs b/crates/executor/src/values/batch.rs index b9b5144bc..2b2f9b96c 100644 --- a/crates/executor/src/values/batch.rs +++ b/crates/executor/src/values/batch.rs @@ -616,7 +616,6 @@ mod tests { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable, - table: None, } } @@ -662,13 +661,11 @@ mod tests { name: "a".into(), dtype: DataType::Int64, nullable: false, - table: None, }, Field:: { name: "b".into(), dtype: DataType::Utf8, nullable: true, - table: None, }, ]; let labels = DataType::Map { @@ -786,7 +783,6 @@ mod tests { name: "state".into(), dtype: dd_family(), nullable: false, - table: None, }, ], time_index: None, @@ -868,7 +864,6 @@ mod tests { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }, ], time_index: None, @@ -1010,7 +1005,6 @@ mod tests { name: "state".into(), dtype: family, nullable: false, - table: None, }, ], time_index: None, @@ -1063,7 +1057,6 @@ mod tests { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }], time_index: None, unique_keys: vec![], @@ -1142,7 +1135,6 @@ mod tests { name: "state".into(), dtype: family, nullable: false, - table: None, }], time_index: None, unique_keys: vec![], diff --git a/crates/executor/tests/aggregate_memory_limit.rs b/crates/executor/tests/aggregate_memory_limit.rs index 75e6268b9..ac4a75085 100644 --- a/crates/executor/tests/aggregate_memory_limit.rs +++ b/crates/executor/tests/aggregate_memory_limit.rs @@ -53,7 +53,6 @@ fn run_aggregate( name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/blocking_resources.rs b/crates/executor/tests/blocking_resources.rs index 956f83be8..f0d36a1be 100644 --- a/crates/executor/tests/blocking_resources.rs +++ b/crates/executor/tests/blocking_resources.rs @@ -20,7 +20,6 @@ fn schema(width: usize) -> SchemaRef { closed: false, fields: (0..width) .map(|i| Field { - table: None, name: format!("v{i}"), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, @@ -193,13 +192,11 @@ fn weighted_summary_build_yields_within_a_batch() { closed: false, fields: vec![ Field { - table: None, name: "item".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, }, Field { - table: None, name: "weight".into(), dtype: FieldDataType::Plain(DataType::Float64), nullable: false, diff --git a/crates/executor/tests/cheap_measures_memory_limit.rs b/crates/executor/tests/cheap_measures_memory_limit.rs index 070950a98..873286fef 100644 --- a/crates/executor/tests/cheap_measures_memory_limit.rs +++ b/crates/executor/tests/cheap_measures_memory_limit.rs @@ -90,7 +90,6 @@ fn many_groups_of_cheap_measures_fail_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![field("k", DataType::Int64), field("v", DataType::Float64)], diff --git a/crates/executor/tests/current_series_heap.rs b/crates/executor/tests/current_series_heap.rs index e6e39014c..50618d0cf 100644 --- a/crates/executor/tests/current_series_heap.rs +++ b/crates/executor/tests/current_series_heap.rs @@ -28,7 +28,6 @@ fn schema() -> Arc { ] .into_iter() .map(|(name, dtype)| Field { - table: None, name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, diff --git a/crates/executor/tests/deployment.rs b/crates/executor/tests/deployment.rs index 79b070cb6..c07e6b3b1 100644 --- a/crates/executor/tests/deployment.rs +++ b/crates/executor/tests/deployment.rs @@ -248,7 +248,6 @@ fn heap_sketch_states_load_as_dag_summary_values() { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }], unique_keys: vec![], closed: false, diff --git a/crates/executor/tests/exact_state_columns.rs b/crates/executor/tests/exact_state_columns.rs index 27cf337b7..c9fa8aaec 100644 --- a/crates/executor/tests/exact_state_columns.rs +++ b/crates/executor/tests/exact_state_columns.rs @@ -17,7 +17,6 @@ fn field(name: &str, dtype: DataType) -> Field { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, } } diff --git a/crates/executor/tests/grouped_limit_memory_limit.rs b/crates/executor/tests/grouped_limit_memory_limit.rs index 10f5ceecf..e1b03188a 100644 --- a/crates/executor/tests/grouped_limit_memory_limit.rs +++ b/crates/executor/tests/grouped_limit_memory_limit.rs @@ -43,7 +43,6 @@ fn many_groups_in_one_batch_fail_near_the_memory_limit() { name: "k".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: false, - table: None, }], time_index: None, unique_keys: vec![], diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index a9988947a..eedc1d553 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -115,7 +115,6 @@ fn summary_agg(update: SummaryUpdate, filter: Option>) -> Physi name: "state".into(), dtype: family(), nullable: false, - table: None, }, ]), ) diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index bb81ab532..ed5162e4f 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -156,7 +156,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { name: "state".into(), dtype: heap_family(), nullable: false, - table: None, }, ])) .clone(), diff --git a/crates/executor/tests/physical_dag.rs b/crates/executor/tests/physical_dag.rs index 99d8dd0ea..2a4676b7e 100644 --- a/crates/executor/tests/physical_dag.rs +++ b/crates/executor/tests/physical_dag.rs @@ -25,7 +25,6 @@ fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { fields: fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: FieldDataType::Plain(dtype.clone()), nullable: *nullable, @@ -465,7 +464,6 @@ fn exact_state_and_family_validation() { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "state".into(), dtype: family.clone(), nullable: false, @@ -562,7 +560,6 @@ fn bind_physical_asap_dag_before_execution() { )), }, }], - qualifier: None, child: 0, }), ), diff --git a/crates/executor/tests/physical_semantics.rs b/crates/executor/tests/physical_semantics.rs index 3bc2f98c5..7782eab81 100644 --- a/crates/executor/tests/physical_semantics.rs +++ b/crates/executor/tests/physical_semantics.rs @@ -25,7 +25,6 @@ fn schema(fields: &[(&str, DataType, bool)]) -> SchemaRef { fields: fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: FieldDataType::Plain(dtype.clone()), nullable: *nullable, @@ -925,7 +924,6 @@ fn planner_node( fields .iter() .map(|(name, dtype, nullable)| Field { - table: None, name: (*name).into(), dtype: dtype.clone(), nullable: *nullable, diff --git a/crates/executor/tests/precompute_population.rs b/crates/executor/tests/precompute_population.rs index 31135b390..087fdf147 100644 --- a/crates/executor/tests/precompute_population.rs +++ b/crates/executor/tests/precompute_population.rs @@ -22,7 +22,6 @@ fn logical_schema(family: FieldDataType) -> Schema { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "value".into(), dtype: family, nullable: false, diff --git a/crates/executor/tests/promql_binary.rs b/crates/executor/tests/promql_binary.rs index b70a4d1be..b3b64d831 100644 --- a/crates/executor/tests/promql_binary.rs +++ b/crates/executor/tests/promql_binary.rs @@ -25,7 +25,6 @@ fn schema() -> SchemaRef { closed: false, fields: vec![ Field { - table: None, name: "labels".into(), dtype: FieldDataType::Plain(DataType::Map { key: Box::new(DataType::Utf8), @@ -35,7 +34,6 @@ fn schema() -> SchemaRef { nullable: false, }, Field { - table: None, name: "value".into(), dtype: FieldDataType::Plain(DataType::Float64), nullable: false, @@ -344,13 +342,11 @@ fn stored_series_evaluations_support_filters_and_sets() { closed: false, fields: vec![ Field { - table: None, name: PROMQL_SERIES_IDENTITY.into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, }, Field { - table: None, name: "value".into(), dtype: family.clone(), nullable: false, diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index dfa473590..aa222032a 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -34,7 +34,6 @@ fn fixture() -> (planner_types::ir::NonASAPOp, SchemaRef, Vec) { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "value".into(), dtype: FieldDataType::Plain(DataType::Int64), nullable: true, @@ -826,7 +825,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { dag.edges.retain(|edge| edge.consumer < 3); let family = FieldDataType::ExactAggregate(ExactKind::Count, ExactParams::Count); let state = planner_types::ir::schema::Schema::new(vec![Field { - table: None, name: "count".into(), dtype: family.clone(), nullable: false, diff --git a/crates/executor/tests/series_window_memory_limit.rs b/crates/executor/tests/series_window_memory_limit.rs index e8c83bcd5..07ca7d29f 100644 --- a/crates/executor/tests/series_window_memory_limit.rs +++ b/crates/executor/tests/series_window_memory_limit.rs @@ -40,7 +40,6 @@ fn a_long_subquery_over_many_series_fails_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/state_column_updater.rs b/crates/executor/tests/state_column_updater.rs index 05a57b1cc..6c488182c 100644 --- a/crates/executor/tests/state_column_updater.rs +++ b/crates/executor/tests/state_column_updater.rs @@ -27,7 +27,6 @@ fn plain(name: &str, dtype: DataType) -> Field { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, } } diff --git a/crates/executor/tests/stored_tumbling_window_memory_limit.rs b/crates/executor/tests/stored_tumbling_window_memory_limit.rs index ea92264ca..ace992ddd 100644 --- a/crates/executor/tests/stored_tumbling_window_memory_limit.rs +++ b/crates/executor/tests/stored_tumbling_window_memory_limit.rs @@ -32,13 +32,11 @@ fn tumbling_window_of(series_count: u32) -> (Arc, Batch) { name: "series".into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, - table: None, }, Field { name: "state".into(), dtype: family.clone(), nullable: false, - table: None, }, ], time_index: None, diff --git a/crates/executor/tests/summary_build_memory_limit.rs b/crates/executor/tests/summary_build_memory_limit.rs index 0b6a7d134..f80be498f 100644 --- a/crates/executor/tests/summary_build_memory_limit.rs +++ b/crates/executor/tests/summary_build_memory_limit.rs @@ -40,7 +40,6 @@ fn one_large_batch_fails_near_the_memory_limit() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![ diff --git a/crates/executor/tests/summary_projection.rs b/crates/executor/tests/summary_projection.rs index 7d5a95ec9..a51286344 100644 --- a/crates/executor/tests/summary_projection.rs +++ b/crates/executor/tests/summary_projection.rs @@ -83,13 +83,11 @@ fn summary_projection_survives_recovery() { closed: false, fields: vec![ Field { - table: None, name: "state".into(), dtype: family.clone(), nullable: false, }, Field { - table: None, name: "service".into(), dtype: FieldDataType::Plain(DataType::Utf8), nullable: false, @@ -149,7 +147,6 @@ fn summary_projection_survives_recovery() { expr: planner_types::ir::ScalarExpr::Column(index), }) .collect(), - qualifier: None, child: 0, }), output_schema: output.clone(), @@ -247,7 +244,6 @@ fn summary_merge_without_inputs_is_rejected() { unique_keys: vec![], closed: false, fields: vec![Field { - table: None, name: "state".into(), dtype: family, nullable: false, diff --git a/crates/executor/tests/text_extremum_memory.rs b/crates/executor/tests/text_extremum_memory.rs index 7aec609c3..0cc056fa4 100644 --- a/crates/executor/tests/text_extremum_memory.rs +++ b/crates/executor/tests/text_extremum_memory.rs @@ -65,7 +65,6 @@ fn text_max_charges_only_the_current_value() { name: name.into(), dtype: FieldDataType::Plain(dtype), nullable: false, - table: None, }; let input = Arc::new(Schema { fields: vec![field("k", DataType::Int64), field("t", DataType::Utf8)], diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index 7a7b083e4..cf0aed856 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -157,7 +157,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() false, )])); let state = Arc::new(Schema::new(vec![Field { - table: None, name: "state".into(), dtype: family(), nullable: false, diff --git a/crates/frontend-common/src/canonicalize.rs b/crates/frontend-common/src/canonicalize.rs index ffd5b7e0d..63f08b96b 100644 --- a/crates/frontend-common/src/canonicalize.rs +++ b/crates/frontend-common/src/canonicalize.rs @@ -316,7 +316,6 @@ fn try_rewrite_row_number_top_k( ) -> Result>, SchemaDerivationError> { let Some(NonASAPOp::Project { cols: select_list, - qualifier, child: filter, }) = node.non_asap() else { @@ -429,14 +428,12 @@ fn try_rewrite_row_number_top_k( (Some(partition), Some(order)) if !partition_by.is_without() => { let selected = new(NonASAPOp::Project { cols: kept, - qualifier: qualifier.clone(), child: Rc::clone(ranked), })?; limit(GroupKeys::by(partition), order, selected)? } _ => new(NonASAPOp::Project { cols: kept, - qualifier: qualifier.clone(), child: limit(partition_by.clone(), order_by.clone(), Rc::clone(ranked))?, })?, }; @@ -638,7 +635,6 @@ mod tests { expr: ScalarExpr::Column(1), }, ], - qualifier: None, child, }) } @@ -756,7 +752,6 @@ mod tests { alias: None, expr: ScalarExpr::Column(0), }], - qualifier: None, child: Rc::clone(&ranking), }); let out = canonicalize(Rc::clone(&q)).unwrap(); @@ -984,7 +979,6 @@ mod tests { expr: ScalarExpr::Column(col), }) .collect(), - qualifier: None, child, }) } @@ -1222,7 +1216,6 @@ mod tests { let sub = one_column_subquery(); let root = node(NonASAPOp::Project { child: scan(), - qualifier: None, cols: vec![ ProjectItem { alias: Some("a".into()), diff --git a/crates/frontend-common/src/resolve.rs b/crates/frontend-common/src/resolve.rs index 1a4c230f3..87b80d73a 100644 --- a/crates/frontend-common/src/resolve.rs +++ b/crates/frontend-common/src/resolve.rs @@ -20,8 +20,8 @@ use std::rc::Rc; use thiserror::Error; -use asap_types::ir::operator::{AggIntent, GroupKeys, Reduction}; -use asap_types::ir::scalar::{resolve_column_ref, resolve_column_refs, ColumnRef, ResolveError}; +use asap_types::ir::operator::{AggIntent, GroupKeys, JoinKind, Reduction}; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, ResolveError}; use asap_types::ir::schema::{ColumnId, Schema}; use asap_types::ir::SchemaDerivationError; use asap_types::ir::{NonASAPOp, OperatorNode, Predicate, ProjectItem, ScalarExpr, SortKey}; @@ -43,23 +43,102 @@ pub enum ResolveDAGError { use crate::canonicalize::canonicalize; +/// The names an operator's parent resolves against: the operator's output +/// schema and, per column, the SQL table or alias that qualifies it (`None` +/// when unqualified; an empty list qualifies no column). A qualified +/// reference (`f.src`) binds here, while lowering; the IR keeps no qualifier. +#[derive(Clone, Copy)] +struct Scope<'a> { + schema: &'a Schema, + qualifiers: &'a [Option], +} + +impl<'a> Scope<'a> { + /// A scope whose columns have no qualifier. + fn unqualified(schema: &'a Schema) -> Self { + Self { + schema, + qualifiers: &[], + } + } + + /// The column `column` names: by qualifier and name when it is + /// qualified and a column has both, else by its unique name. + fn resolve(&self, column: &ColumnRef) -> Result { + if let ColumnRef::Qualified { table, name } = column { + let qualified = + self.schema + .fields + .iter() + .zip(self.qualifiers) + .position(|(field, qualifier)| { + field.name == *name && qualifier.as_deref() == Some(table.as_str()) + }); + if let Some(id) = qualified { + return Ok(id); + } + } + resolve_column_ref(column, self.schema) + } + + fn resolve_all(&self, columns: &[ColumnRef]) -> Result, ResolveError> { + columns.iter().map(|column| self.resolve(column)).collect() + } + + /// The qualifier of column `column`. + fn qualifier(&self, column: ColumnId) -> Option { + self.qualifiers.get(column).cloned().flatten() + } +} + +/// A resolved operator with the qualifier of each output column: the scope +/// its parent resolves names in. +struct Bound { + node: Rc, + qualifiers: Vec>, +} + +impl Bound { + /// `node` with `qualifiers`; unqualified unless there is one per column. + fn new(node: Rc, qualifiers: Vec>) -> Self { + let qualifiers = if qualifiers.len() == node.schema.fields.len() { + qualifiers + } else { + Vec::new() + }; + Self { node, qualifiers } + } + + fn unqualified(node: Rc) -> Self { + Self::new(node, Vec::new()) + } + + fn scope(&self) -> Scope<'_> { + Scope { + schema: &self.node.schema, + qualifiers: &self.qualifiers, + } + } +} + /// Resolve the whole tree rooted at `tree`: bind every `ColumnRef` to a /// `ColumnId` via the usage-derived schema resolver, then canonicalize the result. pub fn resolve_root(tree: &UnresolvedOp) -> Result, ResolveDAGError> { - resolve_root_with_inherited(tree, &[]) + Ok(resolve_root_with_inherited(tree, &[])?.node) } /// [`resolve_root`] with label names inherited from an enclosing scope seeded /// into the leaf schema (a `BinaryOp` side, a scalar operand's operator). +/// Canonicalizing keeps the root's output columns, so their qualifiers. fn resolve_root_with_inherited( tree: &UnresolvedOp, inherited: &[String], -) -> Result, ResolveDAGError> { +) -> Result { let usage_derived_schema = resolve_schema_with_inherited(tree, inherited); let root = resolve(tree, &usage_derived_schema)?; - let root = canonicalize(root)?; - root.validate_structure()?; - Ok(root) + let node = canonicalize(root.node)?; + node.validate_structure()?; + Ok(Bound::new(node, root.qualifiers)) } /// Bind `tree` as a root in its own scope, inheriting from `enclosing` the @@ -73,7 +152,7 @@ fn resolve_nested_root( .into_iter() .filter(|n| !own.contains(n)) .collect(); - resolve_root_with_inherited(tree, &inherited) + Ok(resolve_root_with_inherited(tree, &inherited)?.node) } fn node(op: NonASAPOp) -> Result, ResolveDAGError> { @@ -83,31 +162,32 @@ fn node(op: NonASAPOp) -> Result, ResolveDAGError> { } /// The generic substitution walk. `usage_derived_schema` is the schema a -/// schemaless `Scan` in this scope binds to. -fn resolve( - tree: &UnresolvedOp, - usage_derived_schema: &Schema, -) -> Result, ResolveDAGError> { +/// schemaless `Scan` in this scope binds to. Qualifiers follow the columns: +/// a `Scan` qualifies its columns by its own; an operator that passes +/// columns through, a join and an aggregate's group keys keep them; a +/// `Project` qualifies all its columns by its own or none. +fn resolve(tree: &UnresolvedOp, usage_derived_schema: &Schema) -> Result { use UnresolvedOp as U; - let expr = - |e: &UnresolvedScalar, schema: &Schema| resolve_expr_in(e, schema, usage_derived_schema); - let pred = |p: &UnresolvedScalar, schema: &Schema| { - Ok::<_, ResolveDAGError>(Predicate(expr(p, schema)?)) - }; - let sort_keys = |keys: &[UnresolvedSortKey], schema: &Schema| { + let expr = |e: &UnresolvedScalar, scope: Scope| resolve_expr_in(e, scope, usage_derived_schema); + let pred = + |p: &UnresolvedScalar, scope: Scope| Ok::<_, ResolveDAGError>(Predicate(expr(p, scope)?)); + let sort_keys = |keys: &[UnresolvedSortKey], scope: Scope| { keys.iter() .map(|k| { Ok::<_, ResolveDAGError>(SortKey { - expr: expr(&k.expr, schema)?, + expr: expr(&k.expr, scope)?, ascending: k.ascending, nulls_first: k.nulls_first, }) }) .collect::, _>>() }; + // A node over `child` with `child`'s columns and qualifiers. + let passing = |op: NonASAPOp, child: Bound| Ok(Bound::new(node(op)?, child.qualifiers)); + let unqualified = |op: NonASAPOp| Ok(Bound::unqualified(node(op)?)); // A PromQL child over an open schema gets an opaque series-identity column. let promql_series = |child: &UnresolvedOp| -> Result, ResolveDAGError> { - let child = resolve(child, usage_derived_schema)?; + let child = resolve(child, usage_derived_schema)?.node; if child.schema.closed { return Ok(child); } @@ -119,19 +199,28 @@ fn resolve( source, predicates, schema, + qualifier, } => { let schema = schema .clone() .unwrap_or_else(|| usage_derived_schema.clone()); + let qualifiers = vec![qualifier.clone(); schema.fields.len()]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let predicates = predicates .iter() - .map(|p| pred(&p.0, &schema)) + .map(|p| pred(&p.0, scope)) .collect::, _>>()?; - node(NonASAPOp::Scan { - source: source.clone(), - predicates, - schema, - }) + Ok(Bound::new( + node(NonASAPOp::Scan { + source: source.clone(), + predicates, + schema, + })?, + qualifiers, + )) } // Row expressions have no input-column scope. @@ -141,11 +230,11 @@ fn resolve( .iter() .map(|row| { row.iter() - .map(|e| expr(e, &empty)) + .map(|e| expr(e, Scope::unqualified(&empty))) .collect::, _>>() }) .collect::, _>>()?; - node(NonASAPOp::Values { + unqualified(NonASAPOp::Values { rows, schema: schema.clone(), }) @@ -162,7 +251,13 @@ fn resolve( } => { let child = promql_series(child)?; let scalar = resolve_expr(scalar, &Schema::default())?; - lower_scalar_vector(child, scalar, op, *scalar_left, *return_bool) + Ok(Bound::unqualified(lower_scalar_vector( + child, + scalar, + op, + *scalar_left, + *return_bool, + )?)) } U::PromqlMap { child, @@ -171,42 +266,61 @@ fn resolve( } => { let child = promql_series(child)?; let sample = resolve_expr(sample, &child.schema)?; - project_sample(child, sample, *drop_metric_name) + Ok(Bound::unqualified(project_sample( + child, + sample, + *drop_metric_name, + )?)) } - U::PromqlVectorFromScalar(inner) => node(NonASAPOp::PromqlVectorFromScalar(expr( + U::PromqlVectorFromScalar(inner) => unqualified(NonASAPOp::PromqlVectorFromScalar(expr( inner, - usage_derived_schema, + Scope::unqualified(usage_derived_schema), )?)), U::PromqlRelabel { dst, value, child } => { let child = resolve(child, usage_derived_schema)?; - let value = expr(value, &child.schema)?; - node(NonASAPOp::PromqlRelabel { + let value = expr(value, child.scope())?; + unqualified(NonASAPOp::PromqlRelabel { dst: dst.clone(), value, - child, + child: child.node, }) } - U::PromqlInfoEnrich { selector, child } => node(NonASAPOp::PromqlInfoEnrich { - selector: selector.clone(), - child: resolve(child, usage_derived_schema)?, - }), + U::PromqlInfoEnrich { selector, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::PromqlInfoEnrich { + selector: selector.clone(), + child: Rc::clone(&child.node), + }, + child, + ) + } U::PromqlSeriesSample { by, kind, child } => { let child = resolve(child, usage_derived_schema)?; - let by = resolve_group_keys(by, &child.schema)?; - node(NonASAPOp::PromqlSeriesSample { - by, - kind: *kind, + let by = resolve_group_keys(by, child.scope())?; + passing( + NonASAPOp::PromqlSeriesSample { + by, + kind: *kind, + child: Rc::clone(&child.node), + }, child, - }) + ) } U::Filter { pred: p, child } => { let child = resolve(child, usage_derived_schema)?; - let pred = pred(&p.0, &child.schema)?; - node(NonASAPOp::Filter { pred, child }) + let pred = pred(&p.0, child.scope())?; + passing( + NonASAPOp::Filter { + pred, + child: Rc::clone(&child.node), + }, + child, + ) } U::Project { @@ -220,15 +334,18 @@ fn resolve( .map(|item| { Ok::<_, ResolveDAGError>(ProjectItem { alias: item.alias.clone(), - expr: expr(&item.expr, &child.schema)?, + expr: expr(&item.expr, child.scope())?, }) }) .collect::, _>>()?; - node(NonASAPOp::Project { - cols, - qualifier: qualifier.clone(), - child, - }) + let qualifiers = vec![qualifier.clone(); cols.len()]; + Ok(Bound::new( + node(NonASAPOp::Project { + cols, + child: child.node, + })?, + qualifiers, + )) } U::Aggregate { @@ -240,42 +357,76 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let reduction = resolve_reduction(reduction, &child.schema)?; + let scope = child.scope(); + let reduction = resolve_reduction(reduction, scope)?; let measures = measures .iter() - .map(|m| resolve_agg_intent(m, &child.schema)) + .map(|m| resolve_agg_intent(m, scope)) .collect::, ResolveError>>()?; let filters = filters .iter() - .map(|p| p.as_ref().map(|p| pred(&p.0, &child.schema)).transpose()) + .map(|p| p.as_ref().map(|p| pred(&p.0, scope)).transpose()) .collect::, _>>()?; + // A `by` key keeps its qualifier; a per-series reduction keeps + // its input's columns in place. A top-k emits ranked rows. + let qualifiers: Vec> = match &reduction { + _ if matches!(measures.as_slice(), [AggIntent::TopK { .. }]) => Vec::new(), + Reduction::PerEntity => child.qualifiers.clone(), + Reduction::Reduce(by) if by.is_without() => Vec::new(), + Reduction::Reduce(by) => by + .keys() + .iter() + .map(|&key| scope.qualifier(key)) + .chain(std::iter::repeat_n(None, measures.len())) + .collect(), + }; // HAVING is evaluated over the aggregate's own output. let having = having .as_ref() .map(|h| { let out_schema = aggregate_output_schema( - &child.schema, + &child.node.schema, &reduction, &measures, output_names, )?; - pred(&h.0, &out_schema) + let out_qualifiers = if qualifiers.len() == out_schema.fields.len() { + &qualifiers[..] + } else { + &[] + }; + pred( + &h.0, + Scope { + schema: &out_schema, + qualifiers: out_qualifiers, + }, + ) }) .transpose()?; - node(NonASAPOp::Aggregate { - reduction, - measures, - output_names: output_names.clone(), - filters, - having, - child, - }) + Ok(Bound::new( + node(NonASAPOp::Aggregate { + reduction, + measures, + output_names: output_names.clone(), + filters, + having, + child: child.node, + })?, + qualifiers, + )) } U::Dedup { cols, child } => { let child = resolve(child, usage_derived_schema)?; - let cols = resolve_column_refs(cols, &child.schema)?; - node(NonASAPOp::Dedup { cols, child }) + let cols = child.scope().resolve_all(cols)?; + passing( + NonASAPOp::Dedup { + cols, + child: Rc::clone(&child.node), + }, + child, + ) } U::Concat { children } => { @@ -283,7 +434,15 @@ fn resolve( .iter() .map(|c| resolve(c, usage_derived_schema)) .collect::, _>>()?; - node(NonASAPOp::Concat { children }) + let qualifiers = children + .first() + .map_or_else(Vec::new, |first| first.qualifiers.clone()); + Ok(Bound::new( + node(NonASAPOp::Concat { + children: children.into_iter().map(|child| child.node).collect(), + })?, + qualifiers, + )) } U::Join { @@ -296,15 +455,37 @@ fn resolve( // sets); the predicate sees left ++ right. let left = resolve_root_with_inherited(left, &[])?; let right = resolve_root_with_inherited(right, &[])?; - let mut concat = left.schema.clone(); - concat.fields.extend(right.schema.fields.iter().cloned()); - let pred = pred(&p.0, &concat)?; - node(NonASAPOp::Join { - kind: kind.clone(), - pred, - left, - right, - }) + let qualify = |side: &Bound| { + (0..side.node.schema.fields.len()) + .map(|column| side.scope().qualifier(column)) + .collect::>() + }; + let mut concat = left.node.schema.clone(); + concat + .fields + .extend(right.node.schema.fields.iter().cloned()); + let mut qualifiers = qualify(&left); + qualifiers.extend(qualify(&right)); + let pred = pred( + &p.0, + Scope { + schema: &concat, + qualifiers: &qualifiers, + }, + )?; + // A semi or anti join outputs the left side's columns. + if matches!(kind, JoinKind::Semi | JoinKind::Anti) { + qualifiers.truncate(left.node.schema.fields.len()); + } + Ok(Bound::new( + node(NonASAPOp::Join { + kind: kind.clone(), + pred, + left: left.node, + right: right.node, + })?, + qualifiers, + )) } U::SetOp { @@ -312,12 +493,19 @@ fn resolve( all, left, right, - } => node(NonASAPOp::SetOp { - kind: kind.clone(), - all: *all, - left: resolve_root_with_inherited(left, &[])?, - right: resolve_root_with_inherited(right, &[])?, - }), + } => { + let left = resolve_root_with_inherited(left, &[])?; + let right = resolve_root_with_inherited(right, &[])?; + Ok(Bound::new( + node(NonASAPOp::SetOp { + kind: kind.clone(), + all: *all, + left: left.node, + right: right.node, + })?, + left.qualifiers, + )) + } U::Sort { keys, @@ -325,13 +513,16 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let keys = sort_keys(keys, &child.schema)?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - node(NonASAPOp::Sort { - keys, - partition_by, + let keys = sort_keys(keys, child.scope())?; + let partition_by = resolve_group_keys(partition_by, child.scope())?; + passing( + NonASAPOp::Sort { + keys, + partition_by, + child: Rc::clone(&child.node), + }, child, - }) + ) } U::Limit { @@ -341,35 +532,56 @@ fn resolve( child, } => { let child = resolve(child, usage_derived_schema)?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - node(NonASAPOp::Limit { - n: *n, - offset: *offset, - partition_by, + let partition_by = resolve_group_keys(partition_by, child.scope())?; + passing( + NonASAPOp::Limit { + n: *n, + offset: *offset, + partition_by, + child: Rc::clone(&child.node), + }, child, - }) + ) } U::PromqlSubquery { range, resolution, child, - } => node(NonASAPOp::PromqlSubquery { - range: *range, - resolution: *resolution, - child: resolve(child, usage_derived_schema)?, - }), - - U::TimeRange { range, kind, child } => node(NonASAPOp::TimeRange { - range: *range, - kind: *kind, - child: resolve(child, usage_derived_schema)?, - }), - - U::TimeShift { shift, child } => node(NonASAPOp::TimeShift { - shift: *shift, - child: resolve(child, usage_derived_schema)?, - }), + } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::PromqlSubquery { + range: *range, + resolution: *resolution, + child: Rc::clone(&child.node), + }, + child, + ) + } + + U::TimeRange { range, kind, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::TimeRange { + range: *range, + kind: *kind, + child: Rc::clone(&child.node), + }, + child, + ) + } + + U::TimeShift { shift, child } => { + let child = resolve(child, usage_derived_schema)?; + passing( + NonASAPOp::TimeShift { + shift: *shift, + child: Rc::clone(&child.node), + }, + child, + ) + } U::SQLWindowFunc { func, @@ -383,19 +595,27 @@ fn resolve( let child = resolve(child, usage_derived_schema)?; let args = args .iter() - .map(|a| expr(a, &child.schema)) + .map(|a| expr(a, child.scope())) .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child.schema)?; - let order_by = sort_keys(order_by, &child.schema)?; - node(NonASAPOp::SQLWindowFunc { - func: func.clone(), - args, - partition_by, - order_by, - frame: frame.clone(), - output_name: output_name.clone(), - child, - }) + let partition_by = resolve_group_keys(partition_by, child.scope())?; + let order_by = sort_keys(order_by, child.scope())?; + // The window function's own column comes last, unqualified. + let mut qualifiers = child.qualifiers.clone(); + if !qualifiers.is_empty() { + qualifiers.push(None); + } + Ok(Bound::new( + node(NonASAPOp::SQLWindowFunc { + func: func.clone(), + args, + partition_by, + order_by, + frame: frame.clone(), + output_name: output_name.clone(), + child: child.node, + })?, + qualifiers, + )) } U::BinaryOp { @@ -414,11 +634,11 @@ fn resolve( .into_iter() .filter(|n| !own.contains(n)) .collect(); - node(NonASAPOp::BinaryOp { + unqualified(NonASAPOp::BinaryOp { operator: operator.clone(), return_bool: *return_bool, - lhs: resolve_root_with_inherited(lhs, &inherited)?, - rhs: resolve_root_with_inherited(rhs, &inherited)?, + lhs: resolve_root_with_inherited(lhs, &inherited)?.node, + rhs: resolve_root_with_inherited(rhs, &inherited)?.node, }) } } @@ -442,28 +662,29 @@ pub(crate) fn resolve_expr( expr: &UnresolvedScalar, schema: &Schema, ) -> Result { - resolve_expr_in(expr, schema, schema) + resolve_expr_in(expr, Scope::unqualified(schema), schema) } -/// [`resolve_expr`] where the operators the expression reads inherit from -/// `enclosing` (the owning root's usage-derived schema) rather than from `schema`. +/// [`resolve_expr`] in `scope`, where the operators the expression reads +/// inherit from `enclosing` (the owning root's usage-derived schema) rather +/// than from the scope's schema. fn resolve_expr_in( expr: &UnresolvedScalar, - schema: &Schema, + scope: Scope, enclosing: &Schema, ) -> Result { use UnresolvedScalar as S; let bx = |e: &UnresolvedScalar| -> Result, ResolveDAGError> { - Ok(Box::new(resolve_expr_in(e, schema, enclosing)?)) + Ok(Box::new(resolve_expr_in(e, scope, enclosing)?)) }; let each = |es: &[UnresolvedScalar]| -> Result, ResolveDAGError> { es.iter() - .map(|e| resolve_expr_in(e, schema, enclosing)) + .map(|e| resolve_expr_in(e, scope, enclosing)) .collect() }; let op = |o: &UnresolvedOp| resolve_nested_root(o, enclosing); Ok(match expr { - S::Column(c) => ScalarExpr::Column(resolve_column_ref(c, schema)?), + S::Column(c) => ScalarExpr::Column(scope.resolve(c)?), S::Literal(s) => ScalarExpr::Literal(s.clone()), S::EvalTimestamp => ScalarExpr::EvalTimestamp, S::CurrentTimestamp => ScalarExpr::CurrentTimestamp, @@ -492,7 +713,7 @@ fn resolve_expr_in( // nanosecond timestamp, which is the same IR type. S::Cast { expr, to, try_cast } => { let operand = bx(expr)?; - if matches!(operand.scalar_type(schema), Ok((dtype, _)) if dtype == *to) { + if matches!(operand.scalar_type(scope.schema), Ok((dtype, _)) if dtype == *to) { *operand } else { ScalarExpr::Cast { @@ -536,8 +757,8 @@ fn resolve_expr_in( .iter() .map(|(w, t)| { Ok(( - resolve_expr_in(w, schema, enclosing)?, - resolve_expr_in(t, schema, enclosing)?, + resolve_expr_in(w, scope, enclosing)?, + resolve_expr_in(t, scope, enclosing)?, )) }) .collect::, ResolveDAGError>>()?, @@ -564,9 +785,9 @@ fn resolve_expr_in( /// Resolve name-based group keys positionally, preserving `by`/`without`. fn resolve_group_keys( keys: &GroupKeys, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { - let ids = resolve_column_refs(keys.keys(), schema)?; + let ids = scope.resolve_all(keys.keys())?; Ok(if keys.is_without() { GroupKeys::without(ids) } else { @@ -587,15 +808,15 @@ fn resolve_group_keys( /// may exist at runtime, and PromQL leaves seed every referenced label via the /// schema resolver, so an unresolved key over an open schema indicates a resolution bug, /// not an absent label. (SQL is unaffected — its `GROUP BY` resolves through -/// the strict [`resolve_column_refs`], and DataFusion has already validated +/// the strict [`resolve_group_keys`], and DataFusion has already validated /// the columns anyway.) fn resolve_group_keys_promql( cols: &[ColumnRef], - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { cols.iter() - .filter_map(|c| match resolve_column_ref(c, schema) { - Err(ResolveError::NotFound { .. }) if schema.closed => None, + .filter_map(|c| match scope.resolve(c) { + Err(ResolveError::NotFound { .. }) if scope.schema.closed => None, other => Some(other), }) .collect() @@ -610,11 +831,11 @@ fn resolve_group_keys_promql( /// path is a no-op difference there. fn resolve_reduction( reduction: &Reduction, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { Ok(match reduction { Reduction::Reduce(by) => { - let ids = resolve_group_keys_promql(by.keys(), schema)?; + let ids = resolve_group_keys_promql(by.keys(), scope)?; Reduction::Reduce(if by.is_without() { GroupKeys::without(ids) } else { @@ -630,18 +851,16 @@ fn resolve_reduction( /// convention); every other field carries through unchanged. fn resolve_agg_intent( intent: &AggIntent, - schema: &Schema, + scope: Scope, ) -> Result, ResolveError> { let col = |c: &Option| -> Result, ResolveError> { - c.as_ref() - .map(|r| resolve_column_ref(r, schema)) - .transpose() + c.as_ref().map(|r| scope.resolve(r)).transpose() }; Ok(match intent { AggIntent::Count => AggIntent::Count, AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { - left: resolve_column_ref(left, schema)?, - right: resolve_column_ref(right, schema)?, + left: scope.resolve(left)?, + right: scope.resolve(right)?, }, AggIntent::Sum { col: c } => AggIntent::Sum { col: col(c)? }, AggIntent::Min { col: c } => AggIntent::Min { col: col(c)? }, @@ -661,10 +880,7 @@ fn resolve_agg_intent( }, AggIntent::TopK { k } => AggIntent::TopK { k: *k }, AggIntent::Cardinality { cols } => AggIntent::Cardinality { - cols: cols - .iter() - .map(|c| resolve_column_ref(c, schema)) - .collect::>()?, + cols: scope.resolve_all(cols)?, }, AggIntent::FrequencyL2 { col: c } => AggIntent::FrequencyL2 { col: col(c)? }, AggIntent::FrequencyEntropy { col: c } => AggIntent::FrequencyEntropy { col: col(c)? }, @@ -692,7 +908,7 @@ fn resolve_agg_intent( }, AggIntent::HistogramQuantile { q, le } => AggIntent::HistogramQuantile { q: *q, - le: resolve_column_ref(le, schema)?, + le: scope.resolve(le)?, }, AggIntent::Math(f) => AggIntent::Math(f.clone()), AggIntent::Absent { labels } => AggIntent::Absent { @@ -832,11 +1048,7 @@ fn project_sample( } }) .collect(); - node(NonASAPOp::Project { - child, - cols, - qualifier: None, - }) + node(NonASAPOp::Project { child, cols }) } #[cfg(test)] @@ -859,6 +1071,7 @@ mod tests { }, predicates: vec![], schema: None, + qualifier: None, } } @@ -888,9 +1101,14 @@ mod tests { #[test] fn resolve_pearson_corr_inputs() { let schema = Schema::new(vec![ - Field::plain("x", DataType::Float64, true).with_table("a"), - Field::plain("x", DataType::Float64, true).with_table("b"), + Field::plain("x", DataType::Float64, true), + Field::plain("x", DataType::Float64, true), ]); + let qualifiers = [Some("a".to_string()), Some("b".to_string())]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let intent = AggIntent::PearsonCorr { left: ColumnRef::Qualified { table: "a".into(), @@ -902,7 +1120,7 @@ mod tests { }, }; assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), + resolve_agg_intent(&intent, scope).unwrap(), AggIntent::PearsonCorr { left: 0, right: 1 } ); let missing = AggIntent::PearsonCorr { @@ -912,7 +1130,7 @@ mod tests { }, right: ColumnRef::Named("missing".into()), }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); + assert!(resolve_agg_intent(&missing, scope).is_err()); } // Every leg resolves independently, qualifiers included; one unknown leg @@ -920,9 +1138,14 @@ mod tests { #[test] fn resolve_distinct_tuple_columns() { let schema = Schema::new(vec![ - Field::plain("k", DataType::Int64, true).with_table("a"), - Field::plain("k", DataType::Int64, true).with_table("b"), + Field::plain("k", DataType::Int64, true), + Field::plain("k", DataType::Int64, true), ]); + let qualifiers = [Some("a".to_string()), Some("b".to_string())]; + let scope = Scope { + schema: &schema, + qualifiers: &qualifiers, + }; let qualified = |table: &str| ColumnRef::Qualified { table: table.into(), name: "k".into(), @@ -931,13 +1154,13 @@ mod tests { cols: vec![qualified("b"), qualified("a")], }; assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), + resolve_agg_intent(&intent, scope).unwrap(), AggIntent::Cardinality { cols: vec![1, 0] } ); let missing = AggIntent::Cardinality { cols: vec![qualified("a"), ColumnRef::Named("missing".into())], }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); + assert!(resolve_agg_intent(&missing, scope).is_err()); } // ` > `: the bridged literal comes through unchanged, the @@ -1028,6 +1251,7 @@ mod tests { }, predicates: vec![], schema: Some(input), + qualifier: None, }), }; let resolved = resolve_root(&unresolved).expect("resolves"); @@ -1051,9 +1275,8 @@ mod tests { table_ref: table.into(), }, predicates: vec![], - schema: Some(Schema::new(vec![ - Field::plain("k", DataType::Int64, false).with_table(table) - ])), + schema: Some(Schema::new(vec![Field::plain("k", DataType::Int64, false)])), + qualifier: Some(table.into()), }; let qualified = |table: &str| { UnresolvedScalar::Column(ColumnRef::Qualified { @@ -1093,6 +1316,7 @@ mod tests { source: Source::TimeSeries { metric: "x".into() }, predicates: vec![UnresolvedPredicate(eq_lit(named("a"), "1"))], schema: None, + qualifier: None, }; let unresolved = UnresolvedOp::PromqlScalarOp { child: Rc::new(scan("m")), @@ -1182,7 +1406,7 @@ mod tests { closed: true, }; assert_eq!( - resolve_group_keys_promql(&[ColumnRef::Named("job".into())], &s), + resolve_group_keys_promql(&[ColumnRef::Named("job".into())], Scope::unqualified(&s)), Ok(vec![]) ); // Present keys still resolve positionally; absent ones drop around them. @@ -1192,7 +1416,7 @@ mod tests { ColumnRef::Named("job".into()), ColumnRef::Named("group".into()) ], - &s + Scope::unqualified(&s) ), Ok(vec![0]) ); @@ -1205,7 +1429,7 @@ mod tests { // absent label. Keep the strict error. let open = ts_value_schema(); // closed: false assert!(matches!( - resolve_group_keys_promql(&[ColumnRef::Named("job".into())], &open), + resolve_group_keys_promql(&[ColumnRef::Named("job".into())], Scope::unqualified(&open)), Err(ResolveError::NotFound { .. }) )); } diff --git a/crates/frontend-common/src/schema_resolver.rs b/crates/frontend-common/src/schema_resolver.rs index 5462304cc..96be18f6e 100644 --- a/crates/frontend-common/src/schema_resolver.rs +++ b/crates/frontend-common/src/schema_resolver.rs @@ -222,6 +222,7 @@ mod tests { }, predicates: vec![], schema: None, + qualifier: None, } } diff --git a/crates/frontend-common/src/unresolved.rs b/crates/frontend-common/src/unresolved.rs index 52255beeb..0e55023d4 100644 --- a/crates/frontend-common/src/unresolved.rs +++ b/crates/frontend-common/src/unresolved.rs @@ -114,6 +114,10 @@ pub enum UnresolvedOp { /// `Some` for a catalog-backed (SQL) leaf; `None` defers to the /// usage-derived schema resolver. schema: Option, + /// The SQL table or alias qualifying the leaf's columns, which + /// qualified references (`t.col`) above resolve by; `None` for a + /// PromQL leaf. + qualifier: Option, }, Values { rows: Vec>, @@ -125,6 +129,9 @@ pub enum UnresolvedOp { }, Project { cols: Vec, + /// The alias of a derived table (`FROM (SELECT …) t`) qualifying + /// every output column, which qualified references above resolve + /// by; `None` for an ordinary SELECT list. qualifier: Option, child: Rc, }, diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs index 23df96169..4c38d3ce7 100644 --- a/crates/frontend-metricsql/src/lib.rs +++ b/crates/frontend-metricsql/src/lib.rs @@ -153,6 +153,7 @@ impl Lowerer<'_> { .map(|f| UnresolvedPredicate(matcher(f))) .collect(), schema: None, + qualifier: None, }) } diff --git a/crates/frontend-promql/src/promql.rs b/crates/frontend-promql/src/promql.rs index 0313186f8..4750ef0d0 100644 --- a/crates/frontend-promql/src/promql.rs +++ b/crates/frontend-promql/src/promql.rs @@ -1671,6 +1671,7 @@ fn filtered_source(metric: String, matchers: Vec, shift: TimeShift) -> U predicates: matchers.into_iter().map(UnresolvedPredicate).collect(), // Usage-derived (PromQL is schemaless) — the schema resolver fills this in. schema: None, + qualifier: None, }; if shift.is_identity() { scan diff --git a/crates/frontend-sql/src/frequency.rs b/crates/frontend-sql/src/frequency.rs index 755e64514..3c4a044a9 100644 --- a/crates/frontend-sql/src/frequency.rs +++ b/crates/frontend-sql/src/frequency.rs @@ -336,7 +336,7 @@ fn sql_frequency_result( name: &str, value: ScalarExpr, ) -> Option> { - let NonASAPOp::Project { qualifier, .. } = root.non_asap()? else { + let NonASAPOp::Project { .. } = root.non_asap()? else { return None; }; let aggregate = |measure, name: &str| { @@ -380,7 +380,6 @@ fn sql_frequency_result( else_expr: Some(Box::new(value)), }, }], - qualifier: qualifier.clone(), child, })) .ok()?; diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index 3fd2debc7..551632dcd 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -528,7 +528,7 @@ impl<'a> SqlLowerer<'a> { self.scan_source(&table, &table) } - /// A `Scan` over catalog table `table`, with its columns qualified by + /// A `Scan` over catalog table `table`, its columns qualified by /// `qualifier` (the table name, or an alias from a `SubqueryAlias`) so /// `Qualified` column refs resolve to the right side across a join. fn scan_source(&self, table: &str, qualifier: &str) -> Result { @@ -537,24 +537,17 @@ impl<'a> SqlLowerer<'a> { .tables .get(table) .ok_or_else(|| LoweringError::TableNotFound(table.to_string()))?; - let qualified = Schema { - fields: schema - .fields - .iter() - .cloned() - .map(|c| c.with_table(qualifier)) - .collect(), - time_index: schema.time_index, - unique_keys: schema.unique_keys.clone(), - // Catalog-backed: the table's columns are fully declared → closed. - closed: true, - }; Ok(Unresolved::Scan { source: Source::Table { table_ref: table.to_string(), }, predicates: vec![], - schema: Some(qualified), + schema: Some(Schema { + // Catalog-backed: the table's columns are fully declared → closed. + closed: true, + ..schema.clone() + }), + qualifier: Some(qualifier.to_string()), }) } @@ -1929,12 +1922,14 @@ fn filter_or_fold(pred: Scalar, child: Unresolved) -> Unresolved { source, mut predicates, schema, + qualifier, } => { predicates.push(Predicate(pred)); Unresolved::Scan { source, predicates, schema, + qualifier, } } other => Unresolved::Filter { diff --git a/crates/frontend-sql/tests/sql_lowering.rs b/crates/frontend-sql/tests/sql_lowering.rs index 2a83de38e..0c81c48b3 100644 --- a/crates/frontend-sql/tests/sql_lowering.rs +++ b/crates/frontend-sql/tests/sql_lowering.rs @@ -2865,6 +2865,34 @@ async fn lower_err(sql: &str) -> String { .to_string() } +/// Both join sides have a `service` column; the front end binds `h.service` +/// with its own scope while lowering, so the group key is the right side's +/// column, the IR schema carrying no qualifier. +#[tokio::test] +async fn a_qualified_group_key_binds_its_join_side() { + let qe = lower( + "SELECT h.service, SUM(m.bytes) FROM metrics m JOIN hosts h ON m.service = h.service \ + GROUP BY h.service", + ) + .await; + let aggregate = find_aggregate_node(&qe).expect("aggregate"); + let NonASAPOp::Aggregate { + reduction, child, .. + } = op(aggregate) + else { + unreachable!() + }; + let [key] = reduction.expect_reduce().keys() else { + panic!("one key: {reduction:?}") + }; + let metrics_columns = 4; + assert!( + *key >= metrics_columns, + "{key} is a metrics column: {child:#?}" + ); + assert_eq!(child.schema.fields[*key].name, "service"); +} + /// After a join the `Project` drops table qualifiers, so `h.service` must not /// fall back to the first bare `service` (`m.service`): the reference is /// ambiguous and rejected rather than sorting by the wrong column. @@ -2875,7 +2903,7 @@ async fn qualified_order_by_after_a_join_does_not_bind_the_wrong_column() { ORDER BY h.service", ) .await; - assert!(err.contains("several unqualified columns"), "{err}"); + assert!(err.contains("several columns"), "{err}"); } /// `WITHIN GROUP (ORDER BY x DESC)` reads the percentile from the top, as diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index cdd114fb9..c1e3fb575 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -59,9 +59,10 @@ pub use asap_types::ir::materialized_view::{ IngestionRefreshGroup, InstallableDagError, MaterializedView, MaterializedViewInputs, MaterializedViewRefresh, RawDataRetention, RawDataWindow, RefreshWindow, ViewCoverage, }; +pub use asap_types::ir::operator::SourceColumn; pub use asap_types::ir::physical_export::{PhysicalASAPDAG, PhysicalASAPNodeId}; pub use asap_types::ir::properties::summary_coverage::{ - ColumnIdentity, Constraint, IntervalConstraintLimit, SelectionBox, + Constraint, IntervalConstraintLimit, SelectionBox, }; pub use asap_types::ir::properties::ExecutionTiming; pub use asap_types::ir::schema::{SketchAlgorithm, SketchParams, StatisticKind}; diff --git a/crates/stage0-memo/src/lib.rs b/crates/stage0-memo/src/lib.rs index d34901d2b..8d3660786 100644 --- a/crates/stage0-memo/src/lib.rs +++ b/crates/stage0-memo/src/lib.rs @@ -511,7 +511,6 @@ mod tests { alias: Some("v".into()), expr: ScalarExpr::literal_f64(value), }], - qualifier: None, child: scan(), }) } diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index f5c940069..731def21b 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -129,7 +129,6 @@ mod tests { alias: Some("s".into()), expr: ScalarExpr::ScalarSubquery(Rc::clone(&v)), }], - qualifier: None, child: Rc::clone(&v), })) .unwrap(); diff --git a/crates/types/src/ir/operator/agg_intent.rs b/crates/types/src/ir/operator/agg_intent.rs index 799ce1ade..2449391bf 100644 --- a/crates/types/src/ir/operator/agg_intent.rs +++ b/crates/types/src/ir/operator/agg_intent.rs @@ -499,9 +499,11 @@ impl AggIntent { ), // Ranked rows retain the ranked value's type; the enclosing // operator also preserves item identity via schema::topk. - AggIntent::TopK { k, .. } => { - col(&format!("topk_{k}"), input.dtype.clone(), input.nullable) - } + AggIntent::TopK { k, .. } => col( + &crate::ir::schema::topk::topk_output_name(*k, &[]), + input.dtype.clone(), + input.nullable, + ), AggIntent::Cardinality { .. } => col("cardinality", DataType::Int64, false), AggIntent::FrequencyL2 { .. } => col("frequency_l2", DataType::Float64, false), AggIntent::FrequencyEntropy { .. } => { diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 956a85dc6..6aba895d7 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -121,7 +121,7 @@ impl OperatorNode { Ok(Self::with_schema(operator, schema)) } - /// Build a node with caller-supplied output names and qualifiers. For + /// Build a node with caller-supplied output names. For /// either operator category, `validate_structure` requires all other /// schema metadata to agree with derivation; output kind is derived here. pub fn with_schema(operator: Operator, schema: Schema) -> Self { @@ -190,10 +190,9 @@ impl OperatorNode { /// Replace the children and rebuild: each child is replaced by `f(child)` /// and the node is rebuilt over the new children, re-deriving all - /// structural schema metadata. Only names and qualifiers that override - /// the old derived schema are retained, for either operator category. A - /// change in output arity with such overrides needs an explicit new - /// naming assignment. + /// structural schema metadata. Only names that override the old derived + /// schema are retained, for either operator category. A change in output + /// arity with such overrides needs an explicit new naming assignment. /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, @@ -203,22 +202,15 @@ impl OperatorNode { let mut rebuilt = Self::new(self.operator.map_children(f))?; for (i, (derived, retained)) in previous.fields.iter().zip(&self.schema.fields).enumerate() { - let renamed = retained.name != derived.name; - let requalified = retained.table != derived.table; - if !renamed && !requalified { + if retained.name == derived.name { continue; } if rebuilt.schema.fields.len() != previous.fields.len() { return Err(SchemaDerivationError::InvalidScalarSignature( - "output arity changed; reassign explicit output names and qualifiers".into(), + "output arity changed; reassign explicit output names".into(), )); } - if renamed { - rebuilt.schema.fields[i].name = retained.name.clone(); - } - if requalified { - rebuilt.schema.fields[i].table = retained.table.clone(); - } + rebuilt.schema.fields[i].name = retained.name.clone(); } Ok(rebuilt) } @@ -262,7 +254,7 @@ impl OperatorNode { /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field - /// names and qualifiers; all structural metadata must match derivation. + /// names; all structural metadata must match derivation. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { for node in Self::reachable(self) { if node.schema.time_index.is_some_and(|i| { @@ -291,7 +283,6 @@ impl OperatorNode { // so new structural metadata cannot accidentally escape validation. for (field, retained) in derived.fields.iter_mut().zip(&node.schema.fields) { field.name = retained.name.clone(); - field.table = retained.table.clone(); } let agree = derived == node.schema; if !agree { diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index 42800d44b..a3ba976af 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -75,14 +75,7 @@ pub enum NonASAPOp> { /// σ — row-level filter. Output schema = child schema. Filter { pred: Predicate, child: C }, /// π — projection. - Project { - cols: Vec>, - /// Re-qualifies every output column with this table alias (a derived - /// table / inline view). `None` for an ordinary SELECT list. - #[serde(default)] - qualifier: Option, - child: C, - }, + Project { cols: Vec>, child: C }, /// γ + α — grouping + aggregate intents. Aggregate { reduction: Reduction, @@ -300,11 +293,7 @@ impl NonASAPOp { child: f(child), } } - Project { - cols, - qualifier, - child, - } => { + Project { cols, child } => { let cols = cols .iter() .map(|c| ProjectItem { @@ -314,7 +303,6 @@ impl NonASAPOp { .collect(); Project { cols, - qualifier: qualifier.clone(), child: f(child), } } @@ -597,16 +585,11 @@ impl NonASAPOp { .. } => { if let [AggIntent::TopK { k, .. }] = measures.as_slice() { - let name = output_names - .first() - .filter(|name| !name.is_empty()) - .cloned() - .unwrap_or_else(|| format!("topk_{k}")); + let name = crate::ir::schema::topk::topk_output_name(*k, output_names); let cols = crate::ir::schema::topk::ranked_row_projection(child, reduction, &name)?; return NonASAPOp::Project { cols, - qualifier: None, child: child.clone(), } .output_schema(); @@ -695,11 +678,7 @@ impl NonASAPOp { // keeps its field verbatim, so an `ExactAggregate` state column // can pass through a projection unchanged; any other expression // is typed against the input and must read plain values. - Project { - cols, - qualifier, - child, - } => { + Project { cols, child } => { let in_schema = child.schema(); let fields: Vec = cols .iter() @@ -707,9 +686,7 @@ impl NonASAPOp { .map(|(i, item)| { let mut field = match &item.expr { ScalarExpr::Column(id) if in_schema.fields.get(*id).is_some() => { - let mut f = in_schema.fields[*id].clone(); - f.table = None; - f + in_schema.fields[*id].clone() } expr => { let (dtype, nullable) = expr.scalar_type(in_schema)?; @@ -720,10 +697,7 @@ impl NonASAPOp { .alias .clone() .unwrap_or_else(|| default_proj_name(&item.expr, i, in_schema)); - Ok(match qualifier { - Some(q) => field.with_table(q), - None => field, - }) + Ok(field) }) .collect::, SchemaDerivationError>>()?; let time_index = fields.iter().position(|c| c.name == "ts"); @@ -1428,7 +1402,6 @@ mod tests { fn project(cols: Vec, child: NonASAPOp) -> NonASAPOp { NonASAPOp::Project { cols, - qualifier: None, child: node(child), } } diff --git a/crates/types/src/ir/operator/operator_properties.rs b/crates/types/src/ir/operator/operator_properties.rs index 603f4a4aa..fc9a505b3 100644 --- a/crates/types/src/ir/operator/operator_properties.rs +++ b/crates/types/src/ir/operator/operator_properties.rs @@ -180,7 +180,10 @@ pub enum Source { /// A column of a source (a `Scan`'s metric or table), by name: what the data /// workload declares facts about, such as a column's sign -/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). +/// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), +/// the lineage of a column copied from it +/// ([`ColumnLineage::SourceColumn`](crate::stages::ColumnLineage::SourceColumn)), +/// and the key a summary coverage's selection names a column by. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SourceColumn { @@ -523,7 +526,7 @@ mod tests { use super::*; use crate::ir::schema::{DataType, Field, Schema}; - /// A key moves to the input's column of the same name and qualifier; a + /// A key moves to the input's column of the same name; a /// key that is missing or ambiguous there, or a `without` grouping, has no /// partitions over the input. #[test] @@ -544,11 +547,6 @@ mod tests { Reduction::by(vec![0]).partitions_over(&inner, &ambiguous), None ); - let qualified = Schema::new(vec![field("job").with_table("t")]); - assert_eq!( - Reduction::by(vec![0]).partitions_over(&inner, &qualified), - None - ); assert_eq!( Reduction::Reduce(GroupKeys::without(vec![0])).partitions_over(&inner, &input), None diff --git a/crates/types/src/ir/properties/accuracy.rs b/crates/types/src/ir/properties/accuracy.rs index a4eb1b576..4ae1db96b 100644 --- a/crates/types/src/ir/properties/accuracy.rs +++ b/crates/types/src/ir/properties/accuracy.rs @@ -704,7 +704,6 @@ mod tests { else_expr: Some(Box::new(otherwise)), }, }], - qualifier: None, child: joined.clone(), })) }; diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 498aa07fa..24b681ddc 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -10,10 +10,12 @@ //! predicate conjunct moves into `selection` when it can be lifted to the //! top (through `Filter`, a range `TimeRange`, a `TimeShift` and //! direct-column `Project` items) and it is a value set or an interval on -//! one column. The `TimeRange` and `TimeShift`s become one interval on the -//! timestamp column ([`IntervalConstraintLimit`]). Everything else stays in -//! `definition` as a residual, so two states are merged only when they -//! compute the same thing over disjoint rows. Node trees carry no coverage. +//! one column whose lineage is a source column: a selection names the +//! column by that [`SourceColumn`]. The `TimeRange` and `TimeShift`s become +//! one interval on the timestamp column ([`IntervalConstraintLimit`]). +//! Everything else stays in `definition` as a residual, so two states are +//! merged only when they compute the same thing over disjoint rows. Node +//! trees carry no coverage. use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; @@ -25,13 +27,16 @@ use crate::ir::operator::asap::{selected_from, summary_agg_schema, ASAPOp}; use crate::ir::operator::input::{InputOrTemporaryBuild, OperatorInput}; use crate::ir::operator::node::{Operator, OperatorResultKind}; use crate::ir::operator::non_asap::{NonASAPOp, TimeRangeKind}; -use crate::ir::operator::operator_properties::{AtModifier, Reduction}; +use crate::ir::operator::operator_properties::{AtModifier, Reduction, SourceColumn}; use crate::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; use crate::ir::scalar::{Predicate, ScalarExpr}; use crate::ir::schema::summary_type_capability::{MergeInputRule, SummaryTypeCapability}; -use crate::ir::schema::{ColumnId, Schema}; +use crate::ir::schema::ColumnId; use crate::ir::SchemaDerivationError; -use crate::stages::{MemoGroupAsInput, MemoGroupId}; +use crate::stages::column_properties::{ + columns_of_group, output_column_properties, ColumnProperties, +}; +use crate::stages::{ColumnLineage, MemoGroupAsInput, MemoGroupId}; /// What a summary state computes, with every lifted selection removed. #[derive(Debug, Clone, PartialEq)] @@ -80,24 +85,24 @@ impl SummaryCoverage { #[serde(deny_unknown_fields)] pub struct SelectionBox { #[serde(with = "columns_as_list")] - pub(crate) columns: BTreeMap, + pub(crate) columns: BTreeMap, } /// `SelectionBox::columns` as a list of `{column, constraint}`. mod columns_as_list { - use super::{ColumnIdentity, Constraint}; + use super::{Constraint, SourceColumn}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Entry { - column: ColumnIdentity, + column: SourceColumn, constraint: Constraint, } pub(super) fn serialize( - columns: &BTreeMap, + columns: &BTreeMap, serializer: S, ) -> Result { let entries: Vec = columns @@ -112,7 +117,7 @@ mod columns_as_list { pub(super) fn deserialize<'de, D: Deserializer<'de>>( deserializer: D, - ) -> Result, D::Error> { + ) -> Result, D::Error> { let entries = Vec::::deserialize(deserializer)?; Ok(entries .into_iter() @@ -150,26 +155,6 @@ impl From> for Bound { } } -/// A column of the definition child's output, by `(table, name)`. A -/// `Project` below renames its columns and sets their table to its -/// qualifier (none by default). -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ColumnIdentity { - pub(crate) table: Option, - pub(crate) name: String, -} - -#[cfg(any(test, feature = "test-support"))] -impl ColumnIdentity { - pub fn new(table: Option, name: impl Into) -> Self { - Self { - table, - name: name.into(), - } - } -} - /// A constraint on one column of a selection. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -445,7 +430,7 @@ fn selected_build<'m>( /// rows with timestamp column `time_column`. fn of_merge( inputs: &[SummaryCoverage], - time_column: Option<&ColumnIdentity>, + time_column: Option<&SourceColumn>, ) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; if inputs @@ -518,9 +503,11 @@ fn regrouped( if !SummaryTypeCapability::merges_groups(family) { return Err(CoverageError::SummaryTypeCannotRegroup); } - let state = first.schema(); - let rows = build_input_schema(first).ok_or(CoverageError::NotSummary)?; - // The build's input column each key of the state names. + let state = first.group_properties(); + let rows = build_input(first).ok_or(CoverageError::NotSummary)?; + let rows_lineage = &rows.group_properties().column_lineage; + // The build's input column each key of the state names: the column of + // the key's lineage. let mapped = keys .iter() .map(|&key| { @@ -532,20 +519,19 @@ fn regrouped( }; column .filter(|&column| { - let (Some(at_state), Some(at_input)) = - (state.fields.get(key), rows.fields.get(column)) - else { - return false; - }; - at_state.is_plain() - && at_state.name == at_input.name - && at_state.table == at_input.table + state + .schema + .fields + .get(key) + .is_some_and(|field| field.is_plain()) + && state.column_lineage.get(key).is_some() + && state.column_lineage.get(key) == rows_lineage.get(column) }) .ok_or(CoverageError::NotAGroupKey { key }) }) .collect::, _>>()?; let reduction = Reduction::by(mapped); - summary_agg_schema(&rows, family, &reduction).map_err(|error| { + summary_agg_schema(rows.schema(), family, &reduction).map_err(|error| { CoverageError::RegroupedBuild { reason: error.to_string(), } @@ -565,30 +551,31 @@ fn regrouped( }) } -/// The schema of the rows the build of summary state `state` reads (its -/// `SummaryAgg`'s input; one build over a chain's window reads its range of -/// the chain's input, which has the input's schema). `None` when `state` -/// is no summary state. -fn build_input_schema(state: &I) -> Option { +/// The input group of the rows the build of summary state `state` reads +/// (its `SummaryAgg`'s input; one build over a chain's window reads its +/// range of the chain's input, which has the input's schema and lineage). +/// `None` when `state` is no summary state. +fn build_input<'m>(state: &MemoGroupAsInput<'m>) -> Option> { if let Some(states) = selected_from(state) { - return build_input_schema(&states); + return build_input(&states); } match state.operator() { Operator::ASAP( ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }, - ) => Some(child.schema().clone()), - Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => { - build_input_schema(children.first()?) - } + ) => Some(child), + Operator::ASAP(ASAPOp::SummaryMerge { children, .. }) => build_input(children.first()?), _ => None, } } /// The timestamp column of the rows the build of summary state `state` -/// reads ([`build_input_schema`]), as a selection names it. -fn time_column_of_build_input(state: &I) -> Option { - let rows = build_input_schema(state)?; - column_identity(&rows, rows.time_index?) +/// reads ([`build_input`]), as a selection names it. +fn time_column_of_build_input(state: &MemoGroupAsInput<'_>) -> Option { + let rows = build_input(state)?; + source_column( + &rows.group_properties().column_lineage, + rows.schema().time_index?, + ) } /// The coverage of `build`, a `SummaryAgg` over memo groups (or over an @@ -651,17 +638,17 @@ fn of_aggregate( let InputOrTemporaryBuild::Input(input_group) = &base else { unreachable!("a rule builds only liftable operators over memo groups"); }; - let top = child.schema(); + let top = column_lineage(child); // Top-down: decide which conjuncts are lifted. `kept[d]` is the residual // of `chain[d]` when it is a `Filter`. let mut lifted = BTreeMap::new(); - let agg_filter = filter.and_then(|pred| residual(pred, top, &[], &mut lifted)); + let agg_filter = filter.and_then(|pred| residual(pred, &top, &[], &mut lifted)); let kept: Vec>>> = chain .iter() .enumerate() .map(|(depth, link)| match link { Operator::NonASAP(NonASAPOp::Filter { pred, .. }) => { - residual(pred, top, &chain[..depth], &mut lifted) + residual(pred, &top, &chain[..depth], &mut lifted) } _ => None, }) @@ -678,7 +665,7 @@ fn of_aggregate( let lifted_above = lifted.clone(); let kept = predicates .iter() - .filter_map(|pred| residual(pred, top, &chain, &mut lifted)) + .filter_map(|pred| residual(pred, &top, &chain, &mut lifted)) .collect(); (lifted != lifted_above).then_some(NonASAPOp::Scan { source, @@ -698,7 +685,7 @@ fn of_aggregate( .position(|link| matches!(link, Operator::NonASAP(NonASAPOp::TimeRange { .. })))?; let column = chain[depth].children()[0].schema().time_index?; let column = column_at_top(column, &chain[..=depth])?; - column_identity(top, column)?; + source_column(&top, column)?; let combined = match lifted.get(&column) { Some(existing) => existing.intersect(&constraint)?, None => constraint, @@ -735,7 +722,7 @@ fn of_aggregate( let selection = SelectionBox { columns: lifted .into_iter() - .filter_map(|(column, constraint)| Some((column_identity(top, column)?, constraint))) + .filter_map(|(column, constraint)| Some((source_column(&top, column)?, constraint))) .collect(), }; Ok(SummaryCoverage { @@ -748,13 +735,13 @@ fn of_aggregate( } /// Lift what `pred`'s conjuncts can into `lifted` (keyed by the column of -/// `top`, the agg child's schema) and return the rest, each scalar subquery -/// naming the group it reads. `above` -/// are the operators between the predicate and the `SummaryAgg`, top to -/// bottom. A column [`column_identity`] cannot name stays. +/// the agg child, whose lineage is `top`) and return the rest, each scalar +/// subquery naming the group it reads. `above` are the operators between +/// the predicate and the `SummaryAgg`, top to bottom. A conjunct on a +/// column [`source_column`] cannot name stays. fn residual( pred: &Predicate>, - top: &Schema, + top: &[ColumnLineage], above: &[Operator>], lifted: &mut BTreeMap, ) -> Option>> { @@ -762,7 +749,7 @@ fn residual( for conjunct in pred.0.conjuncts() { let lift = constraint_of(conjunct).and_then(|(column, constraint)| { let column = column_at_top(column, above)?; - column_identity(top, column)?; + source_column(top, column)?; let combined = match lifted.get(&column) { Some(existing) => existing.intersect(&constraint)?, None => constraint, @@ -821,19 +808,44 @@ fn column_at_top(mut column: ColumnId, above: &[Operator]) -> Option Option { - let field = schema.fields.get(column)?; - let namesakes = schema - .fields +/// The source column a selection names column `column` by: its lineage, +/// one of `lineage` (a group's, per column). `None` when the column is +/// `Computed`, or another column has the same lineage (`Project [a AS k, +/// a AS k2]`), so a selection cannot name it apart. +fn source_column(lineage: &[ColumnLineage], column: ColumnId) -> Option { + let ColumnLineage::SourceColumn(source_column) = lineage.get(column)? else { + return None; + }; + let namesakes = lineage .iter() - .filter(|f| f.name == field.name && f.table == field.table) + .filter(|other| *other == &lineage[column]) .count(); - (namesakes == 1).then(|| ColumnIdentity { - table: field.table.clone(), - name: field.name.clone(), - }) + (namesakes == 1).then(|| source_column.clone()) +} + +/// The column lineage of `input`: a memo group's stored lineage, or that of +/// a rule's temporary build, derived from its inputs' ([`input_columns`]). +fn column_lineage(input: &GroupInput<'_>) -> Vec { + input_columns(input) + .into_iter() + .map(|column| column.lineage) + .collect() +} + +/// The columns of `input`: a memo group's stored ones, or those a rule's +/// temporary build derives from its inputs' by the rules that derive a +/// group's. +fn input_columns(input: &GroupInput<'_>) -> Vec { + match input { + InputOrTemporaryBuild::Input(group) => columns_of_group(group), + // A temporary build is never a `Scan`, so it reads no declared sign. + InputOrTemporaryBuild::TemporaryBuild(_) => output_column_properties( + &input.operator(), + input.schema(), + &BTreeMap::new(), + &input_columns, + ), + } } /// The interval on the timestamp column that `links` (top down, the @@ -1213,12 +1225,12 @@ impl Constraint { #[cfg(any(test, feature = "test-support"))] impl SelectionBox { - pub fn new(columns: BTreeMap) -> Self { + pub fn new(columns: BTreeMap) -> Self { Self { columns } } /// The per-column constraints. - pub fn columns(&self) -> &BTreeMap { + pub fn columns(&self) -> &BTreeMap { &self.columns } } @@ -1360,7 +1372,10 @@ mod tests { }; assert!(!last_5m.disjoint(&at)); assert_eq!(last_5m.intersect(&at), None); - let column = ColumnIdentity::new(None, "ts"); + let column = SourceColumn { + source: "m".into(), + column: "ts".into(), + }; let one = |constraint: Constraint| SelectionBox { columns: BTreeMap::from([(column.clone(), constraint)]), }; diff --git a/crates/types/src/ir/scalar/column_resolution.rs b/crates/types/src/ir/scalar/column_resolution.rs index 8822c6e85..e5bfc9d32 100644 --- a/crates/types/src/ir/scalar/column_resolution.rs +++ b/crates/types/src/ir/scalar/column_resolution.rs @@ -2,8 +2,9 @@ //! //! Front ends emit `ColumnRef` (name-based, optionally table-qualified); the //! IR uses positional [`ColumnId`] resolved against a per-node [`Schema`]. -//! These helpers turn name-based refs into positional ids, qualifier-aware. -//! Front-end name resolution (`asap_frontend_common::resolve`) calls them. +//! These helpers turn name-based refs into positional ids. A schema carries +//! no qualifier, so front-end name resolution (`asap_frontend_common::resolve`) +//! binds a qualified name with its own scope before it calls them. use thiserror::Error; @@ -22,38 +23,24 @@ pub enum ResolveError { NoSampleValue { available: Vec }, #[error("ColumnRef::Wildcard cannot be resolved to a single ColumnId")] WildcardNotPositional, - #[error("column `{name}` matches several unqualified columns")] + #[error("column `{name}` matches several columns")] Ambiguous { name: String }, } /// Resolve a single [`ColumnRef`] to a positional [`ColumnId`]. pub fn resolve_column_ref(col: &ColumnRef, schema: &Schema) -> Result { match col { - // A bare name must be unique: after a join `k` could be `a.k` or `b.k`. - ColumnRef::Named(name) => match schema.fields.iter().filter(|c| &c.name == name).count() { - 0 => Err(ResolveError::NotFound { - name: name.clone(), - available: schema.fields.iter().map(|c| c.name.clone()).collect(), - }), - 1 => Ok(schema.column_id(name).expect("one field has this name")), - _ => Err(ResolveError::Ambiguous { name: name.clone() }), - }, - // Prefer the (table, name) match; fall back to the bare name for - // schemas whose columns carry no qualifier, but only when that name - // is unique: after a join `b.k` must not bind to `a.k`. - ColumnRef::Qualified { table, name } => { - if let Some(id) = schema.column_id_qualified(table, name) { - return Ok(id); - } + // A name must be unique: after a join `k` could be either side's. + // A schema carries no qualifier; the front end binds a qualified + // name with its own scope before it calls this. + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. } => { match schema.fields.iter().filter(|c| &c.name == name).count() { 0 => Err(ResolveError::NotFound { - name: format!("{table}.{name}"), + name: name.clone(), available: schema.fields.iter().map(|c| c.name.clone()).collect(), }), 1 => Ok(schema.column_id(name).expect("one field has this name")), - _ => Err(ResolveError::Ambiguous { - name: format!("{table}.{name}"), - }), + _ => Err(ResolveError::Ambiguous { name: name.clone() }), } } ColumnRef::SampleValue => schema diff --git a/crates/types/src/ir/scalar/expr_ir.rs b/crates/types/src/ir/scalar/expr_ir.rs index 3ee1dd56d..f66493338 100644 --- a/crates/types/src/ir/scalar/expr_ir.rs +++ b/crates/types/src/ir/scalar/expr_ir.rs @@ -18,9 +18,10 @@ use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColumnRef { Named(String), - /// Table-qualified reference (`t.col` / `alias.col`). Resolved by - /// `(table, name)` so a column name shared across a join (`a.k` vs `b.k`) - /// binds to the correct side. + /// Table-qualified reference (`t.col` / `alias.col`), as a front end + /// emits it. The front end binds it by `(table, name)` with its own + /// scope while it lowers, so a column name shared across a join (`a.k` + /// vs `b.k`) binds to the correct side; a schema carries no qualifier. Qualified { table: String, name: String, diff --git a/crates/types/src/ir/scalar/mod.rs b/crates/types/src/ir/scalar/mod.rs index 01ef2984a..8d453a7bd 100644 --- a/crates/types/src/ir/scalar/mod.rs +++ b/crates/types/src/ir/scalar/mod.rs @@ -905,7 +905,6 @@ mod tests { alias: Some("result".into()), expr, }], - qualifier: None, child: OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { table_ref: "t".into(), diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index 26700e30a..a7457a8c1 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -42,8 +42,10 @@ use serde::{Deserialize, Serialize}; /// joins, and does not own data or prescribe a row/column-oriented storage layout. pub type ColumnId = usize; -/// One field of a [`Schema`]: `name + dtype + nullable`, plus an optional -/// table qualifier. The struct describes a column and holds none of its data. +/// One field of a [`Schema`]: `name + dtype + nullable`. The struct describes +/// a column and holds none of its data. Where a column's values come from is +/// its memo group's column lineage, not part of the field; a front end binds +/// qualified names (`t.col`) with its own scope while it lowers. /// /// `T` is the type vocabulary: [`FieldDataType`] on an operator edge (the /// default, and what [`Schema::fields`] holds), plain [`DataType`] for the @@ -59,46 +61,25 @@ pub struct Field { /// Whether NULL values are allowed in this field. PromQL value /// columns are non-nullable; SQL columns inherit their DDL nullability. pub nullable: bool, - /// Optional table/alias qualifier (SQL `t.col` / `t AS a` → `a`). Travels - /// with the field through joins so a `ColumnRef::Qualified` can pick the - /// right side when both carry the same `name`. `None` for PromQL labels and - /// unqualified columns. - #[serde(default)] - pub table: Option, } impl Field { - /// An unqualified field (`table = None`). pub fn new(name: impl Into, dtype: T, nullable: bool) -> Self { Self { name: name.into(), dtype, nullable, - table: None, } } - /// This field re-qualified under `table` (e.g. by a `SubqueryAlias`). - pub fn with_table(mut self, table: impl Into) -> Self { - self.table = Some(table.into()); - self - } - - /// A reference to this field: qualified by its table when it has one. + /// A reference to this field by its name. pub fn column_ref(&self) -> crate::ir::scalar::ColumnRef { - use crate::ir::scalar::ColumnRef; - match &self.table { - Some(table) => ColumnRef::Qualified { - table: table.clone(), - name: self.name.clone(), - }, - None => ColumnRef::Named(self.name.clone()), - } + crate::ir::scalar::ColumnRef::Named(self.name.clone()) } } impl Field { - /// An unqualified field carrying an ordinary readable value. + /// A field carrying an ordinary readable value. pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self { Self::new(name, FieldDataType::Plain(dtype), nullable) } @@ -135,7 +116,6 @@ impl From> for Field { name: field.name, dtype: FieldDataType::Plain(field.dtype), nullable: field.nullable, - table: field.table, } } } @@ -246,7 +226,7 @@ pub enum DataType { /// catalog tables. Date, /// Variable-length sequence. The existing column contract preserves the - /// element field name, type, and nullability. Nested fields are unqualified. + /// element field name, type, and nullability. List { element: Box> }, /// Ordered named fields, including each field's independent nullability. Struct { fields: Vec> }, @@ -311,7 +291,6 @@ impl Schema { field.name == PROMQL_SERIES_IDENTITY && field.dtype == DataType::Utf8 && !field.nullable - && field.table.is_none() }) } @@ -364,15 +343,6 @@ impl Schema { self.fields.iter().position(|c| c.name == name) } - /// Look up a field by `(table, name)` qualifier — disambiguates columns - /// that share a `name` across a join (`a.k` vs `b.k`). `None` if no field - /// has both that qualifier and name. - pub(crate) fn column_id_qualified(&self, table: &str, name: &str) -> Option { - self.fields - .iter() - .position(|c| c.name == name && c.table.as_deref() == Some(table)) - } - /// Whether this schema has *any* provable unique key. #[cfg(any(test, feature = "test-support"))] pub fn has_unique_key(&self) -> bool { @@ -453,12 +423,4 @@ mod tests { fn schema_without_fields_is_rejected() { assert!(serde_json::from_str::(r#"{"closed":true}"#).is_err()); } - - #[test] - fn qualified_field_serde_roundtrip() { - let c = col("service", DataType::Utf8).with_table("hosts"); - let back: Field = serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap(); - assert_eq!(back, c); - assert_eq!(back.table.as_deref(), Some("hosts")); - } } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index adc4b41d5..de89631f5 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -55,6 +55,16 @@ pub fn ranked_column(child: &I) -> Result String { + output_names + .first() + .filter(|name| !name.is_empty()) + .cloned() + .unwrap_or_else(|| format!("topk_{k}")) +} + /// Ordered projection of ranked rows: partition keys, ranked value, then /// item identity. TopK retains rows; partition keys alone are not unique. pub(crate) fn ranked_row_projection( diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 4f530d8d8..e80abf027 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -6,7 +6,7 @@ use crate::ir::operator::input::reads_only_time_series; use std::collections::{BTreeMap, HashMap, HashSet}; -use super::column_sign::output_column_sign; +use super::column_properties::{columns_of_group, output_column_properties}; use super::hash_consing::{expression_hash, same_expression, same_value}; use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId, ValueSign}; use crate::ir::operator::asap::summary_type_of_operator; @@ -311,8 +311,18 @@ fn derive_group_properties( } Err(_) => None, }; + let (column_lineage, column_sign) = output_column_properties( + &operator, + &schema, + &memo.declared_column_signs, + &columns_of_group, + ) + .into_iter() + .map(|column| (column.lineage, column.sign)) + .unzip(); Ok(GroupProperties { - column_sign: output_column_sign(&operator, &schema, &memo.declared_column_signs), + column_lineage, + column_sign, depends_on_evaluation_time: depends_on_evaluation_time(&operator), schema, result_kind: operator.output_kind(), @@ -1052,7 +1062,6 @@ mod memo_tests { alias: None, expr: crate::ir::ScalarExpr::Column(0), }], - qualifier: None, child: input, }) } diff --git a/crates/types/src/stages/column_properties.rs b/crates/types/src/stages/column_properties.rs new file mode 100644 index 000000000..7669e1b5c --- /dev/null +++ b/crates/types/src/stages/column_properties.rs @@ -0,0 +1,569 @@ +//! Column lineage and value sign: two per-column group properties +//! (concepts doc, section 3, "Column lineage" and "Value sign"), derived +//! together, once per group, from the input groups' and, for a `Scan`, from +//! its source's columns and the signs the data workload declares for them. +//! One per-operator column layout says which input column each output +//! column copies (lineage and sign kept) and which it computes (lineage +//! `Computed`, the sign by the operator's rule). + +use std::collections::BTreeMap; + +use super::{ColumnLineage, MemoGroupAsInput, ValueSign}; +use crate::ir::operator::asap::{summary_agg_schema, summary_build}; +use crate::ir::operator::input::OperatorInput; +use crate::ir::operator::operator_properties::{ + BinaryOpKind, GroupSide, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, +}; +use crate::ir::operator::AggIntent; +use crate::ir::scalar::expr_ir::ScalarValue; +use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; +use crate::ir::schema::sketch::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; +use crate::ir::schema::topk::{ranked_row_projection, topk_output_name}; +use crate::ir::schema::{FieldDataType, Schema}; +use crate::ir::{ASAPOp, NonASAPOp, Operator}; +use crate::workload::bounds::source_name; + +use ValueSign::{NonNegative, NonPositive, Unknown}; + +/// One output column's lineage and sign. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ColumnProperties { + pub(crate) lineage: ColumnLineage, + pub(crate) sign: ValueSign, +} + +impl ColumnProperties { + /// A column the operator computes, with values of `sign`. + fn computed(sign: ValueSign) -> Self { + Self { + lineage: ColumnLineage::Computed, + sign, + } + } + + /// What two inputs' columns at one position agree on: each property + /// where they are equal, else `Computed` / `Unknown`. + fn agreed_with(self, other: &Self) -> Self { + Self { + lineage: if self.lineage == other.lineage { + self.lineage + } else { + ColumnLineage::Computed + }, + sign: agree(self.sign, other.sign), + } + } +} + +/// The columns of a memo group, read from its `group_properties`. +pub(crate) fn columns_of_group(group: &MemoGroupAsInput<'_>) -> Vec { + let properties = group.group_properties(); + properties + .column_lineage + .iter() + .zip(&properties.column_sign) + .map(|(lineage, &sign)| ColumnProperties { + lineage: lineage.clone(), + sign, + }) + .collect() +} + +/// The lineage and sign of each column of `output`, the output schema of +/// `operator`, from its inputs' columns (`columns_of`) and, for a `Scan`, +/// the `declared` signs of its source's columns. +pub(crate) fn output_column_properties( + operator: &Operator, + output: &Schema, + declared: &BTreeMap, + columns_of: &dyn Fn(&I) -> Vec, +) -> Vec { + let columns = match operator { + Operator::NonASAP(op) => non_asap_columns(op, output, declared, columns_of), + Operator::ASAP(op) => asap_columns(op, output, columns_of), + }; + debug_assert_eq!( + columns.len(), + output.fields.len(), + "one lineage and sign per output column of {}", + operator.kind_name() + ); + columns +} + +fn non_asap_columns( + op: &NonASAPOp, + output: &Schema, + declared: &BTreeMap, + columns_of: &dyn Fn(&I) -> Vec, +) -> Vec { + use NonASAPOp::*; + let unknown = || vec![ColumnProperties::computed(Unknown); output.fields.len()]; + match op { + Scan { source, schema, .. } => schema + .fields + .iter() + .map(|field| { + let column = SourceColumn { + source: source_name(source).to_owned(), + column: field.name.clone(), + }; + ColumnProperties { + sign: declared.get(&column).copied().unwrap_or(Unknown), + lineage: ColumnLineage::SourceColumn(column), + } + }) + .collect(), + Values { .. } | PromqlVectorFromScalar(_) => unknown(), + Filter { child, .. } + | Sort { child, .. } + | Limit { child, .. } + | Dedup { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | PromqlSubquery { child, .. } + | PromqlSeriesSample { child, .. } + | PromqlInfoEnrich { child, .. } => columns_of(child), + // The relabeled column is written; every other column is copied. + PromqlRelabel { dst, child, .. } => { + let mut columns = columns_of(child); + let written = ColumnProperties::computed(Unknown); + match child.schema().column_id(dst) { + Some(column) => columns[column] = written, + None => columns.push(written), + } + columns + } + Project { cols, child } => project_columns(cols, &columns_of(child)), + Aggregate { + reduction, + measures, + output_names, + child, + .. + } => { + if let [AggIntent::TopK { k }] = measures.as_slice() { + // The ranked rows, projected: each column copied. + let name = topk_output_name(*k, output_names); + return match ranked_row_projection(child, reduction, &name) { + Ok(cols) => project_columns(&cols, &columns_of(child)), + Err(_) => unknown(), + }; + } + aggregate_columns(child, reduction, measures, output, columns_of) + } + Join { + kind, left, right, .. + } => { + let mut columns = columns_of(left); + if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { + columns.extend(columns_of(right)); + } + columns + } + SetOp { left, right, .. } => agreed(&[columns_of(left), columns_of(right)]), + Concat { children } => agreed(&children.iter().map(columns_of).collect::>()), + SQLWindowFunc { child, .. } => { + let mut columns = columns_of(child); + columns.push(ColumnProperties::computed(Unknown)); + columns + } + // The left side's columns, its value computed per the operator, then + // the right side's labels the match adds. A label that may come from + // either side (`or`, `group_right`) keeps what both sides agree on. + BinaryOp { + operator, lhs, rhs, .. + } => { + let left = columns_of(lhs); + let right = columns_of(rhs); + let mut columns = left.clone(); + let right_named = |name: &str| { + rhs.schema() + .column_id(name) + .map_or(ColumnProperties::computed(Unknown), |column| { + right[column].clone() + }) + }; + let right_rows = matches!(operator.kind, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)) + || operator + .vector_match + .as_ref() + .and_then(|m| m.grouping.as_ref()) + .is_some_and(|grouping| grouping.side == GroupSide::Right); + if right_rows { + for (column, field) in lhs.schema().fields.iter().enumerate() { + columns[column] = columns[column] + .clone() + .agreed_with(&right_named(&field.name)); + } + } + if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { + let left_sign = left[value].sign; + let right_sign = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) + .map_or(Unknown, |column| right[column].sign); + columns[value] = ColumnProperties::computed(match &operator.kind { + // A filter keeps the left side's values. + BinaryOpKind::Compare(_) + | BinaryOpKind::Set( + PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, + ) => left_sign, + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(left_sign, right_sign), + BinaryOpKind::CompareBool(_) => NonNegative, + BinaryOpKind::Arithmetic(_) => Unknown, + }); + } + let added = output.fields.get(columns.len()..).unwrap_or_default(); + columns.extend(added.iter().map(|field| right_named(&field.name))); + columns + } + } +} + +fn asap_columns( + op: &ASAPOp, + output: &Schema, + columns_of: &dyn Fn(&I) -> Vec, +) -> Vec { + use ASAPOp::*; + let unknown = || vec![ColumnProperties::computed(Unknown); output.fields.len()]; + match op { + SummaryAgg { + child, + input, + reduction, + .. + } => state_columns(child, input, reduction, output, columns_of), + // The build's layout without its own time column, after the + // computed `window_end` (column 0). + SummaryTumblingWindowChain { + child, + summary_type, + input, + reduction, + .. + } => match summary_agg_schema(child.schema(), summary_type, reduction) { + Ok(build) => { + let mut columns = state_columns(child, input, reduction, &build, columns_of); + if let Some(time) = build.time_index { + columns.remove(time); + } + columns.insert(0, ColumnProperties::computed(Unknown)); + columns + } + Err(_) => unknown(), + }, + SummaryMerge { children, group_by } => { + // The windows a `TimeRange` over a chain selects merge into the + // chain's build over that range. + if let (None, [child]) = (group_by, children.as_slice()) { + if let Some(SummaryTumblingWindowChain { + child, + input, + reduction, + .. + }) = summary_build(child) + { + return state_columns(&child, &input, &reduction, output, columns_of); + } + } + let merged = agreed(&children.iter().map(columns_of).collect::>()); + match group_by { + None => merged, + Some(keys) => { + let state = children[0] + .schema() + .fields + .iter() + .position(|field| !field.is_plain()); + keys.iter() + .chain(state.as_ref()) + .map(|&column| { + merged + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)) + }) + .collect() + } + } + } + SummaryEstimate { + summary_input, + query, + } => readout_columns(summary_input, output, columns_of, |state| match query { + SketchStatistic::Cardinality + | SketchStatistic::FrequencyL2 + | SketchStatistic::FrequencyEntropy => NonNegative, + SketchStatistic::Quantile { .. } + | SketchStatistic::PointCount { .. } + | SketchStatistic::TopK { .. } => state, + }), + FinalizeExactAccumulator { child } => { + let counts = child.schema().fields.iter().any(|field| { + matches!( + field.dtype, + FieldDataType::ExactAggregate(ExactKind::Count, _) + ) + }); + readout_columns(child, output, columns_of, |state| { + if counts { + NonNegative + } else { + state + } + }) + } + MaintainPopulation { child, .. } => columns_of(child), + EvaluatePopulation { child, .. } if output == child.schema() => columns_of(child), + EvaluatePopulation { .. } + | SummarySubtract { .. } + | SummaryDelete { .. } + | SummaryJoin { .. } + | Extension { .. } => unknown(), + } +} + +/// A column item, renamed or not, copies its input column; an expression +/// item is computed, with [`expression_sign`]'s sign. +fn project_columns( + cols: &[ProjectItem], + input: &[ColumnProperties], +) -> Vec { + let signs: Vec = input.iter().map(|column| column.sign).collect(); + cols.iter() + .map(|item| match item.expr { + ScalarExpr::Column(column) => input + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)), + _ => ColumnProperties::computed(expression_sign(&item.expr, &signs)), + }) + .collect() +} + +/// `abs` is non-negative, a negation flips its operand's sign, a numeric +/// cast keeps it and a numeric literal has its own; any other expression is +/// `Unknown`. +fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { + match expr { + ScalarExpr::Column(column) => input.get(*column).copied().unwrap_or(Unknown), + ScalarExpr::Literal(ScalarValue::Float64(value)) => sign_of_number(*value), + ScalarExpr::Literal(ScalarValue::Int64(value)) => sign_of_number(*value as f64), + ScalarExpr::Negative { expr, .. } => flipped(expression_sign(expr, input)), + ScalarExpr::Cast { expr, .. } => expression_sign(expr, input), + // PromQL's `abs(v)` lowers to `promql_abs`, SQL's to `abs`. + ScalarExpr::FunctionCall { name, .. } + if name.eq_ignore_ascii_case("abs") || name == "promql_abs" => + { + NonNegative + } + _ => Unknown, + } +} + +/// An aggregate's, in `aggregate_output_schema`'s layout: a per-entity +/// reduction keeps its input's columns in place, its value replaced by the +/// measure's; a `by` reduction lists its keys, then its measures; a +/// `without` reduction the input's other label columns, then its measures. +/// Keys are copied; measures are computed. +fn aggregate_columns( + child: &I, + reduction: &Reduction, + measures: &[AggIntent], + output: &Schema, + columns_of: &dyn Fn(&I) -> Vec, +) -> Vec { + let input = columns_of(child); + let schema = child.schema(); + let sample_value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); + let value_of = |measure: &AggIntent| measure.input_cols().first().copied().or(sample_value); + let input_sign = |column: Option| { + column + .and_then(|column| input.get(column)) + .map_or(Unknown, |column| column.sign) + }; + let keys: Vec = match reduction { + Reduction::PerEntity => { + let mut columns = input.clone(); + if let Some(measure) = measures.first() { + if let Some(value) = value_of(measure).filter(|&value| value < columns.len()) { + columns[value] = + ColumnProperties::computed(measure_sign(measure, input[value].sign)); + } + } + return columns; + } + Reduction::Reduce(by) if by.is_without() => (0..schema.fields.len()) + .filter(|&column| { + schema.time_index != Some(column) + && sample_value != Some(column) + && !by.keys().contains(&column) + }) + .map(|column| input[column].clone()) + .collect(), + Reduction::Reduce(by) => by + .keys() + .iter() + .map(|&key| { + input + .get(key) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)) + }) + .collect(), + }; + let measured: Vec = measures + .iter() + .flat_map(|measure| { + let sign = measure_sign(measure, input_sign(value_of(measure))); + // `count_values` and `absent` add label columns before their value. + let labels = match measure { + AggIntent::CountValues { label } => { + usize::from(!output.fields[..keys.len()].iter().any(|f| f.name == *label)) + } + AggIntent::Absent { labels } => labels.len(), + _ => 0, + }; + std::iter::repeat_n(ColumnProperties::computed(Unknown), labels) + .chain([ColumnProperties::computed(sign)]) + }) + .collect(); + keys.into_iter().chain(measured).collect() +} + +/// The sign of `measure`'s value over an input value of sign `input`. +fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { + use AggIntent::*; + match measure { + Count + | Cardinality { .. } + | CountValues { .. } + | Changes + | Resets + | Group + | Absent { .. } + | AbsentOverTime + | PresentOverTime + | StdDev { .. } + | Variance { .. } + | FrequencyL2 { .. } + | FrequencyEntropy { .. } + | Math(crate::ir::operator::MathFunc::Abs) => NonNegative, + Sum { .. } + | Avg { .. } + | Min { .. } + | Max { .. } + | Quantile { .. } + | LastOverTime + | FirstOverTime => input, + // A counter's rate and increase are non-negative; over a value of + // another sign a decrease reads as a reset, so no sign is known. + Rate | IRate | Increase if input == NonNegative => NonNegative, + _ => Unknown, + } +} + +/// A summary state's, in `state_schema`'s layout (an aggregate of one +/// measure): its key columns copy their input's, and its state column is +/// computed and records the sign of the values it is updated with. +fn state_columns( + child: &I, + update: &SummaryUpdate, + reduction: &Reduction, + state_schema: &Schema, + columns_of: &dyn Fn(&I) -> Vec, +) -> Vec { + let mut columns = aggregate_columns( + child, + reduction, + &[AggIntent::Sum { col: None }], + state_schema, + columns_of, + ); + let updated_with = match &update.weight { + SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) + .ok() + .and_then(|column| columns_of(child).get(column).map(|column| column.sign)) + .unwrap_or(Unknown), + SummaryInputExpr::Constant(value) => sign_of_number(*value), + SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, + }; + if let Some(state) = state_schema.fields.iter().position(|f| !f.is_plain()) { + columns[state] = ColumnProperties::computed(updated_with); + } + columns +} + +/// A readout of summary state: the state's plain columns are copied in +/// place, and the value read out is computed, with `value(the state's +/// recorded sign)`. All computed with an unknown sign when the output does +/// not keep the state's layout (a top-k's rows). +fn readout_columns( + state: &I, + output: &Schema, + columns_of: &dyn Fn(&I) -> Vec, + value: impl Fn(ValueSign) -> ValueSign, +) -> Vec { + let input = state.schema(); + if input.fields.len() != output.fields.len() { + return vec![ColumnProperties::computed(Unknown); output.fields.len()]; + } + columns_of(state) + .into_iter() + .zip(&input.fields) + .map(|(column, field)| { + if field.is_plain() { + column + } else { + ColumnProperties::computed(value(column.sign)) + } + }) + .collect() +} + +/// Per column, what all inputs agree on ([`ColumnProperties::agreed_with`]). +fn agreed(inputs: &[Vec]) -> Vec { + let Some((first, rest)) = inputs.split_first() else { + return Vec::new(); + }; + first + .iter() + .enumerate() + .map(|(column, properties)| { + rest.iter().fold(properties.clone(), |agreed, other| { + agreed.agreed_with( + &other + .get(column) + .cloned() + .unwrap_or(ColumnProperties::computed(Unknown)), + ) + }) + }) + .collect() +} + +fn agree(left: ValueSign, right: ValueSign) -> ValueSign { + if left == right { + left + } else { + Unknown + } +} + +fn flipped(sign: ValueSign) -> ValueSign { + match sign { + NonNegative => NonPositive, + NonPositive => NonNegative, + Unknown => Unknown, + } +} + +fn sign_of_number(value: f64) -> ValueSign { + if value >= 0.0 { + NonNegative + } else if value < 0.0 { + NonPositive + } else { + Unknown + } +} diff --git a/crates/types/src/stages/column_sign.rs b/crates/types/src/stages/column_sign.rs deleted file mode 100644 index ffef920aa..000000000 --- a/crates/types/src/stages/column_sign.rs +++ /dev/null @@ -1,449 +0,0 @@ -//! Value sign: the sign of each output column of a memo group (concepts -//! doc, section 3, "Value sign"), derived once from the input groups' signs -//! and, for a `Scan`, the signs the data workload declares for its source's -//! columns. Error contracts that hold only for values of one sign read it. - -use std::collections::BTreeMap; - -use super::{MemoGroupAsInput, ValueSign}; -use crate::ir::operator::asap::{summary_agg_schema, summary_build}; -use crate::ir::operator::input::OperatorInput; -use crate::ir::operator::operator_properties::{ - BinaryOpKind, JoinKind, PromQLVectorSetOpKind, Reduction, SourceColumn, -}; -use crate::ir::operator::AggIntent; -use crate::ir::scalar::expr_ir::ScalarValue; -use crate::ir::scalar::{resolve_column_ref, ColumnRef, ProjectItem, ScalarExpr}; -use crate::ir::schema::sketch::{ExactKind, SketchStatistic, SummaryInputExpr, SummaryUpdate}; -use crate::ir::schema::{FieldDataType, Schema}; -use crate::ir::{ASAPOp, NonASAPOp, Operator}; -use crate::workload::bounds::source_name; - -use ValueSign::{NonNegative, NonPositive, Unknown}; - -type Input<'m> = MemoGroupAsInput<'m>; - -/// The sign of each column of `output`, the output schema of `operator`, -/// from its input groups' `column_sign` and, for a `Scan`, `declared`. -pub(crate) fn output_column_sign( - operator: &Operator>, - output: &Schema, - declared: &BTreeMap, -) -> Vec { - let signs = match operator { - Operator::NonASAP(op) => non_asap_signs(op, output, declared), - Operator::ASAP(op) => asap_signs(op, output), - }; - debug_assert_eq!( - signs.len(), - output.fields.len(), - "one sign per output column of {}", - operator.kind_name() - ); - signs -} - -fn signs_of(input: &Input<'_>) -> Vec { - input.group_properties().column_sign.clone() -} - -fn non_asap_signs( - op: &NonASAPOp>, - output: &Schema, - declared: &BTreeMap, -) -> Vec { - use NonASAPOp::*; - let unknown = || vec![Unknown; output.fields.len()]; - match op { - Scan { source, schema, .. } => schema - .fields - .iter() - .map(|field| { - let column = SourceColumn { - source: source_name(source).to_owned(), - column: field.name.clone(), - }; - declared.get(&column).copied().unwrap_or(Unknown) - }) - .collect(), - Values { .. } | PromqlVectorFromScalar(_) => unknown(), - Filter { child, .. } - | Sort { child, .. } - | Limit { child, .. } - | Dedup { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | PromqlSubquery { child, .. } - | PromqlSeriesSample { child, .. } - | PromqlInfoEnrich { child, .. } => signs_of(child), - // The relabeled column is written; every other column is copied. - PromqlRelabel { dst, child, .. } => { - let mut signs = signs_of(child); - match child.schema().column_id(dst) { - Some(written) => signs[written] = Unknown, - None => signs.push(Unknown), - } - signs - } - Project { cols, child, .. } => project_signs(cols, &signs_of(child)), - Aggregate { - reduction, - measures, - child, - .. - } => { - if let [AggIntent::TopK { .. }] = measures.as_slice() { - // The ranked rows, projected: each column copied. - return match crate::ir::schema::topk::ranked_row_projection(child, reduction, "") { - Ok(cols) => project_signs(&cols, &signs_of(child)), - Err(_) => unknown(), - }; - } - aggregate_signs(child, reduction, measures, output) - } - Join { - kind, left, right, .. - } => { - let mut signs = signs_of(left); - if !matches!(kind, JoinKind::Semi | JoinKind::Anti) { - signs.extend(signs_of(right)); - } - signs - } - SetOp { left, right, .. } => agreed(&[signs_of(left), signs_of(right)]), - Concat { children } => agreed(&children.iter().map(signs_of).collect::>()), - SQLWindowFunc { child, .. } => { - let mut signs = signs_of(child); - signs.push(Unknown); - signs - } - // The left side's columns, its value per the operator, then the - // labels the match adds (unknown). - BinaryOp { - operator, lhs, rhs, .. - } => { - let mut signs = signs_of(lhs); - if let Ok(value) = resolve_column_ref(&ColumnRef::SampleValue, lhs.schema()) { - let rhs_value = resolve_column_ref(&ColumnRef::SampleValue, rhs.schema()) - .ok() - .map_or(Unknown, |column| signs_of(rhs)[column]); - signs[value] = match &operator.kind { - // A filter keeps the left side's values. - BinaryOpKind::Compare(_) - | BinaryOpKind::Set( - PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless, - ) => signs[value], - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => agree(signs[value], rhs_value), - BinaryOpKind::CompareBool(_) => NonNegative, - BinaryOpKind::Arithmetic(_) => Unknown, - }; - } - signs.resize(output.fields.len(), Unknown); - signs - } - } -} - -fn asap_signs(op: &ASAPOp>, output: &Schema) -> Vec { - use ASAPOp::*; - match op { - SummaryAgg { - child, - input, - reduction, - .. - } => state_signs(child, input, reduction, output), - // The build's layout without its own time column, after the - // computed `window_end` (column 0). - SummaryTumblingWindowChain { - child, - summary_type, - input, - reduction, - .. - } => match summary_agg_schema(child.schema(), summary_type, reduction) { - Ok(build) => { - let mut signs = state_signs(child, input, reduction, &build); - if let Some(time) = build.time_index { - signs.remove(time); - } - signs.insert(0, Unknown); - signs - } - Err(_) => vec![Unknown; output.fields.len()], - }, - SummaryMerge { children, group_by } => { - // The windows a `TimeRange` over a chain selects merge into the - // chain's build over that range. - if let (None, [child]) = (group_by, children.as_slice()) { - if let Some(SummaryTumblingWindowChain { - child, - input, - reduction, - .. - }) = summary_build(child) - { - return state_signs(&child, &input, &reduction, output); - } - } - let merged = agreed(&children.iter().map(signs_of).collect::>()); - match group_by { - None => merged, - Some(keys) => { - let state = children[0] - .schema() - .fields - .iter() - .position(|field| !field.is_plain()); - keys.iter() - .chain(state.as_ref()) - .map(|&column| merged.get(column).copied().unwrap_or(Unknown)) - .collect() - } - } - } - SummaryEstimate { - summary_input, - query, - } => readout_signs(summary_input, output, |state| match query { - SketchStatistic::Cardinality - | SketchStatistic::FrequencyL2 - | SketchStatistic::FrequencyEntropy => NonNegative, - SketchStatistic::Quantile { .. } - | SketchStatistic::PointCount { .. } - | SketchStatistic::TopK { .. } => state, - }), - FinalizeExactAccumulator { child } => { - let counts = child.schema().fields.iter().any(|field| { - matches!( - field.dtype, - FieldDataType::ExactAggregate(ExactKind::Count, _) - ) - }); - readout_signs( - child, - output, - |state| if counts { NonNegative } else { state }, - ) - } - MaintainPopulation { child, .. } => signs_of(child), - EvaluatePopulation { child, .. } if output == child.schema() => signs_of(child), - EvaluatePopulation { .. } - | SummarySubtract { .. } - | SummaryDelete { .. } - | SummaryJoin { .. } - | Extension { .. } => vec![Unknown; output.fields.len()], - } -} - -/// A column item keeps its input column's sign; an expression item's sign -/// is [`expression_sign`]'s. -fn project_signs(cols: &[ProjectItem], input: &[ValueSign]) -> Vec { - cols.iter() - .map(|item| expression_sign(&item.expr, input)) - .collect() -} - -/// `abs` is non-negative, a negation flips its operand's sign, a numeric -/// cast keeps it and a numeric literal has its own; any other expression is -/// `Unknown`. -fn expression_sign(expr: &ScalarExpr, input: &[ValueSign]) -> ValueSign { - match expr { - ScalarExpr::Column(column) => input.get(*column).copied().unwrap_or(Unknown), - ScalarExpr::Literal(ScalarValue::Float64(value)) => sign_of_number(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => sign_of_number(*value as f64), - ScalarExpr::Negative { expr, .. } => flipped(expression_sign(expr, input)), - ScalarExpr::Cast { expr, .. } => expression_sign(expr, input), - // PromQL's `abs(v)` lowers to `promql_abs`, SQL's to `abs`. - ScalarExpr::FunctionCall { name, .. } - if name.eq_ignore_ascii_case("abs") || name == "promql_abs" => - { - NonNegative - } - _ => Unknown, - } -} - -/// An aggregate's, in `aggregate_output_schema`'s layout: a per-entity -/// reduction keeps its input's columns in place, its value replaced by the -/// measure's; a `by` reduction lists its keys, then its measures; a -/// `without` reduction the input's other label columns, then its measures. -fn aggregate_signs( - child: &Input<'_>, - reduction: &Reduction, - measures: &[AggIntent], - output: &Schema, -) -> Vec { - let input = signs_of(child); - let schema = child.schema(); - let sample_value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); - let value_of = |measure: &AggIntent| measure.input_cols().first().copied().or(sample_value); - let keys: Vec = match reduction { - Reduction::PerEntity => { - let mut signs = input.clone(); - if let Some(measure) = measures.first() { - if let Some(value) = value_of(measure).filter(|&value| value < signs.len()) { - signs[value] = measure_sign(measure, input[value]); - } - } - return signs; - } - Reduction::Reduce(by) if by.is_without() => (0..schema.fields.len()) - .filter(|&column| { - schema.time_index != Some(column) - && sample_value != Some(column) - && !by.keys().contains(&column) - }) - .map(|column| input[column]) - .collect(), - Reduction::Reduce(by) => by - .keys() - .iter() - .map(|&key| input.get(key).copied().unwrap_or(Unknown)) - .collect(), - }; - let measured: Vec = measures - .iter() - .flat_map(|measure| { - let sign = measure_sign( - measure, - value_of(measure) - .and_then(|value| input.get(value).copied()) - .unwrap_or(Unknown), - ); - // `count_values` and `absent` add label columns before their value. - let labels = match measure { - AggIntent::CountValues { label } => { - usize::from(!output.fields[..keys.len()].iter().any(|f| f.name == *label)) - } - AggIntent::Absent { labels } => labels.len(), - _ => 0, - }; - std::iter::repeat_n(Unknown, labels).chain([sign]) - }) - .collect(); - keys.into_iter().chain(measured).collect() -} - -/// The sign of `measure`'s value over an input value of sign `input`. -fn measure_sign(measure: &AggIntent, input: ValueSign) -> ValueSign { - use AggIntent::*; - match measure { - Count - | Cardinality { .. } - | CountValues { .. } - | Changes - | Resets - | Group - | Absent { .. } - | AbsentOverTime - | PresentOverTime - | StdDev { .. } - | Variance { .. } - | FrequencyL2 { .. } - | FrequencyEntropy { .. } - | Math(crate::ir::operator::MathFunc::Abs) => NonNegative, - Sum { .. } - | Avg { .. } - | Min { .. } - | Max { .. } - | Quantile { .. } - | LastOverTime - | FirstOverTime => input, - // A counter's rate and increase are non-negative; over a value of - // another sign a decrease reads as a reset, so no sign is known. - Rate | IRate | Increase if input == NonNegative => NonNegative, - _ => Unknown, - } -} - -/// A summary state's, in `state_schema`'s layout (an aggregate of one -/// measure): its key columns keep their input's sign and its state column -/// records the sign of the values it is updated with. -fn state_signs( - child: &Input<'_>, - update: &SummaryUpdate, - reduction: &Reduction, - state_schema: &Schema, -) -> Vec { - let mut signs = aggregate_signs( - child, - reduction, - &[AggIntent::Sum { col: None }], - state_schema, - ); - let updated_with = match &update.weight { - SummaryInputExpr::Column(column) => resolve_column_ref(column, child.schema()) - .ok() - .and_then(|column| signs_of(child).get(column).copied()) - .unwrap_or(Unknown), - SummaryInputExpr::Constant(value) => sign_of_number(*value), - SummaryInputExpr::Tuple(_) | SummaryInputExpr::EntityIdentity(_) => Unknown, - }; - if let Some(state) = state_schema.fields.iter().position(|f| !f.is_plain()) { - signs[state] = updated_with; - } - signs -} - -/// A readout of summary state: the state's plain columns keep their sign -/// in place, and the value read out has `value(the state's recorded sign)`. -/// `Unknown` everywhere when the output does not keep the state's layout (a -/// top-k's rows). -fn readout_signs( - state: &Input<'_>, - output: &Schema, - value: impl Fn(ValueSign) -> ValueSign, -) -> Vec { - let input = state.schema(); - if input.fields.len() != output.fields.len() { - return vec![Unknown; output.fields.len()]; - } - signs_of(state) - .into_iter() - .zip(&input.fields) - .map(|(sign, field)| if field.is_plain() { sign } else { value(sign) }) - .collect() -} - -/// Per column, the inputs' sign if they all agree, else `Unknown`. -fn agreed(inputs: &[Vec]) -> Vec { - let Some((first, rest)) = inputs.split_first() else { - return Vec::new(); - }; - first - .iter() - .enumerate() - .map(|(column, &sign)| { - rest.iter().fold(sign, |agreed, other| { - agree(agreed, other.get(column).copied().unwrap_or(Unknown)) - }) - }) - .collect() -} - -fn agree(left: ValueSign, right: ValueSign) -> ValueSign { - if left == right { - left - } else { - Unknown - } -} - -fn flipped(sign: ValueSign) -> ValueSign { - match sign { - NonNegative => NonPositive, - NonPositive => NonNegative, - Unknown => Unknown, - } -} - -fn sign_of_number(value: f64) -> ValueSign { - if value >= 0.0 { - NonNegative - } else if value < 0.0 { - NonPositive - } else { - Unknown - } -} diff --git a/crates/types/src/stages/memo.rs b/crates/types/src/stages/memo.rs index 1995a62a8..c6b32e636 100644 --- a/crates/types/src/stages/memo.rs +++ b/crates/types/src/stages/memo.rs @@ -5,6 +5,7 @@ use std::collections::BTreeSet; use super::{CandidateForMemoGroup, Memo, MemoQuery}; use crate::ir::operator::input::OperatorInput; +use crate::ir::operator::operator_properties::SourceColumn; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::schema::Schema; use crate::ir::{Operator, OperatorResultKind}; @@ -43,8 +44,12 @@ pub struct MemoGroup { /// operator nodes. #[derive(Debug, Clone)] pub struct GroupProperties { + /// Fields: name, type, nullability; no table qualifier. pub schema: Schema, pub result_kind: OperatorResultKind, + /// Per output column, in schema order: the source column its values + /// are copied from, or `Computed`. + pub column_lineage: Vec, /// Per output column, in schema order: the sign of its values, from the /// input groups' and, for a `Scan`, the declared ones. pub column_sign: Vec, @@ -68,6 +73,19 @@ pub struct GroupProperties { pub reads_only_time_series: bool, } +/// Where an output column's values come from, a group property derived +/// from the input groups (concepts doc, section 3, "Column lineage"). It +/// names a column across groups where a schema field's name cannot: two +/// join sides' columns of one name have different lineage, and a renamed +/// column keeps its lineage. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ColumnLineage { + /// Copied unchanged from a column of a source. + SourceColumn(SourceColumn), + /// Computed by an operator: a measure, an expression, a summary state or readout. + Computed, +} + /// The sign of one column's values, a group property derived from the /// input groups like lineage (concepts doc, section 3, "Value sign"); a /// `Scan`'s are declared per column in the data workload's source facts. diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index 66bb0da50..f200f995d 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -8,7 +8,7 @@ //! live here. mod candidates; -mod column_sign; +pub(crate) mod column_properties; mod hash_consing; mod logical_memo_candidates; mod memo; @@ -26,7 +26,8 @@ pub use logical_memo_candidates::{ }; pub(crate) use memo::MemoGroupAsInput; pub use memo::{ - queries_rooted_at, root_groups, GroupProperties, MemoGroup, MemoGroupId, ValueSign, + queries_rooted_at, root_groups, ColumnLineage, GroupProperties, MemoGroup, MemoGroupId, + ValueSign, }; pub use per_computation_logical_memo_candidates::{ ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index b6eafaed4..f39e5ccc5 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -88,16 +88,10 @@ pub fn population_over_selector( ) -> Rc { use crate::ir::operator::maintained_population::*; use crate::ir::operator::operator_properties::Source; - if let Some(NonASAPOp::Project { - cols, - qualifier, - child, - }) = root.non_asap() - { + if let Some(NonASAPOp::Project { cols, child }) = root.non_asap() { let child = population_over_selector(child, statistic.clone(), grouping); let project = Operator::NonASAP(NonASAPOp::Project { cols: cols.clone(), - qualifier: qualifier.clone(), child, }); return Rc::new(OperatorNode::with_schema(project, root.schema.clone())); diff --git a/crates/types/tests/column_lineage.rs b/crates/types/tests/column_lineage.rs new file mode 100644 index 000000000..ff64e5022 --- /dev/null +++ b/crates/types/tests/column_lineage.rs @@ -0,0 +1,427 @@ +//! Column lineage, derived once per memo group from its input groups' +//! (concepts doc `memo-groups-properties-requirements.md`, section 3, +//! "Column lineage"): the doc's examples 1 to 3. Example 4 (PromQL label +//! matchers) is `scan_predicates_lift_and_their_states_merge` in +//! `memo_summary_coverage.rs`. +use std::time::Duration; + +use asap_types::ir::operator::operator_properties::{ + AtModifier, JoinKind, Reduction, Source, SourceColumn, TimeShift, +}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::summary_coverage::Constraint; +use asap_types::ir::scalar::expr_ir::{CompareOpKind, ScalarValue}; +use asap_types::ir::scalar::ColumnRef; +use asap_types::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchStatistic, SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, ScalarExpr, TimeRangeKind, +}; +use asap_types::stages::{ + CandidateExpression, CandidateForMemoGroup, ColumnLineage, LogicalProperties, + LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; + +/// `Scan table` of Utf8 `columns`. +fn scan(table: &str, columns: &[&str]) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: table.into(), + }, + predicates: vec![], + schema: Schema::new( + columns + .iter() + .map(|name| Field::plain(*name, DataType::Utf8, false)) + .collect(), + ), + }) +} + +/// `column = value`. +fn equals(column: usize, value: &str) -> Predicate { + Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(value.into()))), + semantics: ExprSemantics::Sql, + }) +} + +fn filter(pred: Predicate, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::Filter { pred, child }) +} + +/// `SummaryAgg[HLL](by keys; column)`: the distinct values of `column`. +fn hll(child: MemoGroupId, keys: Vec, column: &str) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Hll), + input: SummaryUpdate::column(ColumnRef::Named(column.into())), + reduction: Reduction::by(keys), + filter: None, + }) +} + +/// The column `column` of the table or metric `source`. +pub(crate) fn column_of(source: &str, column: &str) -> SourceColumn { + SourceColumn { + source: source.into(), + column: column.into(), + } +} + +fn source_column(source: &str, column: &str) -> ColumnLineage { + ColumnLineage::SourceColumn(column_of(source, column)) +} + +fn lineage(memo: &Memo, group: MemoGroupId) -> &[ColumnLineage] { + &memo.groups()[group.0 as usize] + .group_properties + .column_lineage +} + +/// Example 1: `SELECT f.src, approx_distinct(f.dst) FROM flows f WHERE +/// f.proto = 'tcp' GROUP BY f.src`. The group-by key keeps `flows.src` +/// through the aggregate, the HLL state and its readout; the readout +/// renamed to the aggregate's column name has the aggregate's schema, so +/// `add_candidate` accepts it. +#[test] +fn an_hll_readout_renamed_to_the_aggregate_column_is_its_candidate() { + let mut memo = Memo::default(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst", "proto", "ts"])) + .unwrap(); + let g2 = memo + .find_or_add_group(filter(equals(2, "tcp"), g1)) + .unwrap(); + let g3 = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Cardinality { cols: vec![1] }], + output_names: vec!["approx_distinct(dst)".into()], + filters: vec![], + having: None, + child: g2, + })) + .unwrap(); + let g4 = memo.find_or_add_group(hll(g2, vec![0], "dst")).unwrap(); + let g5 = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: g4, + query: SketchStatistic::Cardinality, + })) + .unwrap(); + let flows = ["src", "dst", "proto", "ts"].map(|column| source_column("flows", column)); + assert_eq!(lineage(&memo, g1), flows); + assert_eq!(lineage(&memo, g2), flows); + let key_then_computed = [source_column("flows", "src"), ColumnLineage::Computed]; + for group in [g3, g4, g5] { + assert_eq!(lineage(&memo, group), key_then_computed); + } + // The state's selection names the filtered column by its lineage. + let coverage = memo.groups()[g4.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert_eq!( + coverage.selection()[0].columns()[&column_of("flows", "proto")], + Constraint::In(vec![ScalarValue::Utf8("tcp".into())]) + ); + + let operator = Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: Some("src".into()), + expr: ScalarExpr::Column(0), + }, + ProjectItem { + alias: Some("approx_distinct(dst)".into()), + expr: ScalarExpr::Column(1), + }, + ], + child: g5, + }); + let expression = CandidateExpression { operator }; + let candidate = CandidateForMemoGroup { + logical_properties: LogicalProperties::new(&expression, None).unwrap(), + expression, + input_requirements: vec![LogicalRequirements::default()], + fused_groups: vec![], + origin: OriginOfCandidateForMemoGroup::Pass1, + }; + let id = memo.add_candidate(g3, candidate).unwrap(); + assert_eq!(id.group, g3); +} + +/// Example 2: `SELECT source, count(*) FROM (SELECT f.src AS source FROM +/// flows f) GROUP BY source`. The renaming `Project` keeps `flows.src`; a +/// state grouped by `source` has the key lineage of one grouped by `src` +/// over the scan, a selection on the renamed `area` names `flows.region`, +/// and the rollup's group-key check, which compares lineage, passes. +#[test] +fn a_rename_keeps_lineage_and_the_rollup_key_check_compares_it() { + let mut memo = Memo::default(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst", "region"])) + .unwrap(); + let renamed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: [("source", 0), ("dst", 1), ("area", 2)] + .map(|(alias, column)| ProjectItem { + alias: Some(alias.into()), + expr: ScalarExpr::Column(column), + }) + .to_vec(), + child: g1, + })) + .unwrap(); + assert_eq!( + lineage(&memo, renamed), + [ + source_column("flows", "src"), + source_column("flows", "dst"), + source_column("flows", "region"), + ] + ); + + let by_source = memo + .find_or_add_group(hll(renamed, vec![0], "dst")) + .unwrap(); + let by_src = memo.find_or_add_group(hll(g1, vec![0], "dst")).unwrap(); + assert_eq!(lineage(&memo, by_source)[0], lineage(&memo, by_src)[0]); + + let in_us = memo + .find_or_add_group(filter(equals(2, "us"), renamed)) + .unwrap(); + let by_source_and_area = memo + .find_or_add_group(hll(in_us, vec![0, 2], "dst")) + .unwrap(); + let coverage = memo.groups()[by_source_and_area.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert_eq!( + coverage.selection()[0].columns()[&column_of("flows", "region")], + Constraint::In(vec![ScalarValue::Utf8("us".into())]) + ); + + let rolled_up = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![by_source_and_area], + group_by: Some(vec![0]), + })) + .unwrap(); + let rolled_up = &memo.groups()[rolled_up.0 as usize].group_properties; + assert_eq!( + rolled_up.column_lineage, + [source_column("flows", "src"), ColumnLineage::Computed] + ); + assert!(rolled_up.aggregate_coverage.is_some()); +} + +/// Example 3: `SELECT s.region, count(*) FROM sales s JOIN users u ON +/// s.user = u.id GROUP BY s.region`. Both join sides have a `region` +/// column; their lineage tells them apart, so selections on them are +/// different columns. +#[test] +fn join_columns_of_one_name_have_distinct_lineage() { + let mut memo = Memo::default(); + let sales = memo + .find_or_add_group(scan("sales", &["user", "region"])) + .unwrap(); + let users = memo + .find_or_add_group(scan("users", &["id", "region"])) + .unwrap(); + let joined = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Join { + kind: JoinKind::Inner, + pred: Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(0)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Column(2)), + semantics: ExprSemantics::Sql, + }), + left: sales, + right: users, + })) + .unwrap(); + assert_eq!( + lineage(&memo, joined), + [ + source_column("sales", "user"), + source_column("sales", "region"), + source_column("users", "id"), + source_column("users", "region"), + ] + ); + + let selected_column = |memo: &mut Memo, region_column: usize| { + let in_us = memo + .find_or_add_group(filter(equals(region_column, "us"), joined)) + .unwrap(); + let state = memo.find_or_add_group(hll(in_us, vec![], "id")).unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + let [selection] = coverage.selection() else { + panic!("one box: {coverage:?}") + }; + selection.columns().keys().cloned().collect::>() + }; + assert_eq!( + selected_column(&mut memo, 1), + [column_of("sales", "region")] + ); + assert_eq!( + selected_column(&mut memo, 3), + [column_of("users", "region")] + ); +} + +/// A predicate on a computed column is not lifted: no selection can name +/// the column, so it stays in the definition's unlifted operators. +#[test] +fn a_predicate_on_a_computed_column_stays_unlifted() { + let mut memo = Memo::default(); + let g1 = memo + .find_or_add_group(scan("flows", &["src", "dst"])) + .unwrap(); + let computed = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + ProjectItem { + alias: Some("src".into()), + expr: ScalarExpr::Column(0), + }, + ProjectItem { + alias: Some("tag".into()), + expr: ScalarExpr::Literal(ScalarValue::Utf8("x".into())), + }, + ], + child: g1, + })) + .unwrap(); + assert_eq!( + lineage(&memo, computed), + [source_column("flows", "src"), ColumnLineage::Computed] + ); + let tagged = memo + .find_or_add_group(filter(equals(1, "x"), computed)) + .unwrap(); + let state = memo.find_or_add_group(hll(tagged, vec![], "src")).unwrap(); + let coverage = memo.groups()[state.0 as usize] + .group_properties + .aggregate_coverage + .as_ref() + .expect("a summary state"); + assert!(coverage.selection()[0].columns().is_empty()); + assert!(coverage + .definition() + .unlifted_operators + .contains(&filter(equals(1, "x"), MemoGroupId(0)).map_children(|_| None))); +} + +/// `Scan latency`: `[ts, value, job]`, `ts` its time index. +fn latency_series() -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: "latency".into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, false), + ], + 0, + vec![], + ), + }) +} + +fn time_range(width_ms: u64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_millis(width_ms), + kind: TimeRangeKind::Range, + child, + }) +} + +fn time_shift(offset_ms: i64, at: Option, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { offset_ms, at }, + child, + }) +} + +/// A chain of `SummaryAgg[KLL](by job; value)` windows: `[window_end, job, +/// state]`. `window_end` (column 0) and the state are computed and the key +/// keeps its lineage. A `TimeShift` and a `TimeRange` over the chain select +/// windows, so they pass the chain's lineage through; the merge of the +/// selected windows has the layout of the chain's build, `[job, state]`. +#[test] +fn a_chain_computes_window_end_and_state_and_its_window_selections_pass_lineage_through() { + let mut memo = Memo::default(); + let series = memo.find_or_add_group(latency_series()).unwrap(); + let chain = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: series, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + tumbling_window_width_ms: 60_000, + })) + .unwrap(); + let chain_lineage = [ + ColumnLineage::Computed, + source_column("latency", "job"), + ColumnLineage::Computed, + ]; + assert_eq!(lineage(&memo, chain), chain_lineage); + + let an_hour_ago = memo + .find_or_add_group(time_shift(3_600_000, None, chain)) + .unwrap(); + assert_eq!(lineage(&memo, an_hour_ago), chain_lineage); + let five_minutes_an_hour_ago = memo + .find_or_add_group(time_range(300_000, an_hour_ago)) + .unwrap(); + assert_eq!(lineage(&memo, five_minutes_an_hour_ago), chain_lineage); + + let merged = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![five_minutes_an_hour_ago], + group_by: None, + })) + .unwrap(); + assert_eq!( + lineage(&memo, merged), + [source_column("latency", "job"), ColumnLineage::Computed] + ); +} + +/// The PromQL front end lowers `m[5m] @ end()` as `TimeShift { at: End }` +/// above the selector lowered without `@`: it pins when the subtree is +/// read and copies its rows, so it passes the selector's lineage through, +/// as `@ start()` does. +#[test] +fn a_time_shift_at_end_or_start_above_a_selector_passes_its_lineage_through() { + let mut memo = Memo::default(); + let series = memo.find_or_add_group(latency_series()).unwrap(); + let selector = memo.find_or_add_group(time_range(300_000, series)).unwrap(); + let series_lineage = ["ts", "value", "job"].map(|column| source_column("latency", column)); + for at in [AtModifier::End, AtModifier::Start] { + let pinned = memo + .find_or_add_group(time_shift(0, Some(at), selector)) + .unwrap(); + assert_eq!(lineage(&memo, pinned), series_lineage); + } +} diff --git a/crates/types/tests/main.rs b/crates/types/tests/main.rs index e4f12481b..cb5e33b19 100644 --- a/crates/types/tests/main.rs +++ b/crates/types/tests/main.rs @@ -2,6 +2,7 @@ //! its own copy of every dependency, which outgrew CI's disk. //! One file: `cargo test -p asap-types --test types ::`. +mod column_lineage; mod memo_group_properties; mod memo_summary_coverage; mod physical_export; diff --git a/crates/types/tests/memo_group_properties.rs b/crates/types/tests/memo_group_properties.rs index 9989fe250..7567d16b2 100644 --- a/crates/types/tests/memo_group_properties.rs +++ b/crates/types/tests/memo_group_properties.rs @@ -132,7 +132,6 @@ fn map_value(expr: ScalarExpr, child: MemoGroupId) -> Operator Constraint { relative(-MINUTE_MS, 0) } -fn ts() -> ColumnIdentity { - ColumnIdentity::new(None, "ts") +fn ts() -> SourceColumn { + latency("ts") +} + +/// The column `column` of the series `latency`. +fn latency(column: &str) -> SourceColumn { + column_of("latency", column) } /// The constraint on the timestamp column of `group`'s one selection box. @@ -268,10 +275,7 @@ fn a_summary_agg_over_a_job_filter_lifts_it_into_the_selection() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], - jobs(&["api"]) - ); + assert_eq!(selection.columns()[&latency("job")], jobs(&["api"])); assert_eq!(selection.columns()[&ts()], last_minute()); } @@ -297,10 +301,7 @@ fn states_with_equal_definitions_and_disjoint_selections_merge() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], - jobs(&["api", "web"]) - ); + assert_eq!(selection.columns()[&latency("job")], jobs(&["api", "web"])); } /// Two states that select the same rows do not merge: KLL counts a row @@ -344,7 +345,7 @@ fn a_filter_that_is_no_selection_stays_unlifted() { ] ); assert_eq!( - coverage_of_g10.selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage_of_g10.selection()[0].columns()[&latency("job")], jobs(&["api"]) ); assert!(matches!( @@ -399,19 +400,28 @@ fn states_whose_definitions_read_the_same_subquery_merge() { assert_eq!(definition, coverage(&memo, web).definition()); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, merged).selection()[0].columns()[&latency("job")], jobs(&["api", "web"]) ); } /// PromQL label matchers are the scan's predicates: they are lifted too, so /// `latency{job="api"}` and `latency{job="web"}` states merge. The scan, -/// without them, is the last unlifted operator. +/// without them, is the last unlifted operator. Each scan's `job` column has +/// lineage `latency.job`, so the selections `latency.job ∈ {api}` and +/// `latency.job ∈ {web}` are disjoint selections on one column (concepts doc, +/// section 3, column lineage example 4). #[test] fn scan_predicates_lift_and_their_states_merge() { let mut memo = Memo::default(); let state_of_job = |memo: &mut Memo, job: &str| { let series = memo.find_or_add_group(scan(vec![job_is(job)])).unwrap(); + assert_eq!( + memo.groups()[series.0 as usize] + .group_properties + .column_lineage[JOB], + ColumnLineage::SourceColumn(latency("job")) + ); let last_minute = memo.find_or_add_group(range(MINUTE_MS, series)).unwrap(); memo.find_or_add_group(kll(last_minute)).unwrap() }; @@ -424,12 +434,12 @@ fn scan_predicates_lift_and_their_states_merge() { vec![kll_unlifted(), scan(vec![]).map_children(|_| None)] ); assert_eq!( - coverage(&memo, api).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, api).selection()[0].columns()[&latency("job")], jobs(&["api"]) ); let merged = memo.find_or_add_group(merge(vec![api, web])).unwrap(); assert_eq!( - coverage(&memo, merged).selection()[0].columns()[&ColumnIdentity::new(None, "job")], + coverage(&memo, merged).selection()[0].columns()[&latency("job")], jobs(&["api", "web"]) ); } @@ -593,7 +603,7 @@ fn value_and_absolute_timestamp_filters_lift_with_fixed_value_limits() { } ); assert_eq!( - coverage(&memo, g3).selection()[0].columns()[&ColumnIdentity::new(None, "value")], + coverage(&memo, g3).selection()[0].columns()[&latency("value")], Constraint::Interval { lower: Bound::Excluded(IntervalConstraintLimit::FixedValue(ScalarValue::Float64( 10.0 @@ -675,9 +685,6 @@ fn an_exact_aggregate_has_coverage() { let [selection] = coverage.selection() else { panic!("one box: {coverage:?}") }; - assert_eq!( - selection.columns()[&ColumnIdentity::new(None, "job")], - jobs(&["api"]) - ); + assert_eq!(selection.columns()[&latency("job")], jobs(&["api"])); assert_eq!(selection.columns()[&ts()], last_minute()); } diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index e9861ccaf..0b798d744 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -175,7 +175,6 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { semantics: ExprSemantics::Promql, }, }], - qualifier: None, child: series("a"), })) .unwrap(); diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 7878f45be..d0ca46268 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -64,7 +64,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { let original = aggregate(scan(DataType::Int64, "key"), asap); let mut schema = original.schema.clone(); schema.fields[0].name = "alias".into(); - schema.fields[0].table = Some("result".into()); let renamed = OperatorNode::with_schema(original.operator.clone(), schema) .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); let original = Rc::new(renamed); @@ -72,7 +71,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { let replacement = scan(DataType::Utf8, "new_key"); let rebuilt = Rc::new(original.with_new_children(|_| replacement.clone()).unwrap()); assert_eq!(rebuilt.schema.fields[0].name, "alias"); - assert_eq!(rebuilt.schema.fields[0].table.as_deref(), Some("result")); assert_eq!( rebuilt.schema.fields[0].plain_dtype(), Some(&DataType::Utf8) diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 1129d0553..d077feb43 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -109,7 +109,6 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { expr: ScalarExpr::Column(0), alias: None, }], - qualifier: None, })) .unwrap(); pass.validate_structure().unwrap(); @@ -162,7 +161,6 @@ fn shared_construction_derives_both_operator_categories() { expr: ScalarExpr::Column(99), alias: None }], - qualifier: None, })) .is_err() ); diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 6326e7a3a..d28a66a93 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -48,8 +48,8 @@ This section fixes the words used below. They follow relational databases and Ap |---|---|---| | **Relation / table** | A set (bag) of rows with the same columns. A base table is stored; a derived relation is the output of a query operator. | Every edge in the DAG carries a relation. A `Scan` reads a base table (SQL table or PromQL metric); every other operator outputs a derived relation. | | **Row / tuple** | One element of a relation: one value per column. | One output row of a node. For PromQL, one sample of one series at one time. | -| **Column** | One position in every row, with a name and a type. Qualified as `table.column` when names can collide (DataFusion `Column { relation, name }`). | `ColumnId` refers to a column of the input schema; `(table, name)` identifies it across nodes (§4.2.2). | -| **Schema** | The ordered list of columns of a relation: name, data type, nullability (Arrow `Schema` of `Field { name, data_type, nullable }`; DataFusion `DFSchema` adds the table qualifier). The schema is *metadata*: it describes rows, it contains none. | `Schema` of `Field { name, dtype, nullable, table }` in `crates/types/src/ir/schema/mod.rs`. Unlike Arrow, `dtype` can be a summary state type (§3). | +| **Column** | One position in every row, with a name and a type. Qualified as `table.column` when names can collide (DataFusion `Column { relation, name }`). | `ColumnId` refers to a column of the input schema; its memo group's column lineage, the source column `(source, column)` its values are copied from, identifies it across nodes (§4.2.2). A front end binds `table.column` with its own scope while it lowers. | +| **Schema** | The ordered list of columns of a relation: name, data type, nullability (Arrow `Schema` of `Field { name, data_type, nullable }`; DataFusion `DFSchema` adds the table qualifier). The schema is *metadata*: it describes rows, it contains none. | `Schema` of `Field { name, dtype, nullable }` in `crates/types/src/ir/schema/mod.rs`, with no table qualifier. Unlike Arrow, `dtype` can be a summary state type (§3). | | **Data type** | The type of a column's values (`Int64`, `Utf8`, `Timestamp`, …). | `DataType`, wrapped as `FieldDataType::Plain`. | | **Aggregate state** | The intermediate value of an aggregate function before its final result, e.g. `(sum, count)` for `AVG` (DataFusion `Accumulator::state`, partial/final aggregation). It is never exposed as a column type to users. | Summary state *is* a column type here (`FieldDataType::Sketch`, `ExactAggregate`, …), so state can flow along edges and be merged, stored and read by later operators. | | **View / materialized view** | A view is a named query (its *definition*). A materialized view also stores the query's result rows; a query can then be answered from it when its definition matches (view matching, §4.1). | A built summary state is a materialized aggregation view whose aggregate is a summary family. Its definition and which rows it took are its coverage (§4). | @@ -218,9 +218,9 @@ With an error target, the planner answers each quantile from a KLL state. All th | State | Sub-DAG | `definition` | `selection` | |---|---|---|---| -| `S_us` | `KLL(latency) by[job]` over `Filter(region = 'us', Scan t)` | `KLL(latency) by[job]` over `Scan t` | `region ∈ {us}` | -| `S_eu` | `KLL(latency) by[job]` over `Filter(region = 'eu', Scan t)` | `KLL(latency) by[job]` over `Scan t` | `region ∈ {eu}` | -| `S_size` | `KLL(size) by[job]` over `Filter(region = 'eu', Scan t)` | `KLL(size) by[job]` over `Scan t` | `region ∈ {eu}` | +| `S_us` | `KLL(latency) by[job]` over `Filter(region = 'us', Scan t)` | `KLL(latency) by[job]` over `Scan t` | `t.region ∈ {us}` | +| `S_eu` | `KLL(latency) by[job]` over `Filter(region = 'eu', Scan t)` | `KLL(latency) by[job]` over `Scan t` | `t.region ∈ {eu}` | +| `S_size` | `KLL(size) by[job]` over `Filter(region = 'eu', Scan t)` | `KLL(size) by[job]` over `Scan t` | `t.region ∈ {eu}` | - `S_us + S_eu` ✓: same computation, different rows. The result is the KLL of latency by job for US and EU, with every row counted once. - `S_us + S_us` ✗: same computation, same rows. Every US row would be counted twice. @@ -295,11 +295,13 @@ G4.aggregate_coverage = SummaryCoverage { input_group: Some(G1), unlifted_operators: [SummaryAgg[KLL]], }, - selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], + selection: [ { latency.job ∈ {api}, latency.ts ∈ (t − 60 s, t] } ], } ``` -**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ job ∈ {web}, ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ job ∈ {api, web}, ts ∈ (t − 60 s, t] }`. +A selection names a column by its lineage: `latency.job` is the `job` column of the series `latency` (see *Which column a condition is on* below). + +**Case 2, G7: the same definition, a disjoint selection.** The walk over G6 and G5 lifts `job = "web"` and the same minute, so G7's definition is G4's (`input_group: Some(G1)`, `unlifted_operators: [SummaryAgg[KLL]]`) and its `definition_id()` is the same; its selection is `{ latency.job ∈ {web}, latency.ts ∈ (t − 60 s, t] }`. The two selections are disjoint, so `SummaryMerge(G4, G7)` is a valid candidate (and group), covering `{ latency.job ∈ {api, web}, latency.ts ∈ (t − 60 s, t] }`. **Case 3, G10: `a + b > 10` cannot be lifted.** It spans two columns, so it is no selection. At G8 the walk lifts `job = "api"` and keeps the rest: @@ -309,7 +311,7 @@ G10.aggregate_coverage = SummaryCoverage { input_group: Some(G1), unlifted_operators: [SummaryAgg[KLL], Filter(a + b > 10)], }, - selection: [ { job ∈ {api}, ts ∈ (t − 60 s, t] } ], + selection: [ { latency.job ∈ {api}, latency.ts ∈ (t − 60 s, t] } ], } ``` @@ -380,11 +382,11 @@ GROUP BY job; | Condition | Found at | Rule 1: same rows at the `SummaryAgg`? | Rule 2: simple? | Result | |---|---|---|---|---| -| `latency < 100` | `SummaryAgg.filter` | yes, it is already there | yes, an interval | `selection`: `latency ∈ (−∞, 100)` | -| `region = 'us'` | `Scan.predicates` | yes: `Project` passes `region` through (renamed `r`) | yes, a value set | `selection`: `r ∈ {us}` | +| `latency < 100` | `SummaryAgg.filter` | yes, it is already there | yes, an interval | `selection`: `t.latency ∈ (−∞, 100)` | +| `region = 'us'` | `Scan.predicates` | yes: `Project` passes `region` through (renamed `r`) | yes, a value set | `selection`: `t.region ∈ {us}` | | `latency * 2 > 10` | `Scan.predicates` | yes | **no**: it is on an expression, not a column | stays in `definition` | -So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL example with a time window is under **Time** below. +So the `selection` is `t.latency ∈ (−∞, 100)`, `t.region ∈ {us}`. A PromQL example with a time window is under **Time** below. **Rule 1 in detail.** Imagine moving the condition up, one operator at a time, until it is just below the `SummaryAgg`. Every operator it passes must leave the picked rows unchanged. Whether it can pass depends on what the operator does: @@ -392,16 +394,16 @@ So the `selection` is `latency ∈ (−∞, 100)`, `r ∈ {us}`. A PromQL exampl |---|---|---|---| | another `Filter` (also `Scan.predicates`, `SummaryAgg.filter`) | yes | filters only drop rows, so their order does not matter | `region = 'us'` below `Filter(value < 100)` ✓ | | `TimeRange` (range) or `TimeShift` (with or without `@`) | yes | they choose a time window, but do not change any row's values | `region = 'us'` below `TimeRange(1m)` ✓ | -| `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, it becomes `r = 'us'`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | +| `Project` | only if its column is passed through unchanged (a rename is fine) | the column must still be there, with the same values, above the `Project` | `Project [job, region AS r]`: `region = 'us'` ✓, on `r`, whose lineage is still `t.region`. `Project [job, value * 2 AS v2]`: `value > 5` ✗, `value` is gone | | `Aggregate` (later) | only if it uses group columns | a group column has one value per group, so filtering before or after grouping keeps the same groups | below `SUM(value) by job`: `job = 'api'` ✓; `value > 5` ✗, it changes the sums | | `rate`, a window function, or an instant `TimeRange` (later) | only if it uses series labels | a label is the same for every sample of a series | below `rate(...)`: `job = 'api'` ✓; `value > 5` ✗, dropping raw samples changes the rate. Below an instant selector: `job = 'api'` ✓; `value > 5` ✗, the latest sample with value > 5 is not the latest sample | | any other operator, e.g. `Join`, a vector/vector `BinaryOp`, `Limit` | no | | | -A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. How a PromQL comparison is lowered decides whether it is such a `Filter`: +A `Filter` *above* `rate` has nothing to pass: it keeps some rate outputs, and those are exactly the rows the `SummaryAgg` reads. But its condition is on the rate value, a computed column with no source column to name it by, so it stays in `definition` too. How a PromQL comparison is lowered decides whether it is such a `Filter`: | PromQL | Lowered to | Coverage | |---|---|---| -| `rate(m[5m]) > 0` (vector and scalar) | `Filter` | `selection`: `value ∈ (0, ∞)` | +| `rate(m[5m]) > 0` (vector and scalar) | `Filter` | stays in `definition`: `value` is computed by `rate` | | `rate(m[5m]) > bool 0` | `Project` (values become 0 or 1, no row is dropped) | stays in `definition`: it does not pick rows | | `rate(a[5m]) > rate(b[5m])` (vector and vector) | `BinaryOp` (matches the two sides by labels, like a join) | stays in `definition`: the walk does not enter a `BinaryOp` | @@ -427,11 +429,11 @@ Any other shape stays in `definition`: **Which column a condition is on.** -- A column is named by its table and name, `(table, name)`, **as the `SummaryAgg` reads it** (in the schema of its child). Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions. -- A `Project` gives its columns new names and replaces their table with its own qualifier (none by default): after `region AS r`, the column is `(none, r)`, so the condition becomes `r ∈ {us}`. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. -- Only a `Project` may rename. Any other operator whose output names a column differently from its input (allowed for every operator, e.g. a `Filter` with an explicit schema) stays in `definition`, and so do the conditions at and below it. Removing it would drop the rename: if a `Filter` calls the `tier` column `region`, its `tier = 'eu'` would read as `region ∈ {eu}` and look disjoint from another state's `region ∈ {us}`, though a row with `region = 'us', tier = 'eu'` is in both. -- If two columns the `SummaryAgg` reads have the same `(table, name)` (for example `Project [a AS k, b AS k]`), a condition on `k` cannot tell them apart and stays in `definition`. -- PromQL labels have no table, so a label is named by its name alone. +- A column is named by its **lineage** ([concepts doc, section 3](../concepts/memo-groups-properties-requirements.md#column-lineage)) in the group the `SummaryAgg` reads: the source column `(source, column)` its values are copied from, a `Scan`'s table or metric and its column there. Directly above a join, `shipping.region = 'us'` and `billing.region = 'us'` are different conditions, though both columns are named `region`. +- A rename keeps lineage: after `Project [region AS r]`, `r` is still `t.region`, so the condition is `t.region ∈ {us}`. A memo group's schema is derived, so no other operator renames a column. Since the `definition` contains the same `Project`, two states that rename the same way still compare equal. +- A condition on a computed column (a `Project` expression item, an aggregate's measure, a readout's value) has no source column to name and stays in `definition`. +- If two columns the `SummaryAgg` reads have the same lineage (for example `Project [a AS k, a AS k2]`), the selection cannot name one of them apart, and their conditions stay in `definition`. +- A PromQL label is named by its metric and label: `latency.job`. - Values of different types are never treated as different: `1` and `1.0` might be equal, so `x = 1` and `x = 1.0` are treated as possibly overlapping. - A constant may come from a cast: type coercion turns `latency < 100` on a `DOUBLE` column into `latency < CAST(100 AS DOUBLE)`, and the SQL front end folds a numeric cast of a literal as DataFusion's constant folding does, so the condition reads `latency < 100.0`. @@ -455,11 +457,11 @@ Any other shape stays in `definition`: | Query | Sub-DAG below `SummaryAgg` | `definition` keeps | `selection` takes | |---|---|---|---| -| SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `region ∈ {us}` | -| SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `latency ∈ (−∞, 100)` | -| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `ts ∈ (t − 3m, t − 2m]` | +| SQL `… FROM t WHERE region = 'us'` | `Scan t {region = 'us'}` | `Scan t` | `t.region ∈ {us}` | +| SQL `… FROM t WHERE latency < 100` | `Scan t {latency < 100}` | `Scan t` | `t.latency ∈ (−∞, 100)` | +| PromQL `quantile_over_time(0.99, m[1m] offset 2m)` | `TimeRange(1m, TimeShift(2m, Scan m))` | `Scan m` | `m.ts ∈ (t − 3m, t − 2m]` | | PromQL `quantile(0.99, rate(m{job="api"}[5m]))` | `rate(TimeRange(5m, Scan m {job = 'api'}))` | all of it | nothing: `job = 'api'` is under `rate`, which the walk cannot pass yet (Rule 1, later) | -| PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | `rate(TimeRange(5m, Scan m))` | `value ∈ (0, ∞)`: the `Filter` is above `rate` | +| PromQL `quantile(0.99, rate(m[5m]) > 0)` | `Filter(value > 0, rate(TimeRange(5m, Scan m)))` | all of it | nothing: the `Filter` is above `rate`, but `value` is computed by `rate`, so no selection can name it | | PromQL `quantile(0.99, rate(a[5m]) > rate(b[5m]))` | `BinaryOp(>, rate(TimeRange(5m, Scan a)), rate(TimeRange(5m, Scan b)))` | all of it | nothing: a vector/vector comparison is a `BinaryOp`, not a `Filter` | `Scan t {…}` is a `Scan` with those predicates. In the last three rows `TimeRange(5m)` stays in `definition` in any case: it is the input window of `rate` and changes the rate values, so it does not just pick rows. @@ -557,7 +559,7 @@ With an error target, the planner answers the quantile from a KLL sketch with `k coverage() of the SummaryAgg ┌──────────────────────────────────────────────────────────────────┐ │ definition: this SummaryAgg over Scan t (the Filter removed) │ -│ selection: region ∈ {us}, latency ∈ (−∞, 10000) │ +│ selection: t.region ∈ {us}, t.latency ∈ (−∞, 10000) │ └──────────────────────────────────────────────────────────────────┘ ``` @@ -796,13 +798,13 @@ The first two build `KLL(latency) by[job]` states. The third can be answered by │ State: job Utf8, state Sketch(KLL k=200) │ [[ SummaryMerge ]] - ▲ selection: region ∈ {us, eu} + ▲ selection: t.region ∈ {us, eu} │ ┌─────────────────┴─────────────────┐ │ │ [[ SummaryAgg ]] [[ SummaryAgg ]] KLL k=200, by job KLL k=200, by job - selection: region ∈ {us} selection: region ∈ {eu} + selection: t.region ∈ {us} selection: t.region ∈ {eu} ▲ ▲ │ │ [ Filter region = 'us' ] [ Filter region = 'eu' ] @@ -1138,9 +1140,6 @@ impl Schema { pub fn is_all_plain(&self) -> bool; /// Position of the first field with this name, or `None`. pub fn column_id(&self, name: &str) -> Option; - /// Position of the field with this table and name, or `None`. Tells apart - /// `a.k` and `b.k` after a join. - pub fn column_id_qualified(&self, table: &str, name: &str) -> Option; } /// One column of a `Schema`. `T` is `FieldDataType` on DAG edges, and plain @@ -1153,13 +1152,9 @@ pub struct Field { pub dtype: T, /// Whether the column may contain NULL. PromQL value columns are never NULL. pub nullable: bool, - /// Table or alias the column comes from (`t` in `t.col`), so two columns - /// with the same name from a join stay apart. `None` for PromQL labels and - /// unqualified columns. - pub table: Option, } impl Field { - /// An unqualified field holding a plain value of type `dtype`. + /// A field holding a plain value of type `dtype`. pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self; /// The value type if the field is plain; `None` for summary state. pub fn plain_dtype(&self) -> Option<&DataType>; @@ -1393,15 +1388,17 @@ pub struct SelectionBox { /// unrestricted. /// The timestamp column is one of them; there is no separate time /// field. - pub columns: BTreeMap, + pub columns: BTreeMap, } -/// Names a column across nodes, independent of its position. -pub struct ColumnIdentity { - /// The table or alias the column comes from; `None` if unqualified. - pub table: Option, - /// The column name. - pub name: String, +/// Names a column across nodes, independent of its position and name: the +/// source column of its lineage (`ColumnLineage::SourceColumn`), the same +/// type the lineage holds. +pub struct SourceColumn { + /// The table or metric a `Scan` reads. + pub source: String, + /// The column's name in that source. + pub column: String, } /// A constraint on one column. From 0c11cd7111a75c4d7ff986ad1aa72a086e3e439e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:18:15 +0000 Subject: [PATCH 28/30] refactor(stage1-pass1): per-computation candidates over memo groups (#1207 PR 3) Pass 1 fills Stage 0's memo with each group's initial candidates, every intermediate result its own group and every candidate one operator over input groups: - computation_targets walks memo.groups(): one target per measure of each Aggregate group with a realization besides plain exact execution. - add_pass1_candidates adds every group's own expression, exact accumulators (state group + FinalizeExactAccumulator; avg from sum and count), one SummaryAgg state group per sketch algorithm with one build candidate per configuration of parameter_list(algorithm) and separate SummaryEstimate readout groups, fused builds (whole-expression top-k heaps; nested sum/min/max merges) listing the replaced groups in fused_groups, multi-measure joins, and key-only HAVING pushed below in the consumer Filter group's candidates. - Pass 1 sets no input requirement and no accuracy floor: every candidate's LogicalRequirements are empty; Pass 2 gives each readout its required accuracy (a top-k readout's heap_size >= k included). - ComputationTarget and FusedComputation move into Pass 1 (crate-private), FusedComputation::absorbed_groups replaces target indices, HavingEntry, HavingPlacement and Pass1Error are deleted; pass1_memo returns PerComputationCandidateError. - types: topk::ranked_column_of_group (ranked_column becomes pub(crate)); one unique-key rule for a top-k's ranked rows, exact or a heap readout; any_measure_filtered generic over the predicate's input. - Tests build their memo through the Memo methods; the integration tests stage1_logical_asap_optimization and per_computation_candidates_sql run SQL and PromQL queries through Stage 0 and Pass 1. --- Cargo.lock | 2 - crates/accuracy/src/propagation.rs | 39 + crates/accuracy/src/test_support.rs | 20 +- crates/devtools/Cargo.toml | 2 - crates/devtools/src/planning_stages.rs | 15 +- crates/executor/Cargo.toml | 2 - crates/integration-tests/Cargo.toml | 2 +- crates/integration-tests/tests/main.rs | 1 + .../tests/per_computation_candidates_sql.rs | 196 +- .../tests/stage1_logical_asap_optimization.rs | 631 ++++++ crates/planner/src/run.rs | 18 +- .../Cargo.toml | 4 - .../src/lib.rs | 52 +- .../src/logical_memo_candidates.rs | 32 +- .../src/per_computation_candidates.rs | 1966 ++++++++++++----- .../src/per_computation_candidates_tests.rs | 981 +++++++- .../src/realization.rs | 295 ++- .../src/test_fixtures.rs | 90 +- .../src/test_support.rs | 4 - crates/types/src/ir/operator/asap.rs | 14 +- crates/types/src/ir/operator/non_asap.rs | 12 +- crates/types/src/ir/schema/topk.rs | 228 +- crates/types/src/stages/candidates.rs | 8 +- crates/types/src/stages/mod.rs | 4 +- ...per_computation_logical_memo_candidates.rs | 130 +- .../design_docs/proposals/planner-layering.md | 4 +- 26 files changed, 3812 insertions(+), 940 deletions(-) create mode 100644 crates/integration-tests/tests/stage1_logical_asap_optimization.rs delete mode 100644 crates/stage1-pass1-per-computation-candidates/src/test_support.rs diff --git a/Cargo.lock b/Cargo.lock index 6b77c8a73..afe947361 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -332,8 +332,6 @@ dependencies = [ "asap-executor", "asap-frontend-promql", "asap-planner", - "asap-stage0-memo", - "asap-stage1-pass1-per-computation-candidates", "asap-types", "asap_sketchlib", "chrono", diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index 2a78698b9..d63a81c3e 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -1599,3 +1599,42 @@ pub fn pinned_column(conjunct: &ScalarExpr) -> Option { _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::schema::{NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr}; + + /// A Count-Min estimate has a bound only over weights proven + /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. + #[test] + fn count_min_estimate_has_no_bound_over_an_open_requirement() { + let summary_type = FieldDataType::Sketch(SketchAlgorithm::Cms); + let query = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let with = |weight_domain| SummaryUpdate { + item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain, + }; + assert!(!estimate_has_bound( + &summary_type, + &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), + &query + )); + assert!(!estimate_has_bound( + &summary_type, + &with(WeightDomain::UnknownOrSigned), + &query + )); + assert!(estimate_has_bound( + &summary_type, + &with(WeightDomain::NonNegative { + proof: NonNegativeWeightProof::CounterRawDataSamples + }), + &query + )); + } +} diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs index 9fd42c4a9..f70085555 100644 --- a/crates/accuracy/src/test_support.rs +++ b/crates/accuracy/src/test_support.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::properties::guarantee::AccuracyGuarantee; -use asap_types::ir::schema::{FieldDataType, SketchStatistic, SummaryUpdate}; +use asap_types::ir::schema::{FieldDataType, SketchStatistic}; use asap_types::ir::{OperatorNode, ScalarExpr}; use crate::composition::StreamNorm; @@ -41,24 +41,6 @@ pub fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option { propagation::estimates_raw_data_samples_of(node) } -/// `propagation::estimate_has_bound`. -pub fn estimate_has_bound( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> bool { - propagation::estimate_has_bound(family, update, query) -} - -/// `propagation::estimate_bound_over_raw_data_samples`. -pub fn estimate_bound_over_raw_data_samples( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> Option { - propagation::estimate_bound_over_raw_data_samples(family, update, query) -} - /// `propagation::source_distinct_guarantee`. pub fn source_distinct_guarantee( family: &FieldDataType, diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 1e6c69df1..80079502b 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -37,8 +37,6 @@ tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } -# Pass 1's target list (`per_computation_candidates`) for the tests. -asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } [[test]] diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index 21fea4c96..a93f9fd98 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -7,7 +7,7 @@ use asap_planner::{ run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError, }; use asap_stage0_memo::memo_of_queries; -use asap_stage1_pass1_per_computation_candidates::{pass1_memo, Pass1Error}; +use asap_stage1_pass1_per_computation_candidates::pass1_memo; use asap_stage1_pass2_reuse_candidates::{ pass2_after_local_window_composition, pass2_local_window_composition, }; @@ -253,23 +253,14 @@ pub fn run_stages_until( .deployment_capabilities .supported_summaries .as_deref(), - ); + ) // As `run_planning`: a query Pass 1 cannot plan is reported before // contradictory deployment capabilities. - if let Err(Pass1Error::PerComputation(error)) = &pass1 { - return Err(PlanningStagesError::Pass1(error.to_string())); - } + .map_err(|error| PlanningStagesError::Pass1(error.to_string()))?; if let Err(error) = models.deployment_capabilities.validate(data) { run.selection = Err(SelectionError::InvalidDeploymentCapabilities(error)); return Ok(stages); } - let pass1 = match pass1 { - Ok(pass1) => pass1, - Err(error) => { - run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); - return Ok(stages); - } - }; run.pass1_memo = Some(pass1.clone()); if until == RunUntil::Pass1 { return Ok(stages); diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index 7661ad112..e3706488f 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -27,8 +27,6 @@ test-support = ["planner-types/test-support"] [dev-dependencies] asap-executor = { path = ".", features = ["test-support"] } rmp-serde = "1" -asap-stage0-memo = { path = "../stage0-memo" } -asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } # The planning stages through the facade (a dev-dependency cycle, for # integration tests only). asap-planner = { path = "../planner" } diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index dbe28ea89..8f5628b71 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -10,7 +10,7 @@ asap-types = { path = "../types", features = ["test-support"] } asap-frontend-promql = { path = "../frontend-promql" } asap-frontend-sql = { path = "../frontend-sql" } asap-stage0-memo = { path = "../stage0-memo" } -asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates", features = ["test-support"] } +asap-stage1-pass1-per-computation-candidates = { path = "../stage1-pass1-per-computation-candidates" } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-accuracy = { path = "../accuracy" } asap-cost = { path = "../cost" } diff --git a/crates/integration-tests/tests/main.rs b/crates/integration-tests/tests/main.rs index fa844281e..8c80e1755 100644 --- a/crates/integration-tests/tests/main.rs +++ b/crates/integration-tests/tests/main.rs @@ -30,4 +30,5 @@ mod sharing_issue_regressions; mod sql_cardinality; mod sql_frequency; mod sql_to_post_asap; +mod stage1_logical_asap_optimization; mod time_range; diff --git a/crates/integration-tests/tests/per_computation_candidates_sql.rs b/crates/integration-tests/tests/per_computation_candidates_sql.rs index 4a5a3c6b3..63be8e630 100644 --- a/crates/integration-tests/tests/per_computation_candidates_sql.rs +++ b/crates/integration-tests/tests/per_computation_candidates_sql.rs @@ -1,69 +1,84 @@ -//! SQL → Pass 1 per-computation candidates: where each `HAVING` is applied, -//! the guarantees composed over estimates, and what a Top-K heap sketch -//! ranks. +//! SQL → Pass 1 memo candidates: where each `HAVING` is applied, the +//! guarantees composed over estimates, and what a Top-K heap sketch ranks. use asap_frontend_sql::{lower_sql, SqlCatalog}; -use asap_stage1_pass1_per_computation_candidates::test_support::generate_per_computation_candidates; +use asap_stage1_pass1_per_computation_candidates::pass1_memo; use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::QueryRoot; -use asap_types::stages::{ComputationTarget, HavingPlacement}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SummaryInputExpr}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, QueryRoot}; +use asap_types::stages::Memo; -async fn targets(sql: &str) -> Vec { - let catalog = SqlCatalog::new().with_table( - "flows", - Schema::new(vec![ +/// Pass 1's memo of `sql` over `flows(src, bytes)`. +async fn pass1(sql: &str) -> Memo { + pass1_over( + vec![ Field::plain("src", DataType::Utf8, false), Field::plain("bytes", DataType::Float64, false), - ]), - ); + ], + sql, + ) + .await +} + +/// Pass 1's memo of `sql` over `flows(fields)`. +async fn pass1_over(fields: Vec, sql: &str) -> Memo { + let catalog = SqlCatalog::new().with_table("flows", Schema::new(fields)); let root = lower_sql(sql, &catalog) .await .unwrap_or_else(|e| panic!("lower failed for {sql:?}: {e}")); - generate_per_computation_candidates( - &asap_stage0_memo::memo_of_queries( - &[QueryRoot::Operator(root)], - &[None], - &Default::default(), - ) - .unwrap(), + let memo = asap_stage0_memo::memo_of_queries( + &[QueryRoot::Operator(root)], + &[None], + &Default::default(), ) - .unwrap() - .targets + .unwrap(); + pass1_memo(memo, None).unwrap_or_else(|e| panic!("Pass 1 failed for {sql:?}: {e}")) } -fn placements(targets: &[ComputationTarget]) -> Vec { - targets - .iter() - .flat_map(|target| target.having_filters.iter().map(|entry| entry.placement)) - .collect() +/// Whether some candidate applies a `HAVING` below the aggregate: a +/// `Filter` group over the aggregate's input, which a state build reads. +fn having_pushed_below(memo: &Memo) -> bool { + let groups = memo.groups(); + groups.iter().any(|group| { + let Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) = &group.expression else { + return false; + }; + groups.iter().any(|filter| { + matches!(&filter.expression, Operator::NonASAP(NonASAPOp::Filter { child: input, .. }) if input == child) + }) + }) } +/// A key-only `HAVING` is pushed below the aggregate. #[tokio::test] async fn key_only_having_is_pushed_below() { - let targets = targets("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'").await; - assert_eq!(placements(&targets), vec![HavingPlacement::BelowAggregate]); + let memo = pass1("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'").await; + assert!(having_pushed_below(&memo)); } +/// A `HAVING` over an aggregate value stays above the output. #[tokio::test] async fn value_having_stays_above() { - let targets = - targets("SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10").await; - assert_eq!(placements(&targets), vec![HavingPlacement::AboveOutput]); + let memo = pass1("SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10").await; + assert!(!having_pushed_below(&memo)); } +/// A global aggregate's `HAVING`, even a constant one, stays above the output. #[tokio::test] async fn global_constant_having_stays_above() { - let targets = targets("SELECT count(*) FROM flows HAVING false").await; - assert_eq!(placements(&targets), vec![HavingPlacement::AboveOutput]); + let memo = pass1("SELECT count(*) FROM flows HAVING false").await; + assert!(!having_pushed_below(&memo)); } +/// A `HAVING` that reads a key and a value stays above the output. #[tokio::test] async fn mixed_key_and_value_having_stays_above() { - let targets = - targets("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a' AND count(*) > 10") + let memo = + pass1("SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a' AND count(*) > 10") .await; - assert_eq!(placements(&targets), vec![HavingPlacement::AboveOutput]); + assert!(!having_pushed_below(&memo)); } /// Whether `node` reads a quantile sketch (KLL or DDSketch) anywhere below it. @@ -140,3 +155,112 @@ async fn a_filter_or_group_key_over_an_estimate_has_no_guarantee() { } } } + +/// The item and weight columns of every heap sketch state group in Pass 1's +/// memo of `sql` over `flows(key: key_type, bytes)`. +async fn heap_updates(key: &str, key_type: DataType, sql: &str) -> Vec<(String, String)> { + heap_updates_over( + vec![ + Field::plain(key, key_type, false), + Field::plain("bytes", DataType::Float64, false), + ], + sql, + ) + .await +} + +/// [`heap_updates`] over `flows(fields)`. +async fn heap_updates_over(fields: Vec, sql: &str) -> Vec<(String, String)> { + let memo = pass1_over(fields, sql).await; + let name = |expr: &SummaryInputExpr| match expr { + SummaryInputExpr::Column(ColumnRef::Named(name)) => name.clone(), + // The ranked value when it is the child's one sample value. + SummaryInputExpr::Column(ColumnRef::SampleValue) => "".into(), + SummaryInputExpr::Tuple(items) => match items.as_slice() { + [SummaryInputExpr::Column(ColumnRef::Named(name))] => name.clone(), + other => panic!("one item column: {other:?}"), + }, + SummaryInputExpr::Constant(value) => value.to_string(), + other => panic!("a named column or constant: {other:?}"), + }; + memo.groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: + FieldDataType::Sketch( + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap, + ), + input, + .. + }) => Some((name(input.item.as_ref().unwrap()), name(&input.weight))), + _ => None, + }) + .collect() +} + +const COUNT_RANKED_TOPK: &str = + "SELECT KEY, count(*) FROM flows GROUP BY KEY ORDER BY count(*) DESC LIMIT 10"; + +/// An Int64 key is not mistaken for the ranked value: both heap sketches +/// are built over raw rows, keyed by the key with unit weight. +#[tokio::test] +async fn topk_heap_over_an_int64_key_counts_raw_rows() { + let updates = heap_updates( + "port", + DataType::Int64, + &COUNT_RANKED_TOPK.replace("KEY", "port"), + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "port" && weight == "1"), + "{updates:?}" + ); +} + +/// A key named `value` is not the ranked value: the heap sketches are +/// fed raw rows with unit weight. +#[tokio::test] +async fn topk_heap_over_a_key_named_value_counts_raw_rows() { + let updates = heap_updates( + "value", + DataType::Utf8, + &COUNT_RANKED_TOPK.replace("KEY", "value"), + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "value" && weight == "1"), + "{updates:?}" + ); +} + +/// SQL's per-group top-k idiom (`ROW_NUMBER() OVER (PARTITION BY city +/// ORDER BY COUNT(*) DESC) … WHERE rn <= 5`) is a heavy-hitter top-k, as +/// PromQL `topk by (city)` is: both heap sketches are offered, keyed by the +/// host and fed raw rows with unit weight. +#[tokio::test] +async fn rownumber_per_group_topk_offers_heap_sketches() { + let updates = heap_updates_over( + vec![ + Field::plain("city", DataType::Utf8, false), + Field::plain("host", DataType::Utf8, false), + ], + "SELECT city, host, c FROM (SELECT city, host, COUNT(*) AS c, ROW_NUMBER() OVER \ + (PARTITION BY city ORDER BY COUNT(*) DESC) AS rn FROM flows GROUP BY city, host) \ + WHERE rn <= 5", + ) + .await; + assert_eq!(updates.len(), 2, "{updates:?}"); + assert!( + updates + .iter() + .all(|(item, weight)| item == "host" && weight == "1"), + "{updates:?}" + ); +} diff --git a/crates/integration-tests/tests/stage1_logical_asap_optimization.rs b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs new file mode 100644 index 000000000..59aefcb24 --- /dev/null +++ b/crates/integration-tests/tests/stage1_logical_asap_optimization.rs @@ -0,0 +1,631 @@ +//! Frontends → Pass 1's memo: every candidate Pass 1 adds outputs its +//! group's schema (`Memo::add_candidate` checks it), so every way to compute +//! a query keeps its output columns (names, types and nullability). + +use std::rc::Rc; +use std::sync::Arc; + +use asap_executor::{ + physical_planner::compile, + runtime::{Limits, RunContext, Scope}, + sources::InMemoryDataStore, + values::{Batch, Value}, +}; +use asap_frontend_sql::{lower_sql, SqlCatalog}; +use asap_stage1_pass1_per_computation_candidates::pass1_memo; +use asap_types::ir::physical_export::PhysicalASAPOperatorPayload; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{DataType, Field, Schema}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SummaryInputExpr, SummaryUpdate}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode, QueryRoot, ScalarExpr}; +use asap_types::stages::{Memo, MemoGroupId}; +use asap_types::workload::{ + BatchEntry, DataWorkload, DurationMs, PlanningWorkload, Query, QueryLanguage, + QueryRequirements, QueryWorkload, +}; +use futures::StreamExt; + +fn accuracy() -> AccuracyTarget { + AccuracyTarget::max_error(0.01).failure_probability(0.01) +} + +async fn sql(query: &str) -> QueryRoot { + let catalog = SqlCatalog::new().with_table( + "flows", + Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("dst", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + ); + let root = lower_sql(query, &catalog) + .await + .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")); + QueryRoot::Operator(root) +} + +fn promql(query: &str) -> QueryRoot { + let workload = PlanningWorkload { + query_workload: QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some(vec![BatchEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy: accuracy(), + ..Default::default() + }, + predictability: Default::default(), + invocations: 1, + execute_at: None, + range_evaluation: None, + }]), + repeating_queries: None, + }, + data_workload: Some(DataWorkload::sampled_every( + asap_frontend_promql::promql_metric_names(query).unwrap_or_default(), + DurationMs(1000), + )), + }; + // As the planner does before Stage 0 (`Stage1ToStage3Pass`): each + // series' identity is realized as a column. + match asap_frontend_promql::lower_promql_query_workload(&workload) + .unwrap_or_else(|e| panic!("lower failed for {query:?}: {e}")) + .remove(0) + { + QueryRoot::Operator(node) => QueryRoot::Operator( + asap_types::ir::schema::schema_support::with_promql_series_identity(&node) + .unwrap_or(node), + ), + scalar => scalar, + } +} + +/// Stage 0's memo of the one query `root`, without an accuracy target. +fn stage0(root: QueryRoot) -> Memo { + asap_stage0_memo::memo_of_queries(&[root], &[None], &DataWorkload::default()).unwrap() +} + +/// Pass 1's memo of the one query `root`. +fn pass1(root: QueryRoot) -> Memo { + pass1_memo(stage0(root), None).unwrap() +} + +/// The one `Aggregate` group of `memo`. +fn root_group(memo: &Memo) -> MemoGroupId { + let [group] = aggregate_groups(memo)[..] else { + panic!("one aggregate group expected") + }; + group +} + +/// Pass 1 over `root`; returns how many candidates its aggregate groups +/// have. Each outputs its group's schema, or `Memo::add_candidate` would +/// have rejected it and Pass 1 failed. +fn candidates_of_aggregates(label: &str, root: QueryRoot) -> usize { + let memo = pass1_memo(stage0(root), None).unwrap_or_else(|e| panic!("{label}: {e}")); + aggregate_groups(&memo) + .iter() + .map(|group| memo.groups()[group.0 as usize].candidates.len()) + .sum() +} + +/// The groups of `memo` whose expression is an `Aggregate`. +fn aggregate_groups(memo: &Memo) -> Vec { + memo.groups() + .iter() + .enumerate() + .filter(|(_, group)| { + matches!( + group.expression, + Operator::NonASAP(NonASAPOp::Aggregate { .. }) + ) + }) + .map(|(index, _)| MemoGroupId(index as u32)) + .collect() +} + +#[tokio::test] +async fn sql_candidates_output_the_query_columns() { + for query in [ + "SELECT src, count(*) FROM flows GROUP BY src HAVING src = 'a'", + "SELECT src, count(*) FROM flows GROUP BY src HAVING count(*) > 10", + "SELECT src, avg(bytes), count(*), approx_distinct(dst) FROM flows GROUP BY src", + "SELECT count(*) FILTER (WHERE bytes > 10), approx_percentile_cont(bytes, 0.99) FROM flows", + "SELECT avg(bytes) FROM flows", + "SELECT sum(flows.bytes) FROM flows", + "SELECT src, approx_percentile_cont(bytes, 0.99) FILTER (WHERE bytes > 10) FROM flows GROUP BY src", + "SELECT approx_percentile_cont(bytes, 0.99) FILTER (WHERE bytes > 10) FROM flows", + ] { + assert!(candidates_of_aggregates(query, sql(query).await) > 1, "{query}"); + } +} + +#[test] +fn promql_candidates_output_the_query_columns() { + for query in [ + "avg by (job) (m)", + "avg_over_time(m[5m])", + "quantile(0.99, m) - quantile(0.5, m)", + "scalar(quantile(0.99, m)) - scalar(quantile(0.5, m))", + "topk by (job) (10, sum_over_time(m[1m]))", + "topk(3, sum by (job) (rate(m[5m])))", + "count by (job) (m)", + "count_over_time(m[5m])", + "quantile_over_time(0.99, m[5m])", + ] { + assert!( + candidates_of_aggregates(query, promql(query)) > 1, + "{query}" + ); + } +} + +/// The algorithm and update of every sketch state group of Pass 1's memo +/// of `root`. +fn sketch_builds(root: QueryRoot) -> Vec<(SketchAlgorithm, SummaryUpdate)> { + pass1(root) + .groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::Sketch(kind), + input, + .. + }) => Some((*kind, input.clone())), + _ => None, + }) + .collect() +} + +/// A PromQL top-k over per-series sums offers both heap algorithms, and a +/// count offers Count-Min. +#[test] +fn topk_offers_both_heap_algorithms() { + let topk = sketch_builds(promql("topk by (job) (10, sum_over_time(m[1m]))")); + let algorithms: Vec<_> = topk.iter().map(|(kind, _)| *kind).collect(); + assert_eq!( + algorithms, + [ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap + ] + ); + let count = sketch_builds(promql("count by (job) (m)")); + assert!(count.iter().any(|(kind, _)| *kind == SketchAlgorithm::Cms)); +} + +/// Top-K over a grouped sum is a whole-expression heap: it absorbs the +/// sum and ranks its group keys, read from the rows the sum reads. +#[test] +fn topk_over_an_aggregate_ranks_its_labels() { + let builds = sketch_builds(promql("topk(3, sum by (job) (rate(m[5m])))")); + assert_eq!(builds.len(), 2); + for (_, update) in &builds { + assert_eq!( + update.item, + Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) + ); + } +} + +/// A subquery inside a measure filter is an input group of the filtered +/// state: the outer count's state reads the inner quantile's group, which +/// has its own candidates (exact and the quantile sketches' readouts). +#[tokio::test] +async fn subquery_in_a_measure_filter_is_an_input_group_of_the_state() { + let query = "SELECT src, count(*) FILTER (WHERE bytes > (SELECT approx_percentile_cont(bytes, 0.5) FROM flows)) \ + FROM flows GROUP BY src"; + let memo = pass1(sql(query).await); + let subqueries: Vec = memo + .groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + filter: Some(filter), + .. + }) => Some( + filter + .0 + .operator_refs() + .into_iter() + .copied() + .collect::>(), + ), + _ => None, + }) + .flatten() + .collect(); + assert!(!subqueries.is_empty()); + // The subquery reads the inner quantile's aggregate group. + for subquery in subqueries { + let below = memo.groups()[subquery.0 as usize].expression.children(); + assert!(below.iter().any(|group| { + matches!( + memo.groups()[group.0 as usize].expression, + Operator::NonASAP(NonASAPOp::Aggregate { .. }) + ) && memo.groups()[group.0 as usize].candidates.len() > 1 + })); + } +} + +/// Run the logical DAG `root` over `rows` of its one scanned table, each row +/// given by column name. Rows come back shown and sorted. +fn execute(root: &Rc, rows: &[Vec<(&str, Value)>]) -> Vec> { + let dag = asap_types::test_support::physical_asap_dag_at_query_time(root).unwrap(); + let scan = dag + .nodes + .iter() + .find(|node| { + matches!( + &node.payload, + PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { .. }) + ) + }) + .expect("one scan"); + let PhysicalASAPOperatorPayload::NonASAP(NonASAPOp::Scan { source, .. }) = &scan.payload else { + unreachable!() + }; + let schema = Arc::new(scan.output_schema.clone()); + let plan = compile(&dag, &[], &[dag.roots[0] as u64]).unwrap(); + let rows = rows + .iter() + .map(|row| { + schema + .fields + .iter() + .map(|field| { + row.iter() + .find(|(name, _)| *name == field.name) + .map(|(_, value)| value.clone()) + .expect("value for every column") + }) + .collect() + }) + .collect(); + let store = InMemoryDataStore::default() + .with_raw_data( + source.clone(), + schema.clone(), + vec![Batch::try_new(schema.clone(), rows).unwrap()], + ) + .unwrap(); + let bound = plan.instantiate(Arc::new(store)).unwrap(); + let context = RunContext::new( + Scope::Query { + evaluation_time_ms: 0, + }, + Limits::default(), + ) + .unwrap(); + let show = |value: &Value| match value { + Value::Null => "NULL".to_string(), + Value::Bool(value) => value.to_string(), + Value::Float64(value) => value.to_string(), + Value::Utf8(value) => format!("{value:?}"), + other => format!("{other:?}"), + }; + let mut out: Vec> = futures::executor::block_on(async { + let mut stream = bound.execute(plan.roots(), context).unwrap().remove(0); + let mut rows = Vec::new(); + while let Some(batch) = stream.next().await { + rows.extend( + batch + .unwrap() + .to_rows() + .iter() + .map(|row| row.iter().map(show).collect()), + ); + } + rows + }); + out.sort(); + out +} + +/// SQL `avg` from its `sum` and `count` states: the exact plan and the +/// plan computing `avg` as `sum` and `count`, to execute. Realizing the +/// second from Pass 1's memo is #1207 PR 6. +async fn avg_as_sum_and_count( + catalog: &SqlCatalog, + query: &str, +) -> (Rc, Rc) { + let _ = (catalog, query); + todo!("#1207 PR 6: realize the sum-and-count plan of `avg` from Pass 1's memo") +} + +/// The candidate of `memo`'s root group that is not its own expression +/// and computes `avg` from its `sum` and `count` states, and every group +/// below it. +fn avg_candidate_groups(memo: &Memo) -> Vec { + let root = root_group(memo); + let group = &memo.groups()[root.0 as usize]; + let mut below: Vec = Vec::new(); + let mut to_visit: Vec = group.candidates[1..] + .iter() + .flat_map(|candidate| { + candidate + .expression + .operator + .children() + .into_iter() + .copied() + }) + .collect(); + while let Some(next) = to_visit.pop() { + if !below.contains(&next) { + below.push(next); + to_visit.extend( + memo.groups()[next.0 as usize] + .expression + .children() + .into_iter() + .copied(), + ); + } + } + below +} + +fn avg_table() -> SqlCatalog { + SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain("k", DataType::Utf8, true), + Field::plain("ok", DataType::Bool, false), + Field::plain("value", DataType::Float64, false), + ]), + ) +} + +/// SQL `avg` as `sum` and `count` states executes as the exact aggregate: +/// NULL and '' groups stay apart, a non-Utf8 key works, and an empty global +/// input averages to NULL. +#[tokio::test] +async fn sql_avg_from_sum_and_count_executes_as_avg() { + let catalog = avg_table(); + let row = |k: Option<&str>, ok: bool, value: f64| { + vec![ + ("k", k.map_or(Value::Null, |k| Value::Utf8(k.into()))), + ("ok", Value::Bool(ok)), + ("value", Value::Float64(value)), + ] + }; + let rows = vec![ + row(None, true, 2.), + row(None, true, 4.), + row(Some(""), false, 10.), + row(Some("a"), false, 200.), + row(Some("a"), true, 1.), + ]; + let rows_of = |rows: &[&[&str]]| -> Vec> { + rows.iter() + .map(|row| row.iter().map(|cell| cell.to_string()).collect()) + .collect() + }; + for (query, input, want) in [ + ( + "SELECT k, avg(value) FROM t GROUP BY k", + rows.clone(), + rows_of(&[&["\"\"", "10"], &["\"a\"", "100.5"], &["NULL", "3"]]), + ), + ( + "SELECT ok, avg(value) FROM t GROUP BY ok", + rows.clone(), + rows_of(&[&["false", "105"], &["true", "2.3333333333333335"]]), + ), + ( + "SELECT avg(value) FROM t", + rows.clone(), + rows_of(&[&["43.4"]]), + ), + ("SELECT avg(value) FROM t", vec![], rows_of(&[&["NULL"]])), + ] { + let (exact, root) = avg_as_sum_and_count(&catalog, query).await; + assert_eq!(execute(&exact, &input), want, "{query}"); + assert_eq!(execute(&root, &input), want, "{query}"); + } +} + +/// A filtered SQL `avg` is NULL where no row passes the filter: its quotient +/// is NULL when the count is 0. The runtime executes no filtered state yet, +/// so the shape is checked. +#[tokio::test] +async fn filtered_sql_avg_is_null_without_matching_rows() { + let catalog = avg_table(); + for query in [ + "SELECT k, avg(value) FILTER (WHERE value > 100) FROM t GROUP BY k", + "SELECT avg(value) FILTER (WHERE value > 100) FROM t", + ] { + let exact = lower_sql(query, &catalog).await.unwrap(); + let memo = pass1(QueryRoot::Operator(exact)); + let root = &memo.groups()[root_group(&memo).0 as usize]; + assert!( + root.group_properties.schema.fields.last().unwrap().nullable, + "{query}" + ); + assert!(root.candidates.len() > 1, "{query}"); + let null_when_no_count = memo.groups().iter().any(|group| { + let Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) = &group.expression + else { + return false; + }; + let count = memo.groups()[child.0 as usize] + .group_properties + .schema + .fields + .len() + - 1; + matches!( + &cols.last().unwrap().expr, + ScalarExpr::Case { branches, .. } if matches!( + &branches[..], + [(ScalarExpr::Compare { left, op: CompareOpKind::Eq, right, .. }, + ScalarExpr::Literal(ScalarValue::Null))] + if **left == ScalarExpr::Column(count) + && **right == ScalarExpr::Literal(ScalarValue::Int64(0)) + ) + ) + }); + assert!(null_when_no_count, "{query}"); + } +} + +/// PromQL `avg` divides its `sum` and `count` vectors with a `BinaryOp`. +#[test] +fn promql_avg_divides_sum_and_count_vectors() { + let memo = pass1(promql("avg by (job) (m)")); + let root = &memo.groups()[root_group(&memo).0 as usize]; + assert!(root.candidates.len() > 1); + let kinds: Vec<_> = avg_candidate_groups(&memo) + .iter() + .map(|group| memo.groups()[group.0 as usize].expression.kind_name()) + .collect(); + assert!( + kinds.contains(&"BinaryOp") && !kinds.contains(&"Join"), + "{kinds:?}" + ); +} + +/// `t(k Utf8 NULL, )` as a catalog. +fn table(columns: &[(&str, bool)]) -> SqlCatalog { + let mut fields = vec![Field::plain("k", DataType::Utf8, true)]; + fields.extend( + columns + .iter() + .map(|(name, nullable)| Field::plain(*name, DataType::Float64, *nullable)), + ); + SqlCatalog::new().with_table("t", Schema::new(fields)) +} + +/// Pass 1's memo of `query` over `catalog`. +async fn pass1_sql(catalog: &SqlCatalog, query: &str) -> Memo { + let root = lower_sql(query, catalog).await.unwrap(); + pass1(QueryRoot::Operator(root)) +} + +/// The update of every `count` state (exact or sketch) in `memo`. +fn count_updates(memo: &Memo) -> Vec { + memo.groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type, + input, + .. + }) if !matches!( + summary_type, + FieldDataType::ExactAggregate(asap_types::ir::schema::ExactKind::Sum, _) + ) => + { + Some(input.clone()) + } + _ => None, + }) + .collect() +} + +/// Whether `update` reads only the column `name` of `t`. +fn reads_only(update: &SummaryUpdate, name: &str) -> bool { + let column = SummaryInputExpr::Column(ColumnRef::Named(name.into())); + match &update.item { + Some(item) => *item == column && update.weight == SummaryInputExpr::Constant(1.0), + None => update.weight == column, + } +} + +/// `avg(c)`'s count counts `c`, not a nullable `value` column: the result is +/// the exact `avg`. +#[tokio::test] +async fn sql_avg_counts_its_own_input() { + let catalog = table(&[("value", true), ("c", false)]); + let query = "SELECT k, avg(c) FROM t GROUP BY k"; + let (exact, root) = avg_as_sum_and_count(&catalog, query).await; + let row = |k: &str, value: Option, c: f64| { + vec![ + ("k", Value::Utf8(k.into())), + ("value", value.map_or(Value::Null, Value::Float64)), + ("c", Value::Float64(c)), + ] + }; + let rows = vec![ + row("a", None, 1.), + row("a", Some(5.), 3.), + row("b", None, 4.), + ]; + let want = vec![ + vec!["\"a\"".to_string(), "2".to_string()], + vec!["\"b\"".to_string(), "4".to_string()], + ]; + assert_eq!(execute(&exact, &rows), want); + assert_eq!(execute(&root, &rows), want); +} + +/// SQL `avg` as `sum` and `count` keeps an explicit output alias: its +/// candidate outputs the root group's schema, which names it. +#[tokio::test] +async fn sql_avg_keeps_its_explicit_output_name() { + let memo = pass1_sql(&avg_table(), "SELECT avg(value) AS avg_latency FROM t").await; + assert!(memo.groups()[root_group(&memo).0 as usize].candidates.len() > 1); + let QueryRoot::Operator(query_root) = &memo.queries[0].root else { + panic!("operator root expected") + }; + let output = &memo.groups()[query_root.0 as usize].group_properties.schema; + assert_eq!(output.fields.last().unwrap().name, "avg_latency"); +} + +/// SQL `avg` of a non-null Int64 column as `sum` and `count` keeps the exact plan's Float64 columns. +#[tokio::test] +async fn sql_avg_of_an_int64_column_keeps_the_exact_columns() { + let catalog = SqlCatalog::new().with_table( + "t", + Schema::new(vec![ + Field::plain("k", DataType::Utf8, true), + Field::plain("n", DataType::Int64, false), + ]), + ); + let query = "SELECT k, avg(n) FROM t GROUP BY k"; + let (exact, root) = avg_as_sum_and_count(&catalog, query).await; + assert_eq!( + root.schema.fields.last().unwrap().plain_dtype(), + Some(&DataType::Float64) + ); + let row = |k: &str, n: i64| vec![("k", Value::Utf8(k.into())), ("n", Value::Int64(n))]; + let rows = vec![row("a", 1), row("a", 2), row("b", 4)]; + let want = vec![ + vec!["\"a\"".to_string(), "1.5".to_string()], + vec!["\"b\"".to_string(), "4".to_string()], + ]; + assert_eq!(execute(&exact, &rows), want); +} + +/// `count(*)` with no column known to be non-null (one nullable numeric +/// column, or several and none named `value`) gets no accumulator or +/// sketch: the root group keeps only its plain exact aggregate, not a count +/// of a nullable column. +#[tokio::test] +async fn sql_count_star_without_a_non_null_column_is_only_exact() { + for columns in [&[("v", true)][..], &[("a", true), ("b", true)][..]] { + let memo = pass1_sql(&table(columns), "SELECT k, count(*) FROM t GROUP BY k").await; + assert_eq!( + memo.groups()[root_group(&memo).0 as usize].candidates.len(), + 1, + "{columns:?}" + ); + assert!(count_updates(&memo).is_empty(), "{columns:?}"); + } +} + +/// `count(v)` of a nullable `v` updates on `v` under its `v IS NOT NULL` +/// filter: it counts the non-null values. The runtime executes no filtered +/// state yet, so the shape is checked. +#[tokio::test] +async fn sql_count_of_a_nullable_column_counts_its_values() { + let catalog = table(&[("v", true), ("n", false)]); + let memo = pass1_sql(&catalog, "SELECT k, count(v) FROM t GROUP BY k").await; + let updates = count_updates(&memo); + assert!(!updates.is_empty()); + for update in &updates { + assert!(reads_only(update, "v"), "{updates:?}"); + } +} diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 337cb032c..4335f8609 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -8,9 +8,7 @@ use asap_stage0_memo::memo_of_queries; use std::rc::Rc; use std::time::Instant; -use asap_stage1_pass1_per_computation_candidates::{ - pass1_memo, Pass1Error, PerComputationCandidateError, -}; +use asap_stage1_pass1_per_computation_candidates::{pass1_memo, PerComputationCandidateError}; use asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates; use asap_stage2_physical_optimization::generate_physical_asap_memo_candidates; use asap_stage3_plan_selection::{select_plan, ErrorSelectingPlan, Selection}; @@ -107,24 +105,20 @@ pub fn run_planning( }) .collect(); let initial = memo_of_queries(&roots, &accuracy_targets, data)?; - // Pass 1: the per-computation candidates the deployment can run. + // Pass 1: the initial candidates the deployment can run. A query Pass 1 + // cannot plan is an input error. let pass1 = pass1_memo( initial, models .deployment_capabilities .supported_summaries .as_deref(), - ); - // A query Pass 1 cannot plan is an input error. - let pass1 = match pass1 { - Err(Pass1Error::PerComputation(error)) => return Err(error), - pass1 => pass1, - }; + )?; let mut report = PlanningReport { pass1_ms: elapsed_ms(started), ..PlanningReport::default() }; - let pass1_memo = pass1.as_ref().ok().filter(|_| settings.inspect).cloned(); + let pass1_memo = settings.inspect.then(|| pass1.clone()); let mut logical_asap_candidates = None; let mut physical_asap_candidates = None; let selection = (|| { @@ -132,8 +126,6 @@ pub fn run_planning( .deployment_capabilities .validate(data) .map_err(SelectionError::InvalidDeploymentCapabilities)?; - let pass1 = - pass1.map_err(|error| SelectionError::LogicalASAPCandidates(error.to_string()))?; // Pass 2: the reuse rules, readout requirements and logical edges. let started = Instant::now(); let logical = generate_logical_asap_memo_candidates(pass1, entries, models.evidence); diff --git a/crates/stage1-pass1-per-computation-candidates/Cargo.toml b/crates/stage1-pass1-per-computation-candidates/Cargo.toml index 0a4df4c1a..6df7c52df 100644 --- a/crates/stage1-pass1-per-computation-candidates/Cargo.toml +++ b/crates/stage1-pass1-per-computation-candidates/Cargo.toml @@ -13,10 +13,6 @@ asap-types = { path = "../types" } asap-accuracy = { path = "../accuracy" } thiserror = "2" -[features] -# The test helpers (`test_support`) other crates' tests share. -test-support = [] - [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } asap-accuracy = { path = "../accuracy", features = ["test-support"] } diff --git a/crates/stage1-pass1-per-computation-candidates/src/lib.rs b/crates/stage1-pass1-per-computation-candidates/src/lib.rs index 06aa8d712..26bcdb057 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/lib.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/lib.rs @@ -1,49 +1,37 @@ //! `asap-stage1-pass1-per-computation-candidates` — #509 Stage 1, Pass 1: -//! per-computation candidate generation and the summary realizations it -//! offers. Its one entry is [`pass1_memo`], which fills the -//! memo groups of Stage 0's [`Memo`](asap_types::stages::Memo) with their -//! initial candidates; Pass 2 extends that memo. +//! each memo group's initial candidates. Its one entry is [`pass1_memo`], +//! which fills the memo groups of Stage 0's +//! [`Memo`](asap_types::stages::Memo) with their initial candidates (exact, +//! exact accumulator, one summary build per listed parameter configuration +//! with separate readouts, fused builds), every intermediate result its own +//! group; Pass 2 extends that memo. //! //! It depends only on `asap-types` (its input and output memo are in -//! `asap_types::stages`) and the shared accuracy model (`asap-accuracy`). -//! It depends on no other stage crate, and the planner facade calls it -//! after Stage 0. It depends on no query-language frontend either; its own -//! tests lower PromQL through `asap-frontend-promql` (a dev-dependency). -//! The `test-support` feature exports the Pass 1 internals other crates' -//! tests call (`test_support`). +//! `asap_types::stages`) and the shared accuracy model (`asap-accuracy`, +//! for the statistic each intent reads). It depends on no other stage +//! crate, and the planner facade calls it after Stage 0. It depends on no +//! query-language frontend either; its tests build their input memo through +//! `Memo::find_or_add_group` and `Memo::add_query`. //! -//! Accuracy guarantees come from the built-in analytical accuracy rules in -//! `asap-accuracy` (per-family estimators, composition and budget -//! allocation). They derive a machine-readable -//! [`AccuracyGuarantee`](asap_types::ir::properties::AccuracyGuarantee) -//! for every finalized value and propagate it through -//! approximate-over-approximate compositions under conservative rules (no -//! independence assumptions, unknown statistics stay unknown). A -//! composition with no rule leaves the guarantee `None` and keeps the -//! candidate. Pass 1 prunes nothing for accuracy; Pass 2 gives each readout -//! its required accuracy and eliminates the candidates no query can use, -//! with the deployment's measured accuracy (`AccuracyEvidence`) read before -//! each summary's theoretical guarantee. +//! Pass 1 reads no accuracy target and sets no input requirement: the error +//! of each build follows from its parameters (`parameter_list`), and Pass 2 +//! gives each readout its required accuracy, decides the logical edges and +//! eliminates the candidates no query can use. /// Stage 1, Pass 1's entry: the initial candidates of the memo groups. mod logical_memo_candidates; -/// Stage 1, Pass 1's computation targets and their candidates. No execution timing is assigned: that is a Stage 2 -/// materialization decision. Its target list -/// (`PerComputationCandidates`) is Pass 1's working form, public only to -/// tests (`crate::test_support`, feature `test-support`). +/// Stage 1, Pass 1's computation targets over the memo groups and the +/// candidates that compute them. mod per_computation_candidates; -/// Frontend-to-Pass-1 acceptance tests over Pass 1's working form. +/// Pass 1's tests over memos built through the `Memo` methods. #[cfg(test)] mod per_computation_candidates_tests; -/// The summary families that can serve an aggregate intent, the input a +/// The sketch algorithms that can serve an aggregate intent, the input a /// summary consumes and the column a top-k ranks. mod realization; /// Fixtures this crate's own tests share. #[cfg(test)] mod test_fixtures; -/// Test helpers other crates' tests call (feature `test-support`). -#[cfg(any(test, feature = "test-support"))] -pub mod test_support; -pub use logical_memo_candidates::{pass1_memo, Pass1Error}; +pub use logical_memo_candidates::pass1_memo; pub use per_computation_candidates::PerComputationCandidateError; diff --git a/crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs index c1c024dfd..9ab5c67b8 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/logical_memo_candidates.rs @@ -3,20 +3,12 @@ //! configuration, fused builds), with every intermediate result its own //! group (a [`Memo`]), which Pass 2 extends. -use crate::per_computation_candidates::{ - generate_per_computation_candidates, PerComputationCandidateError, -}; use asap_types::deployment_capabilities::SupportedSummary; use asap_types::stages::Memo; -/// Why [`pass1_memo`] failed. -#[derive(Debug, thiserror::Error)] -pub enum Pass1Error { - /// A query's DAG cannot be planned: it already holds ASAP or physical - /// annotations, or a measure's accuracy target is invalid. - #[error(transparent)] - PerComputation(#[from] PerComputationCandidateError), -} +use crate::per_computation_candidates::{ + add_pass1_candidates, computation_targets, PerComputationCandidateError, +}; /// Stage 1, Pass 1: the initial candidates of the memo groups of `memo` /// (Stage 0's, without candidates). It finds one computation target per @@ -25,14 +17,16 @@ pub enum Pass1Error { /// (`supported_summaries`, the deployment's /// `DeploymentCapabilities::supported_summaries`; `None` supports every /// one), and adds each way to compute a group as candidates, through -/// `Memo::find_or_add_group` and `Memo::add_candidate`. Accuracy prunes -/// nothing here: Pass 2's dead-candidate elimination does. +/// `Memo::find_or_add_group` and `Memo::add_candidate`: every group's own +/// expression, exact accumulators, one summary build per configuration of +/// the algorithm's predefined list with separate readouts, and fused builds. +/// Accuracy prunes nothing here: Pass 2 gives each readout its required +/// accuracy and eliminates the candidates no query can use. pub fn pass1_memo( - memo: Memo, + mut memo: Memo, supported_summaries: Option<&[SupportedSummary]>, -) -> Result { - let per_computation = generate_per_computation_candidates(&memo)? - .without_unsupported_summaries(supported_summaries); - let _ = (memo, per_computation); - todo!("#1207 PR 3: add each way to compute as candidates, one per listed θ") +) -> Result { + let targets = computation_targets(&memo, supported_summaries)?; + add_pass1_candidates(&mut memo, &targets)?; + Ok(memo) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs index 52d9424a5..1bb79cb78 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs @@ -1,68 +1,120 @@ -//! Stage 1 (logical ASAP-aware optimization), Pass 1: per-computation -//! candidate generation, as defined in -//! `docs/design_docs/proposals/planner-layering.md`. +//! Stage 1, Pass 1: each memo group's initial candidates (concepts doc +//! `memo-groups-properties-requirements.md`, section 3, "Stage 1, Pass 1"; +//! #1203, #1207). //! -//! [`generate_per_computation_candidates`] walks the memo groups of the -//! workload's memo groups ([`Memo`]) and -//! returns, for every eligible computation, its unranked candidates. A -//! computation ([`ComputationTarget`]) is one measure of an `Aggregate` -//! group whose intent has a realization besides plain exact execution, once -//! however many queries read the group; nested aggregates and operators -//! read by scalar roots are found too. -//! Pass 1 does not consider sharing across computations; that is Pass 2. +//! [`computation_targets`] walks the memo groups and finds, for every +//! aggregate group, one target ([`ComputationTarget`]) per measure whose +//! intent has a realization besides plain exact execution, once however +//! many queries read the group. [`add_pass1_candidates`] then adds each way +//! to compute a group as candidates, through `Memo::find_or_add_group` and +//! `Memo::add_candidate`: //! -//! - A candidate says how a target is computed: exact or a summary -//! ([`HowToComputeAggregateMeasure`]). Each target has exactly one exact option: -//! its exact accumulator states (`ExactAccumulator`; `[Sum, Count]` for -//! `avg`) when they can be built, otherwise plain `Exact`. Whether they are -//! materialized is decided in Stage 2. -//! - A computation that fuses a target with the targets beneath it is the -//! outer target's [`FusedComputation`], which absorbs the ones beneath and -//! has its own candidates: the whole-expression top-k over a per-item -//! `sum` or `count` (`Summary` of a heap family) and the nested sum, a -//! grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). -//! - Summary choices here are unsized families. Adding them to memo groups -//! creates one build candidate per listed parameter configuration -//! (`parameter_list`); Pass 2 gives readouts their required accuracy. -//! - Candidates are pruned only when provably unable to run, or when the -//! exact candidate provably dominates them (a PromQL top-k's heap over its -//! instant vector, one row per item); accuracy prunes nothing here (Pass -//! 2's dead-candidate elimination does). -//! - A measure filter or `HAVING` prunes nothing. The target records the -//! measure filter (applied as `SummaryAgg.filter`) and one [`HavingEntry`] -//! per consumer `Filter` directly above the aggregate: a key-only predicate -//! applied by every consumer of the aggregate's group, across queries, may go -//! below the aggregate, anything else stays above the output (approximate -//! outputs then get guarantee `None`). So every candidate of a group -//! computes the group's own rows. +//! - **Exact.** Every group's own expression is a candidate of it. +//! - **Exact accumulator.** An accumulator-state group +//! (`SummaryAgg` of an exact summary type) and a `FinalizeExactAccumulator` +//! candidate over it; `avg` finalizes its `sum` and `count` states and +//! divides them. +//! - **Summary.** One state group per sketch algorithm (`SummaryAgg` of +//! it), with one build candidate per configuration of the algorithm's predefined +//! list (`parameter_list`), and a readout (`SummaryEstimate`) group over +//! it, read by the result group. +//! - **Fused builds.** A whole-expression top-k (a heap over the rows below +//! a per-item `sum` or `count`) and a nested distributive aggregate (one +//! exact `Sum`, `Min` or `Max` state over the rows below a `sum` of a +//! `sum`, a `min` of a `min` or a `max` of a `max`): each computes several +//! operators of one appearance as one operator, and its builds list the +//! groups of the operators it replaces in `fused_groups` (concepts doc +//! §3, "Fused groups"). //! -//! The input must be a logical DAG: ASAP operators are rejected. A logical -//! DAG has no physical properties (#1200). No cost or accuracy model is -//! used; ranking and discarding valid candidates belong to Stage 3. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::input::OperatorInput; -use asap_types::ir::operator::operator_properties::Reduction; +//! Every intermediate result is its own group, and every candidate is one +//! operator over input groups. A candidate outputs its group's schema +//! exactly (`add_candidate` checks it), so a readout whose column names +//! differ from the group's is read through a renaming `Project` group. +//! $\boldsymbol\theta$ lives only in `LogicalProperties::parameters`, never +//! in an operator. +//! +//! Pass 1 sets no input requirement: each candidate's are empty, and Pass 2 +//! gives each readout's input its required accuracy (a top-k readout's +//! `heap_size` ≥ k is part of that rule; concepts doc, section 2). +//! +//! A measure filter or `HAVING` prunes nothing. The measure filter is +//! applied as `SummaryAgg.filter`. A `HAVING` is the aggregate's own +//! `having` or a consumer `Filter` group directly above the aggregate group, +//! read from the memo: a key-only predicate ([`key_only_having`]) is pushed +//! below the aggregate by the candidates that compute the filtered result +//! (the aggregate group, or that `Filter` group); any other stays a `Filter` +//! above the output. +//! +//! The input must be a logical memo: a group whose expression is an ASAP +//! operator is rejected. No cost model and no accuracy target is read. +use std::collections::BTreeMap; + +use asap_types::ir::operator::non_asap::any_measure_filtered; +use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{ - ExactKind, FieldDataType, SketchAlgorithm, StatisticKind, SummaryInputExpr, SummaryUpdate, + DataType, EntityIdentity, ExactKind, FieldDataType, NonNegativeWeightProof, Schema, + SketchAlgorithm, SketchParams, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, }; use asap_types::ir::{ - NonASAPOp, OperatorNode, OperatorResultKind, Predicate, QueryRoot, SchemaDerivationError, + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, + ProjectItem, ScalarExpr, SchemaDerivationError, +}; +use asap_types::stages::{ + parameter_list, CandidateError, CandidateExpression, CandidateForMemoGroup, + HowToComputeAggregateMeasure, LogicalProperties, LogicalPropertiesError, LogicalRequirements, + Memo, MemoGroupId, OriginOfCandidateForMemoGroup, }; use thiserror::Error; use asap_accuracy::sizing::sketch_statistic; - -use crate::realization::summary_candidates; use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; -use asap_types::stages::{ - ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, - HowToComputeAggregateMeasure, Memo, MemoGroupId, + +use crate::realization::{ + expression, raw_data_samples_of_metric, realize_keyed_additive_summary_input, schema, + summary_candidates, }; +/// One measure of an aggregate group with at least one realization besides +/// exact execution. +#[derive(Debug, Clone)] +pub(crate) struct ComputationTarget { + /// The memo group of the `Aggregate` the measure belongs to. + pub aggregate_group: MemoGroupId, + /// Index into the aggregate's measures. + pub measure_index: usize, + /// The measure's unranked ways to compute it, the exact option first. + pub ways_to_compute: Vec, + /// The fused computation of this measure together with the target + /// directly beneath it, offered beside this target's own candidates; + /// `None` when no fusion applies. + pub fused_computation: Option, +} + +/// A target's measure computed together with the targets beneath it, as +/// one operator that replaces their operators in this appearance of the +/// sub-expression (concepts doc §3, "Fused groups"): the whole-expression +/// top-k over a per-item `sum` or `count` (`Summary` of a heap algorithm), or +/// a nested distributive aggregate, a grouped `sum`, `min` or `max` over the +/// same reduction (`ExactAccumulator([Sum])`, `[Min]` or `[Max]`). It has +/// the outer target's aggregate, measure, filter and `HAVING`. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct FusedComputation { + /// The groups of the operators it replaces, which its builds list in + /// `fused_groups`: the aggregate directly beneath, then, for a + /// whole-expression heap over the raw data samples beneath a grouped sum + /// of per-series sums or counts (`topk(k, sum by (G) + /// (sum_over_time(m[W])))`), the per-series aggregate beneath that, then + /// the groups any of these targets' own fused computations replace. + pub absorbed_groups: Vec, + /// How the fused computation is computed: exact or a summary. Empty + /// when Pass 1 pruned every one (today only for an algorithm the deployment + /// does not support); the fusion is then not offered. + pub ways_to_compute: Vec, +} + /// Unranked candidates of one measure and the candidates pruned for it. #[derive(Debug, Clone, PartialEq)] pub(super) struct ComputationChoices { @@ -99,140 +151,21 @@ fn prune_own( pruned } -/// Compact Pass 1 inventory over the memo groups it walked: one target per -/// measure of each aggregate group, however many queries read the group. -/// Pass 1's working form, not its output: -/// [`pass1_memo`](crate::pass1_memo) -/// builds, prunes and attaches it to the memo groups. Public only to tests -/// (feature `test-support`). -#[derive(Debug, Clone)] -pub struct PerComputationCandidates { - /// Each query's root over the memo groups' logical nodes, in workload - /// order: a group several queries read is one node. Whole-workload - /// candidates are these roots with chosen candidates substituted - /// (test support only). - #[cfg(any(test, feature = "test-support"))] - pub roots: Vec, - /// Each memo group's logical node, by group index. - pub(crate) logical_nodes: Rc<[Rc]>, - /// The targets; a whole-workload choice has one digit per target, - /// ranging over [`ComputationTarget::choice_count`]. - pub targets: Vec, -} - -impl PerComputationCandidates { - /// The `Aggregate` node of `target`'s memo group. - #[cfg(any(test, feature = "test-support"))] - pub fn aggregate(&self, target: &ComputationTarget) -> &Rc { - &self.logical_nodes[target.aggregate_group.0 as usize] - } - - /// The number of whole-workload candidates: the size of the Cartesian - /// product of the targets' candidates, where a target with a fused - /// computation and the targets beneath that it absorbs count as their - /// own combinations plus the fused computation's candidates (the - /// absorbed targets are not computed then), saturating at `usize::MAX`. - /// A target with `n` candidates and `f` fused ones counts `n · Π m + f`, - /// the product over the targets it absorbs that no other of them - /// absorbs, each counted the same way. It is exactly the number of - /// choices enumeration keeps ([`Self::absorbed_choice_varies`]). - #[cfg(any(test, feature = "test-support"))] - pub fn combinations(&self) -> usize { - let absorbed: HashSet = absorbed_targets(&self.targets).collect(); - (0..self.targets.len()) - .filter(|index| !absorbed.contains(index)) - .map(|index| self.combinations_of(index)) - .fold(1usize, usize::saturating_mul) - } - - /// [`Self::combinations`] of target `index` with the targets it absorbs. - #[cfg(any(test, feature = "test-support"))] - fn combinations_of(&self, index: usize) -> usize { - let target = &self.targets[index]; - let n = target.ways_to_compute.len(); - let Some(fused) = &target.fused_computation else { - return n; - }; - let absorbed_by_another = |a: usize| { - fused.absorbed_targets.iter().any(|&b| { - self.targets[b] - .fused_computation - .as_ref() - .is_some_and(|inner| inner.absorbed_targets.contains(&a)) - }) - }; - fused - .absorbed_targets - .iter() - .filter(|&&a| !absorbed_by_another(a)) - .map(|&a| self.combinations_of(a)) - .fold(n, usize::saturating_mul) - .saturating_add(fused.ways_to_compute.len()) - } - - /// Whether `choice` picks a fused computation and a candidate other - /// than the first for a target it absorbs: the same whole-workload - /// candidate as the one with the first, so enumeration skips it. - #[cfg(any(test, feature = "test-support"))] - pub fn absorbed_choice_varies(&self, choice: &[usize]) -> bool { - self.targets.iter().zip(choice).any(|(target, &index)| { - target - .computation_for(index) - .and_then(|(_, absorbed)| absorbed) - .is_some_and(|inner| inner.iter().any(|&i| choice[i] != 0)) - }) - } - - /// This inventory without the summary candidates the executor does not - /// support (`supported_summaries`, the deployment's - /// `DeploymentCapabilities::supported_summaries`; `None` supports - /// every one): a target's own are removed, and so are a fused - /// computation's (with all of them removed, the fusion is not offered). - /// The readout is the statistic the measure's estimate reads (the outer measure's for a - /// fused computation). With every summary supported it is unchanged. - pub fn without_unsupported_summaries( - mut self, - supported_summaries: Option<&[SupportedSummary]>, - ) -> Self { - for target in self.targets.iter_mut() { - let aggregate = &self.logical_nodes[target.aggregate_group.0 as usize]; - let Some(NonASAPOp::Aggregate { measures, .. }) = aggregate.non_asap() else { - continue; - }; - let Some(readout) = sketch_statistic(&measures[target.measure_index]).map(|s| s.kind()) - else { - continue; - }; - let unsupported = |candidate: &HowToComputeAggregateMeasure| { - matches!(candidate, HowToComputeAggregateMeasure::Summary(family) - if !summary_supported(supported_summaries, family, readout)) - }; - prune_own(&mut target.ways_to_compute, unsupported); - if let Some(fused) = &mut target.fused_computation { - fused.ways_to_compute.retain(|way| !unsupported(way)); - } - } - self - } -} - -/// The indices of `targets` some target's fused computation absorbs. -#[cfg(any(test, feature = "test-support"))] -fn absorbed_targets(targets: &[ComputationTarget]) -> impl Iterator + '_ { - targets - .iter() - .filter_map(|target| target.fused_computation.as_ref()) - .flat_map(|fused| fused.absorbed_targets.iter().copied()) -} - +/// Why Pass 1 could not add a group's candidates. #[derive(Debug, Error)] pub enum PerComputationCandidateError { #[error(transparent)] Structure(#[from] SchemaDerivationError), #[error("logical candidate input already contains ASAP operators")] ContainsAsap, - #[error("choice must name one listed candidate per target")] - InvalidChoice, + /// A candidate Pass 1 built breaks the node rules or does not output + /// its group's result: a bug in the rule that built it. + #[error(transparent)] + Candidate(#[from] CandidateError), + /// A candidate Pass 1 built has logical properties that cannot go + /// together: a bug in the rule that built it. + #[error(transparent)] + LogicalProperties(#[from] LogicalPropertiesError), #[error("unsupported substitution in logical ASAP-aware optimization: {0}")] Unsupported(&'static str), } @@ -279,28 +212,23 @@ pub(super) fn computation_candidates_for_intent(intent: &AggIntent) -> Computati /// Whether an `avg` measure's input may be null, so `count` rows differ from /// counted values. An unresolvable input is treated as possibly null. -fn nullable_avg_input(node: &OperatorNode, intent: &AggIntent) -> bool { - let Some(NonASAPOp::Aggregate { child, .. }) = node.non_asap() else { - return true; - }; +fn nullable_avg_input(input: &Schema, intent: &AggIntent) -> bool { let column = match intent.input_cols().first() { Some(column) => Some(*column), - None => resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok(), + None => resolve_column_ref(&ColumnRef::SampleValue, input).ok(), }; column - .and_then(|column| child.schema.fields.get(column)) + .and_then(|column| input.fields.get(column)) .is_none_or(|field| field.nullable) } -/// Whether `having` over `aggregate`'s output reads only `by(...)` group keys. -/// A global aggregate (no group keys) never qualifies: its one output row -/// exists even when no input row does, so no row predicate can stand in. -fn key_only_having(aggregate: &OperatorNode, having: &Predicate) -> bool { - let keys = match aggregate.non_asap() { - Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) if !by.is_without() => by.len(), +/// Whether `having` over an aggregate's output reads only its `by(...)` +/// group keys. A global aggregate (no group keys) never qualifies: its one +/// output row exists even when no input row does, so no row predicate can +/// stand in. +fn key_only_having(reduction: &Reduction, having: &Predicate) -> bool { + let keys = match reduction { + Reduction::Reduce(by) if !by.is_without() => by.len(), _ => 0, }; keys > 0 @@ -308,131 +236,68 @@ fn key_only_having(aggregate: &OperatorNode, having: &Predicate) -> bool { && having.0.columns_referenced().iter().all(|id| *id < keys) } -fn placement(below: bool) -> HavingPlacement { - if below { - HavingPlacement::BelowAggregate - } else { - HavingPlacement::AboveOutput - } -} - -/// Each memo group's expression as an operator-node tree, by group id: the -/// trees target discovery still walks. `Memo::logical_node_trees` is -/// deleted; discovery reads the groups in #1207 PR 3. -fn logical_node_trees(memo: &Memo) -> Vec> { - let _ = memo; - todo!("#1207 PR 3: discover targets over the memo groups, not node trees") -} - -/// Discover one target per measure of each eligible aggregate memo group of -/// `dag`, including operator plans read by scalar roots: Pass 1 walks the -/// groups, so an aggregate several queries read is one target per measure. -/// An aggregate measure is eligible when it has a realization besides exact -/// execution, even if every such realization is pruned; the prune reasons -/// are reported on the target. Measure filters and `HAVING` keep every -/// candidate; the target records how they are applied. A group's consumers -/// are its parent groups and the query roots that read it; a consumer -/// `Filter`'s key-only `HAVING` is pushed below only when every consumer of -/// the group applies the same predicate, so every candidate of a shared -/// aggregate computes the group's own rows (#1089). -pub fn generate_per_computation_candidates( +/// The consumer `Filter` groups directly above `aggregate_group` (its +/// `HAVING` as the SQL frontend lowers it), with their predicates. A +/// `Filter` that reads the aggregate from a scalar expression stays a plain +/// consumer. +fn having_filter_groups( memo: &Memo, -) -> Result { - let logical_nodes: Rc<[Rc]> = logical_node_trees(memo).into(); - let group_of: HashMap<*const OperatorNode, MemoGroupId> = logical_nodes - .iter() - .enumerate() - .map(|(g, node)| (Rc::as_ptr(node), MemoGroupId(g as u32))) - .collect(); - // Each query's root over the groups' logical nodes, in workload order: - // a group several queries read is one node. - let node = |g: &MemoGroupId| Rc::clone(&logical_nodes[g.0 as usize]); - let roots: Vec = memo - .queries + aggregate_group: MemoGroupId, +) -> impl Iterator)> { + memo.groups()[aggregate_group.0 as usize] + .parent_groups() .iter() - .map(|query| match &query.root { - QueryRoot::Operator(g) => QueryRoot::Operator(node(g)), - QueryRoot::Scalar(expr) => QueryRoot::Scalar(expr.map_operator_refs(&mut |g| node(g))), - }) - .collect(); - let mut seen = HashSet::new(); - // Each aggregate group. - let mut aggregates = Vec::new(); - // Every consumer of each group's node: `Some(parent)`, or `None` for a - // query root or scalar reference. - let mut consumers: HashMap<*const OperatorNode, Vec>>> = HashMap::new(); - for root in &roots { - root.validate_structure()?; - let operators = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - for root in operators { - if root.contains_asap() { - return Err(PerComputationCandidateError::ContainsAsap); + .filter_map(move |&parent| match expression(memo, parent) { + Operator::NonASAP(NonASAPOp::Filter { child, pred }) if *child == aggregate_group => { + Some((parent, pred)) } - consumers.entry(Rc::as_ptr(root)).or_default().push(None); - for node in OperatorNode::reachable(root) { - if !seen.insert(Rc::as_ptr(&node)) { - continue; - } - for child in node.children() { - consumers - .entry(Rc::as_ptr(child)) - .or_default() - .push(Some(Rc::clone(&node))); - } - if matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })) { - aggregates.push(node); - } - } - } + _ => None, + }) +} + +/// Whether `aggregate_group` has a `HAVING`: its own `having` or a consumer +/// `Filter` group directly above it ([`having_filter_groups`]). +fn has_having(memo: &Memo, aggregate_group: MemoGroupId) -> bool { + matches!( + expression(memo, aggregate_group), + Operator::NonASAP(NonASAPOp::Aggregate { + having: Some(_), + .. + }) + ) || having_filter_groups(memo, aggregate_group).next().is_some() +} + +/// Discover one target per measure of each eligible aggregate group of +/// `memo`, nested aggregates and those read by scalar roots included: +/// Pass 1 walks the groups, so an aggregate several queries read is one +/// target per measure. An aggregate measure is eligible when it has a +/// realization besides exact execution, even if every such realization is +/// pruned. The summaries the deployment does not support +/// (`supported_summaries`, the deployment's +/// `DeploymentCapabilities::supported_summaries`; `None` supports every +/// one) are dropped: a target's own, and a fused computation's (with all +/// of them dropped, the fusion is not offered). Measure filters and +/// `HAVING` keep every candidate. +pub(crate) fn computation_targets( + memo: &Memo, + supported_summaries: Option<&[SupportedSummary]>, +) -> Result, PerComputationCandidateError> { + if memo + .groups() + .iter() + .any(|group| matches!(group.expression, Operator::ASAP(_))) + { + return Err(PerComputationCandidateError::ContainsAsap); } let mut targets = Vec::new(); - for node in aggregates { - let Some(NonASAPOp::Aggregate { - measures, - filters, - having, - .. - }) = node.non_asap() + for (index, group) in memo.groups().iter().enumerate() { + let Operator::NonASAP(NonASAPOp::Aggregate { + child, measures, .. + }) = &group.expression else { continue; }; - let aggregate_group = group_of[&Rc::as_ptr(&node)]; - let mut entries: Vec = having - .iter() - .map(|having| HavingEntry { - filter: None, - predicate: having.clone(), - placement: placement(key_only_having(&node, having)), - }) - .collect(); - let parents = &consumers[&Rc::as_ptr(&node)]; - // Only a `Filter` whose input is this aggregate is its `HAVING`; one - // that reads it from a scalar expression stays a plain consumer. - let mut filters_above: Vec<(&Rc, &Predicate)> = Vec::new(); - for parent in parents.iter().flatten() { - if let Some(NonASAPOp::Filter { child, pred }) = parent.non_asap() { - if Rc::ptr_eq(child, &node) - && !filters_above - .iter() - .any(|(filter, _)| Rc::ptr_eq(filter, parent)) - { - filters_above.push((parent, pred)); - } - } - } - for (filter, predicate) in &filters_above { - // Every consumer of the group, across queries, applies it. - let shared_by_all = filters_above.len() == parents.len() - && filters_above.iter().all(|(_, other)| other == predicate); - entries.push(HavingEntry { - filter: Some(Rc::clone(filter)), - predicate: (*predicate).clone(), - placement: placement(shared_by_all && key_only_having(&node, predicate)), - }); - } + let aggregate_group = MemoGroupId(index as u32); for (measure, intent) in measures.iter().enumerate() { let mut choices = computation_candidates_for_intent(intent); if choices.candidates == [HowToComputeAggregateMeasure::Exact] @@ -440,13 +305,16 @@ pub fn generate_per_computation_candidates( { continue; } - if matches!(intent, AggIntent::Avg { .. }) && nullable_avg_input(&node, intent) { + if matches!(intent, AggIntent::Avg { .. }) + && nullable_avg_input(schema(memo, *child), intent) + { choices.prune(|choice| { matches!(choice, HowToComputeAggregateMeasure::ExactAccumulator(_)) }); } if matches!(intent, AggIntent::Count) - && node.children()[0].result_kind == OperatorResultKind::Relation + && memo.groups()[child.0 as usize].group_properties.result_kind + == OperatorResultKind::Relation { choices.prune(|choice| matches!(choice, HowToComputeAggregateMeasure::Summary(_))); } @@ -454,29 +322,52 @@ pub fn generate_per_computation_candidates( aggregate_group, measure_index: measure, ways_to_compute: choices.candidates, - measure_filter: filters.get(measure).cloned().flatten(), - having_filters: entries.clone(), fused_computation: None, }); } } - add_whole_expression_targets(&mut targets, &logical_nodes); - prune_topk_heaps_over_one_row_per_item(&mut targets, &logical_nodes); - add_nested_sum_targets(&mut targets, &logical_nodes); + add_whole_expression_targets(memo, &mut targets); + prune_topk_heaps_over_one_row_per_item(memo, &mut targets); + add_nested_distributive_targets(memo, &mut targets); close_absorptions(&mut targets); - Ok(PerComputationCandidates { - #[cfg(any(test, feature = "test-support"))] - roots, - logical_nodes, - targets, - }) + without_unsupported_summaries(memo, &mut targets, supported_summaries); + Ok(targets) } -/// Adds to each fused computation's `absorbed_targets` the targets its absorbed -/// targets' own fused computations absorb, transitively: none of them is -/// computed when the fused computation is picked, so enumeration keeps only -/// their first choice ([`PerComputationCandidates::absorbed_choice_varies`]) -/// and [`PerComputationCandidates::combinations`] counts exactly those. +/// Drops from `targets` the summary candidates the executor does not +/// support (`supported_summaries`; `None` supports every one): a target's +/// own, and its fused computation's. The readout is the statistic the +/// measure's estimate reads (the outer measure's for a fused computation). +fn without_unsupported_summaries( + memo: &Memo, + targets: &mut [ComputationTarget], + supported_summaries: Option<&[SupportedSummary]>, +) { + for target in targets.iter_mut() { + let Operator::NonASAP(NonASAPOp::Aggregate { measures, .. }) = + expression(memo, target.aggregate_group) + else { + continue; + }; + let Some(readout) = sketch_statistic(&measures[target.measure_index]).map(|s| s.kind()) + else { + continue; + }; + let unsupported = |candidate: &HowToComputeAggregateMeasure| { + matches!(candidate, HowToComputeAggregateMeasure::Summary(algorithm) + if !summary_supported(supported_summaries, algorithm, readout)) + }; + prune_own(&mut target.ways_to_compute, unsupported); + if let Some(fused) = &mut target.fused_computation { + fused.ways_to_compute.retain(|way| !unsupported(way)); + } + } +} + +/// Adds to each fused computation's `absorbed_groups` the groups its +/// absorbed targets' own fused computations replace, transitively: the +/// fused computation replaces all their operators, so its builds list all +/// their groups in `fused_groups`. fn close_absorptions(targets: &mut [ComputationTarget]) { loop { let mut changed = false; @@ -484,12 +375,18 @@ fn close_absorptions(targets: &mut [ComputationTarget]) { let Some(fused) = &targets[t].fused_computation else { continue; }; - let beneath: Vec = fused - .absorbed_targets + let beneath: Vec = fused + .absorbed_groups .iter() - .filter_map(|&a| targets[a].fused_computation.as_ref()) - .flat_map(|inner| inner.absorbed_targets.iter().copied()) - .filter(|b| !fused.absorbed_targets.contains(b)) + .filter_map(|&group| { + targets + .iter() + .find(|target| target.aggregate_group == group)? + .fused_computation + .as_ref() + }) + .flat_map(|inner| inner.absorbed_groups.iter().copied()) + .filter(|b| !fused.absorbed_groups.contains(b)) .collect(); if let Some(fused) = targets[t] .fused_computation @@ -497,8 +394,8 @@ fn close_absorptions(targets: &mut [ComputationTarget]) { .filter(|_| !beneath.is_empty()) { for b in beneath { - if !fused.absorbed_targets.contains(&b) { - fused.absorbed_targets.push(b); + if !fused.absorbed_groups.contains(&b) { + fused.absorbed_groups.push(b); } } changed = true; @@ -510,119 +407,140 @@ fn close_absorptions(targets: &mut [ComputationTarget]) { } } -/// Nested-sum merge: a grouped `sum` over the value of a `sum` target -/// beneath it gets a fused computation ([`ComputationTarget::fused_computation`]) -/// with the one candidate `ExactAccumulator([Sum])`: one exact `Sum` -/// accumulator over the inner sum's input, grouped by the outer keys (the -/// sum of the inner groups' sums is the sum of their rows), absorbing the -/// inner target. Offered whatever else reads the inner sum: it is one more -/// candidate of the outer sum's group, and the inner sum's group keeps its -/// own candidates for its other consumers. Applied when neither has a filter -/// or `HAVING` -/// ([`nested_sum_input`]), and when the outer sum is not itself absorbed by -/// a merge and has no fused computation yet, and the inner sum has none, so -/// merges do not stack. An outer sum a whole-expression top-k absorbs -/// (`topk(k, sum by (G) (sum_over_time(m[W])))`) keeps its merge for the -/// top-k's own candidates. -fn add_nested_sum_targets(targets: &mut [ComputationTarget], logical_nodes: &[Rc]) { - let aggregate = |target: &ComputationTarget| &logical_nodes[target.aggregate_group.0 as usize]; +/// Nested distributive merge: a grouped `sum`, `min` or `max` over the +/// value of a target of the same reduction beneath it (`sum` of `sum`, +/// `min` of `min`, `max` of `max`) gets a fused computation +/// ([`ComputationTarget::fused_computation`]) with the one candidate +/// `ExactAccumulator([kind])`: one exact accumulator of that kind over the +/// inner aggregate's input, grouped by the outer keys (the sum, min or max +/// of the inner groups' values is that of their rows), absorbing the inner +/// target. Offered whatever else reads the inner aggregate: its group keeps +/// its own candidates for its other consumers. Applied when neither has a +/// filter or `HAVING` ([`nested_distributive_input`]), and when the outer +/// aggregate is not itself absorbed by a merge and has no fused computation +/// yet, and the inner one has none, so merges do not stack. An outer sum a +/// whole-expression top-k absorbs (`topk(k, sum by (G) +/// (sum_over_time(m[W])))`) keeps its merge for the top-k's own candidates. +fn add_nested_distributive_targets(memo: &Memo, targets: &mut [ComputationTarget]) { // The targets a merge absorbs. - let mut absorbed: HashSet = HashSet::new(); + let mut absorbed: Vec = Vec::new(); for t in 0..targets.len() { let target = &targets[t]; if absorbed.contains(&t) || target.fused_computation.is_some() - || !target.having_filters.is_empty() + || has_having(memo, target.aggregate_group) { continue; } - let Some(NonASAPOp::Aggregate { child, .. }) = aggregate(target).non_asap() else { + let Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) = + expression(memo, target.aggregate_group) + else { continue; }; let Some(inner) = targets .iter() - .position(|other| Rc::ptr_eq(aggregate(other), child)) + .position(|other| other.aggregate_group == *child) else { continue; }; - // Merges do not stack: the inner sum must not have a fused + // Merges do not stack: the inner aggregate must not have a fused // computation of its own. - if !targets[inner].having_filters.is_empty() - || targets[inner].fused_computation.is_some() - || nested_sum_input(aggregate(target)).is_none() - { + if has_having(memo, *child) || targets[inner].fused_computation.is_some() { continue; } - absorbed.insert(inner); - let merge = HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Sum]); + let Some((_, _, _, kind)) = nested_distributive_input(memo, target.aggregate_group) else { + continue; + }; + absorbed.push(inner); targets[t].fused_computation = Some(FusedComputation { - absorbed_targets: vec![inner], - ways_to_compute: vec![merge], + absorbed_groups: vec![*child], + ways_to_compute: vec![HowToComputeAggregateMeasure::ExactAccumulator(vec![kind])], }); } } -/// The input rows, the inner sum and the outer grouping (moved onto the -/// input rows by column, [`Reduction::partitions_over`]) of a nested-sum merge of -/// `target`, or `None` when it does not apply: `target` is a grouped `sum` -/// (not by `without`) of the value column of a per-series or grouped `sum` -/// directly beneath it, neither filtered nor with `HAVING`, and not grouped -/// by the metric name a per-series inner sum drops. -pub(super) fn nested_sum_input( - target: &OperatorNode, -) -> Option<(Rc, AggIntent, Reduction)> { - use asap_types::ir::operator::non_asap::any_measure_filtered; - let Some(NonASAPOp::Aggregate { +/// The accumulator kind and input column of a distributive measure, one +/// whose value over a union of groups is that of their values: `sum`, `min` +/// and `max`. +fn distributive(intent: &AggIntent) -> Option<(ExactKind, Option)> { + match intent { + AggIntent::Sum { col } => Some((ExactKind::Sum, *col)), + AggIntent::Min { col } => Some((ExactKind::Min, *col)), + AggIntent::Max { col } => Some((ExactKind::Max, *col)), + _ => None, + } +} + +/// The input rows, the inner measure, the outer grouping (moved onto the +/// input rows by column, [`Reduction::partitions_over`]) and the +/// accumulator kind of a nested distributive merge of `target`, or `None` +/// when it does not apply: `target` is a grouped `sum`, `min` or `max` (not +/// by `without`) of the value column of a per-series or grouped aggregate of +/// the same reduction directly beneath it ([`distributive`]), neither +/// filtered nor with its own `having`, and not grouped by the metric name a +/// per-series inner aggregate drops. +fn nested_distributive_input( + memo: &Memo, + target: MemoGroupId, +) -> Option<(MemoGroupId, AggIntent, Reduction, ExactKind)> { + let Operator::NonASAP(NonASAPOp::Aggregate { child: inner, reduction: reduction @ Reduction::Reduce(by), measures, filters, having: None, .. - }) = target.non_asap() + }) = expression(memo, target) else { return None; }; - let [AggIntent::Sum { col }] = measures.as_slice() else { + let [outer] = measures.as_slice() else { return None; }; - let Some(NonASAPOp::Aggregate { + let (kind, col) = distributive(outer)?; + let Operator::NonASAP(NonASAPOp::Aggregate { child: rows, reduction: inner_reduction, measures: inner_measures, filters: inner_filters, having: None, .. - }) = inner.non_asap() + }) = expression(memo, *inner) else { return None; }; - let [inner_sum @ AggIntent::Sum { .. }] = inner_measures.as_slice() else { + let [inner_measure] = inner_measures.as_slice() else { return None; }; - if by.is_without() || any_measure_filtered(filters) || any_measure_filtered(inner_filters) { + if distributive(inner_measure).map(|(inner_kind, _)| inner_kind) != Some(kind) + || by.is_without() + || any_measure_filtered(filters) + || any_measure_filtered(inner_filters) + { return None; } - // The inner sum's value column: a per-series sum keeps its labels and - // carries the value as the sample value; a grouped one ends with it. + // The inner aggregate's value column: a per-series one keeps its labels + // and carries the value as the sample value; a grouped one ends with it. let value = match inner_reduction { - Reduction::PerEntity => resolve_column_ref(&ColumnRef::SampleValue, &inner.schema).ok()?, + Reduction::PerEntity => { + resolve_column_ref(&ColumnRef::SampleValue, schema(memo, *inner)).ok()? + } Reduction::Reduce(keys) if !keys.is_without() => keys.len(), Reduction::Reduce(_) => return None, }; - let summed = match col { - None => resolve_column_ref(&ColumnRef::SampleValue, &inner.schema).ok()?, - Some(column) => *column, + let reduced = match col { + None => resolve_column_ref(&ColumnRef::SampleValue, schema(memo, *inner)).ok()?, + Some(column) => column, }; - if summed != value || by.keys().contains(&value) { + if reduced != value || by.keys().contains(&value) { return None; } - // A per-series sum drops the metric name, which the raw rows keep: - // grouping them by it would name groups the nested sums leave unnamed. + // A per-series aggregate drops the metric name, which the raw rows + // keep: grouping them by it would name groups the nested aggregates + // leave unnamed. if matches!(inner_reduction, Reduction::PerEntity) && by.keys().iter().any(|&key| { - inner - .schema + schema(memo, *inner) .fields .get(key) .is_some_and(|f| f.name == "__name__") @@ -630,67 +548,55 @@ pub(super) fn nested_sum_input( { return None; } - let partitions = reduction.partitions_over(&inner.schema, &rows.schema)?; - Some((Rc::clone(rows), inner_sum.clone(), partitions)) + let partitions = reduction.partitions_over(schema(memo, *inner), schema(memo, *rows))?; + Some((*rows, inner_measure.clone(), partitions, kind)) } /// Whole-expression top-k (Example 1's Q2): a top-k over a per-item `sum` /// or `count` that is itself a target gets a fused computation -/// ([`ComputationTarget::fused_computation`]) absorbing the inner target, whose -/// candidates are the top-k's heap summaries (Count-Min and CountSketch -/// with a heap), each one heap over the inner aggregate's input, keyed by -/// the ranked item and weighted by the summed value. Those heaps over the -/// exact per-item result leave the top-k's own candidates: the fused -/// computation replaces them. Over a grouped sum of per-series sums or -/// counts of raw data samples, the heap reads those samples and absorbs the -/// per-series target too. Offered whatever else reads the inner aggregate: -/// it is one more candidate of the top-k's group, and the absorbed groups -/// keep their own candidates for their other consumers. Applied when neither -/// has a filter or `HAVING`, and when the keyed-additive rule realizes it -/// ([`whole_expression_input`]). -fn add_whole_expression_targets( - targets: &mut [ComputationTarget], - logical_nodes: &[Rc], -) { - let aggregate = |target: &ComputationTarget| &logical_nodes[target.aggregate_group.0 as usize]; +/// ([`ComputationTarget::fused_computation`]) absorbing the inner target, +/// whose candidates are the top-k's heap summaries (Count-Min and +/// CountSketch with a heap), each one heap over the inner aggregate's +/// input, keyed by the ranked item and weighted by the summed value. Those +/// heaps over the exact per-item result leave the top-k's own candidates: +/// the fused computation replaces them. Over a grouped sum of per-series +/// sums or counts of raw data samples, the heap reads those samples and +/// absorbs the per-series target too. Offered whatever else reads the inner +/// aggregate: the absorbed groups keep their own candidates for their other +/// consumers. Applied when neither has a filter or `HAVING`, and when the +/// keyed-additive rule realizes it ([`whole_expression_input`]). +fn add_whole_expression_targets(memo: &Memo, targets: &mut [ComputationTarget]) { for t in 0..targets.len() { let target = &targets[t]; - let Some(NonASAPOp::Aggregate { child, .. }) = aggregate(target).non_asap() else { - continue; - }; - let Some(inner) = targets - .iter() - .position(|other| Rc::ptr_eq(aggregate(other), child)) + let Operator::NonASAP(NonASAPOp::Aggregate { child, .. }) = + expression(memo, target.aggregate_group) else { continue; }; - if target.measure_filter.is_some() - || !target.having_filters.is_empty() - || targets[inner].measure_filter.is_some() - || !targets[inner].having_filters.is_empty() - { + if !targets.iter().any(|other| other.aggregate_group == *child) { continue; } - let Some((input, ..)) = whole_expression_input(aggregate(target)) else { + // Measure filters: `whole_expression_input` refuses them. + if has_having(memo, target.aggregate_group) || has_having(memo, *child) { + continue; + } + let Some((input, ..)) = whole_expression_input(memo, target.aggregate_group) else { continue; }; // A heap over the raw data samples beneath a grouped sum of per-series // sums or counts also absorbs the per-series target. - let mut absorbs = vec![inner]; - if let Some(NonASAPOp::Aggregate { + let mut absorbs = vec![*child]; + if let Operator::NonASAP(NonASAPOp::Aggregate { child: per_series, .. - }) = aggregate(&targets[inner]).non_asap() + }) = expression(memo, *child) { - let reads_below = per_series - .children() - .iter() - .any(|&rows| Rc::ptr_eq(rows, &input)); - if reads_below { - absorbs.extend( - targets - .iter() - .position(|other| Rc::ptr_eq(aggregate(other), per_series)), - ); + let reads_below = expression(memo, *per_series).children().contains(&&input); + if reads_below + && targets + .iter() + .any(|other| other.aggregate_group == *per_series) + { + absorbs.push(*per_series); } } let (heaps, own): (Vec<_>, Vec<_>) = @@ -699,8 +605,8 @@ fn add_whole_expression_targets( .iter() .cloned() .partition(|candidate| { - matches!(candidate, HowToComputeAggregateMeasure::Summary(family) - if family.capability().answers(StatisticKind::TopK)) + matches!(candidate, HowToComputeAggregateMeasure::Summary(algorithm) + if algorithm.capability().answers(StatisticKind::TopK)) }); if heaps.is_empty() { continue; @@ -708,7 +614,7 @@ fn add_whole_expression_targets( let target = &mut targets[t]; target.ways_to_compute = own; target.fused_computation = Some(FusedComputation { - absorbed_targets: absorbs, + absorbed_groups: absorbs, ways_to_compute: heaps, }); } @@ -717,14 +623,11 @@ fn add_whole_expression_targets( /// Prunes the own heap candidates of each PromQL top-k: the heap reads /// one row per item, so exact is no costlier; a fused whole-expression /// heap ([`add_whole_expression_targets`]) has already moved out of them. -fn prune_topk_heaps_over_one_row_per_item( - targets: &mut [ComputationTarget], - logical_nodes: &[Rc], -) { +fn prune_topk_heaps_over_one_row_per_item(memo: &Memo, targets: &mut [ComputationTarget]) { for target in targets.iter_mut() { - let Some(NonASAPOp::Aggregate { + let Operator::NonASAP(NonASAPOp::Aggregate { measures, child, .. - }) = logical_nodes[target.aggregate_group.0 as usize].non_asap() + }) = expression(memo, target.aggregate_group) else { continue; }; @@ -736,12 +639,15 @@ fn prune_topk_heaps_over_one_row_per_item( } // A PromQL top-k ranks an instant vector: one row per series or // group. (A SQL top-k keeps its heaps.) - if !child.reads_only_time_series() { + if !memo.groups()[child.0 as usize] + .group_properties + .reads_only_time_series + { continue; } prune_own(&mut target.ways_to_compute, |candidate| { - matches!(candidate, HowToComputeAggregateMeasure::Summary(family) - if family.capability().answers(StatisticKind::TopK)) + matches!(candidate, HowToComputeAggregateMeasure::Summary(algorithm) + if algorithm.capability().answers(StatisticKind::TopK)) }); } } @@ -751,48 +657,51 @@ fn prune_topk_heaps_over_one_row_per_item( /// does not apply. Rows that carry the full series identity rank it as a /// column, as [`summary_update`] does. The partitions are `target`'s, moved /// onto the input the heap reads ([`Reduction::partitions_over`]). -pub(super) fn whole_expression_input( - target: &OperatorNode, -) -> Option<(Rc, SummaryUpdate, Reduction)> { - use crate::realization::realize_keyed_additive_summary_input; - let Some(NonASAPOp::Aggregate { +fn whole_expression_input( + memo: &Memo, + target: MemoGroupId, +) -> Option<(MemoGroupId, SummaryUpdate, Reduction)> { + let Operator::NonASAP(NonASAPOp::Aggregate { child, reduction, measures, filters, having: None, .. - }) = target.non_asap() + }) = expression(memo, target) else { return None; }; + let child = *child; let [intent @ AggIntent::TopK { .. }] = measures.as_slice() else { return None; }; - if asap_types::ir::operator::non_asap::any_measure_filtered(filters) { + if any_measure_filtered(filters) { return None; } - // The rule reads only the family's algorithm, and rejects Count-Min over + // The rule reads only the summary type's algorithm, and rejects Count-Min over // weights it cannot prove non-negative. Ask with CountSketch so one - // update serves both heap families; plan selection decides whether + // update serves both heap algorithms; plan selection decides whether // Count-Min is admissible, as for every other Count-Min candidate. - let family = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); - let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; + let summary_type = FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap); + let realized = + realize_keyed_additive_summary_input(memo, intent, &summary_type, reduction, child)?; let mut input = realized.input; let per_series = matches!( - child.non_asap(), - Some(NonASAPOp::Aggregate { + expression(memo, child), + Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::PerEntity, .. }) ); - let partitions = reduction.partitions_over(&child.schema, &realized.child.schema)?; - if per_series && realized.child.schema.has_promql_series_identity() { + let partitions = + reduction.partitions_over(schema(memo, child), schema(memo, realized.child))?; + if per_series && schema(memo, realized.child).has_promql_series_identity() { input.item = Some(asap_types::ir::schema::topk::series_identity_item( - &realized.child.schema, + schema(memo, realized.child), partitions.group_keys().map_or(&[][..], |keys| keys.keys()), )); - } else if realized.child.schema.closed + } else if schema(memo, realized.child).closed && matches!(input.item, Some(SummaryInputExpr::EntityIdentity(_))) { // An encoded label set needs an open PromQL schema. PromQL rows that @@ -804,84 +713,1067 @@ pub(super) fn whole_expression_input( Some((realized.child, input, partitions)) } -#[cfg(test)] -mod tests { - - use super::*; - - use asap_types::ir::schema::{NonNegativeWeightProof, WeightDomain}; - - use asap_accuracy::test_support::estimate_has_bound; - - use asap_types::ir::schema::SketchStatistic; - /// Approximate count keeps one exact option, its accumulator, beside the - /// universal summary; plain exact execution is not offered. - #[test] - fn approximate_count_keeps_one_exact_accumulator_and_universal_choices() { - let choices = computation_candidates_for_intent(&AggIntent::Count).candidates; - assert!(!choices.contains(&HowToComputeAggregateMeasure::Exact)); - assert_eq!( - choices[0], - HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]) - ); - assert!(choices.contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::UnivMon - ))); - } - - /// Count Sketch's signed counters give no total, so Pass 1 offers no - /// Count Sketch for a count, and every count candidate it offers - /// answers the total. - #[test] - fn a_count_offers_no_count_sketch() { - let choices = computation_candidates_for_intent(&AggIntent::Count).candidates; - assert!(!choices.contains(&HowToComputeAggregateMeasure::Summary( - SketchAlgorithm::CountSketch - ))); - let total = sketch_statistic(&AggIntent::Count).unwrap(); - for choice in &choices { - if let HowToComputeAggregateMeasure::Summary(algorithm) = choice { - assert!( - asap_types::ir::schema::SummaryTypeCapability::of(&FieldDataType::Sketch( - *algorithm - )) - .answers(&total), - "{algorithm:?}" - ); - } +/// `update` with the open requirement that its weights are samples of +/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap +/// update whose weights are not otherwise proven. +fn with_raw_data_samples_requirement( + mut update: SummaryUpdate, + metric: Option<&str>, +) -> SummaryUpdate { + if let Some(metric) = metric { + if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { + update.weight_domain = WeightDomain::RawDataSamplesOfMetric { + metric: metric.to_string(), + }; } } + update +} - /// A Count-Min estimate has a bound only over weights proven - /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. - #[test] - fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let family = FieldDataType::Sketch(SketchAlgorithm::Cms); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, +/// Adds Pass 1's candidates to `memo` (Stage 0's, without candidates): +/// every group's own expression, then, for each aggregate group with +/// `targets`, each way to compute it. A consumer `Filter` group whose +/// key-only `HAVING` is pushed below the aggregate gets the candidates that +/// compute its filtered rows. Every group and candidate is added through +/// `Memo::find_or_add_group` and `Memo::add_candidate`. +pub(crate) fn add_pass1_candidates( + memo: &mut Memo, + targets: &[ComputationTarget], +) -> Result<(), PerComputationCandidateError> { + // Stage 0's groups: a consumer `Filter` among them is a query's + // `HAVING`; the `Filter` groups Pass 1 adds are parts of its candidates. + let stage0_groups = memo.groups().len(); + for index in 0..stage0_groups { + let group = MemoGroupId(index as u32); + let own = expression(memo, group).clone(); + add_candidate(memo, group, own, None, Vec::new())?; + } + let mut targets_of: BTreeMap> = BTreeMap::new(); + for target in targets { + targets_of + .entry(target.aggregate_group) + .or_default() + .push(target); + } + for (&aggregate_group, group_targets) in &targets_of { + let Operator::NonASAP(NonASAPOp::Aggregate { + reduction, having, .. + }) = expression(memo, aggregate_group) + else { + continue; }; - let with = |weight_domain| SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain, + let (below, above): (Vec<_>, Vec<_>) = having + .iter() + .cloned() + .partition(|having| key_only_having(reduction, having)); + // Each consumer `Filter` group whose `HAVING` is key-only gets the + // candidates that compute its rows with the predicate pushed below. + let pushed_down: Vec<(MemoGroupId, Predicate)> = + having_filter_groups(memo, aggregate_group) + .filter(|&(filter, pred)| { + (filter.0 as usize) < stage0_groups && key_only_having(reduction, pred) + }) + .map(|(filter, pred)| (filter, pred.clone())) + .collect(); + let ways = AggregateWays { + aggregate_group, + result_group: aggregate_group, + below: &below, + above: &above, + targets: group_targets, }; - assert!(!estimate_has_bound( - &family, - &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), - &query - )); - assert!(!estimate_has_bound( - &family, - &with(WeightDomain::UnknownOrSigned), - &query + add_aggregate_candidates(memo, &ways)?; + for (filter, pred) in pushed_down { + let mut with_consumer = below.clone(); + with_consumer.push(pred); + let filtered = AggregateWays { + result_group: filter, + below: &with_consumer, + ..ways + }; + add_aggregate_candidates(memo, &filtered)?; + } + } + Ok(()) +} + +/// The ways to compute one result from an aggregate group's targets. +struct AggregateWays<'a> { + /// The aggregate group whose measures are computed. + aggregate_group: MemoGroupId, + /// The group the candidates compute: the aggregate group, or a consumer + /// `Filter` group above it whose key-only `HAVING` goes below. + result_group: MemoGroupId, + /// Key-only predicates over the aggregate's output, applied to its + /// input rows (their key columns renumbered to the input's). + below: &'a [Predicate], + /// Predicates applied as `Filter`s above the output. + above: &'a [Predicate], + /// The aggregate's targets, one per measure that has one. + targets: &'a [&'a ComputationTarget], +} + +/// One measure of an aggregate, computed on its own over `input`. +struct SingleMeasure<'a> { + input: MemoGroupId, + reduction: &'a Reduction, + intent: &'a AggIntent, + filter: Option<&'a Predicate>, + /// The metric whose raw data samples, or plain sums of them, are every + /// value the aggregate reads: a heap-sketch weight read from them + /// carries the open requirement `WeightDomain::RawDataSamplesOfMetric`. + raw_data_samples_of: Option<&'a str>, +} + +/// Adds `ways.result_group`'s candidates that compute it from +/// `ways.aggregate_group`'s measures: for a single measure, each of its +/// target's ways and fused ways; for several, one candidate joining the +/// measures' own groups (a single-measure aggregate each, with its own +/// candidates) on the group keys. +fn add_aggregate_candidates( + memo: &mut Memo, + ways: &AggregateWays<'_>, +) -> Result<(), PerComputationCandidateError> { + let Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction, + measures, + output_names, + filters, + .. + }) = expression(memo, ways.aggregate_group).clone() + else { + return Err(PerComputationCandidateError::Unsupported( + "target is not an aggregate", )); - assert!(estimate_has_bound( - &family, - &with(WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples + }; + let keys: Vec = reduction + .group_keys() + .map_or_else(Vec::new, |by| by.keys().to_vec()); + let mut input = child; + for predicate in ways.below { + let pred = Predicate(predicate.0.map_columns(&mut |column| keys[column])); + input = add_group( + memo, + Operator::NonASAP(NonASAPOp::Filter { pred, child: input }), + )?; + } + let raw_data_samples = raw_data_samples_of_metric(memo, child); + let measure = |index: usize| SingleMeasure { + input, + reduction: &reduction, + intent: &measures[index], + filter: filters.get(index).and_then(Option::as_ref), + raw_data_samples_of: raw_data_samples.as_deref(), + }; + let target_of = |index: usize| { + ways.targets + .iter() + .find(|target| target.measure_index == index) + }; + if measures.len() == 1 { + let Some(target) = target_of(0) else { + return Ok(()); + }; + for way in &target.ways_to_compute { + if *way == HowToComputeAggregateMeasure::Exact { + continue; + } + unless_unsupported( + measure_output(memo, &measure(0), way, ways.result_group).and_then(|output| { + add_output_candidate(memo, ways.result_group, output, ways.above) + }), + )?; + } + if let Some(fused) = &target.fused_computation { + for way in &fused.ways_to_compute { + unless_unsupported(add_fused_candidate( + memo, + ways, + &measure(0), + way, + &fused.absorbed_groups, + ))?; + } + } + return Ok(()); + } + // The measures are joined back on their group keys: SQL rows grouped + // `by` keys (PromQL aggregates have one measure each). + if !matches!(&reduction, Reduction::Reduce(by) if !by.is_without()) + || memo.groups()[input.0 as usize].group_properties.result_kind + != OperatorResultKind::Relation + { + return Ok(()); + } + let mut parts = Vec::with_capacity(measures.len()); + for (index, intent) in measures.iter().enumerate() { + let part = add_group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: input, + reduction: reduction.clone(), + measures: vec![intent.clone()], + output_names: vec![output_names.get(index).cloned().unwrap_or_default()], + filters: vec![filters.get(index).cloned().flatten()], + having: None, + }), + )?; + if let Some(target) = target_of(index) { + for way in &target.ways_to_compute { + if *way == HowToComputeAggregateMeasure::Exact { + continue; + } + unless_unsupported( + measure_output(memo, &measure(index), way, part) + .and_then(|output| add_output_candidate(memo, part, output, &[])), + )?; + } + } + parts.push(part); + } + let output = join_measures(memo, &reduction, ways.result_group, parts)?; + add_output_candidate(memo, ways.result_group, output, ways.above) +} + +/// `added`, with a way Pass 1 cannot build (`Unsupported`) not offered: +/// its checks run before the groups it reads are added, so the memo is +/// unchanged, except for the rare checks that need a group's schema (a +/// top-k estimate without its item columns, a per-series `avg` that is not +/// Float64), which leave groups no query reaches. +fn unless_unsupported( + added: Result<(), PerComputationCandidateError>, +) -> Result<(), PerComputationCandidateError> { + match added { + Err(PerComputationCandidateError::Unsupported(_)) => Ok(()), + added => added, + } +} + +/// Adds the fused computation `way` of `ways`' single measure: a whole- +/// expression heap (a summary way) or a nested distributive merge (an +/// exact accumulator way). Its builds list `absorbed_groups`, the groups of the +/// operators it replaces, in `fused_groups`. +fn add_fused_candidate( + memo: &mut Memo, + ways: &AggregateWays<'_>, + measure: &SingleMeasure<'_>, + way: &HowToComputeAggregateMeasure, + absorbed_groups: &[MemoGroupId], +) -> Result<(), PerComputationCandidateError> { + match way { + HowToComputeAggregateMeasure::Summary(algorithm) => { + let (rows, update, partitions) = whole_expression_input(memo, ways.aggregate_group) + .ok_or(PerComputationCandidateError::Unsupported( + "whole-expression top-k input", + ))?; + let query = sketch_statistic(measure.intent).ok_or( + PerComputationCandidateError::Unsupported("sketch evaluation"), + )?; + let state = add_state_group( + memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: rows, + input: with_raw_data_samples_requirement(update, measure.raw_data_samples_of), + summary_type: FieldDataType::Sketch(*algorithm), + reduction: partitions, + filter: None, + }), + absorbed_groups, + )?; + let estimate = add_group( + memo, + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query, + }), + )?; + let output = ranked_as_topk(memo, measure.reduction, ways.result_group, estimate)?; + add_candidate(memo, ways.result_group, output, None, Vec::new())?; + } + HowToComputeAggregateMeasure::ExactAccumulator(_) => { + // The merged build computes the outer aggregate's own + // accumulator state, from the rows below the inner aggregate: a + // candidate of that state group (#1203, fused builds). + let (rows, inner, partitions, kind) = + nested_distributive_input(memo, ways.aggregate_group).ok_or( + PerComputationCandidateError::Unsupported("nested distributive merge input"), + )?; + let read = measure_input(measure.intent, memo, measure.input, measure.filter)?; + let outer_state = accumulator_state(memo, kind, measure.intent, measure, read)?; + let below_inner = SingleMeasure { + input: rows, + reduction: &partitions, + intent: &inner, + filter: None, + raw_data_samples_of: None, + }; + let read = measure_input(&inner, memo, rows, None)?; + let merged = accumulator_build(memo, kind, &inner, &below_inner, read)?; + add_candidate(memo, outer_state, merged, None, absorbed_groups.to_vec())?; + } + HowToComputeAggregateMeasure::Exact => { + return Err(PerComputationCandidateError::Unsupported( + "plain exact execution of a fused computation", + )) + } + } + Ok(()) +} + +/// The operator that computes `measure` by `way` (not `Exact`) for +/// `result_group`: its value after its group keys, its column names not +/// yet the group's ([`add_output_candidate`] renames them). Every group it +/// reads is added with its candidates. +fn measure_output( + memo: &mut Memo, + measure: &SingleMeasure<'_>, + way: &HowToComputeAggregateMeasure, + result_group: MemoGroupId, +) -> Result, PerComputationCandidateError> { + Ok(match way { + HowToComputeAggregateMeasure::Exact => { + return Err(PerComputationCandidateError::Unsupported( + "plain exact execution is the group's own expression", + )) + } + HowToComputeAggregateMeasure::ExactAccumulator(kinds) if kinds.len() == 1 => { + let read = measure_input(measure.intent, memo, measure.input, measure.filter)?; + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { + child: accumulator_state(memo, kinds[0], measure.intent, measure, read)?, + }) + } + HowToComputeAggregateMeasure::Summary(algorithm) => { + let query = sketch_statistic(measure.intent).ok_or( + PerComputationCandidateError::Unsupported("sketch evaluation"), + )?; + let summary_type = FieldDataType::Sketch(*algorithm); + let read = measure_input(measure.intent, memo, measure.input, measure.filter)?; + let update = with_raw_data_samples_requirement( + summary_update( + measure.intent, + &summary_type, + measure.reduction, + schema(memo, measure.input), + read, + )?, + measure.raw_data_samples_of, + ); + let state = add_state_group( + memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: measure.input, + input: update, + summary_type, + reduction: measure.reduction.clone(), + filter: measure.filter.cloned(), + }), + &[], + )?; + let estimate = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query, + }); + match measure.intent { + AggIntent::TopK { .. } => { + let estimate = add_group(memo, estimate)?; + ranked_as_topk(memo, measure.reduction, result_group, estimate)? + } + _ => estimate, + } + } + HowToComputeAggregateMeasure::ExactAccumulator(kinds) => { + let (AggIntent::Avg { col }, [ExactKind::Sum, ExactKind::Count]) = + (measure.intent, kinds.as_slice()) + else { + return Err(PerComputationCandidateError::Unsupported( + "several exact states other than avg's sum then count", + )); + }; + // The count counts the averaged column, as `avg` does: it is + // non-null (a nullable input prunes this candidate), so it counts + // the rows the sum reads. + let sum_intent = AggIntent::Sum { col: *col }; + let count_intent = AggIntent::Count; + let read = measure_input(&sum_intent, memo, measure.input, measure.filter)?; + let sum = accumulator_state(memo, ExactKind::Sum, &sum_intent, measure, read.clone())?; + let count = accumulator_state(memo, ExactKind::Count, &count_intent, measure, read)?; + let sum = add_group( + memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: sum }), + )?; + let count = add_group( + memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: count }), + )?; + if memo.groups()[measure.input.0 as usize] + .group_properties + .result_kind + == OperatorResultKind::Relation + { + return sql_avg( + memo, + measure.reduction, + measure.filter.is_some(), + sum, + count, + ); + } + // `avg` is Float64 whatever the summed type, as `Avg`'s output is, + // and a PromQL binary operator reads Float64 sample values. A + // grouped sum or count ends with its value, cast when not Float64 + // (a count is Int64). A per-series sum keeps the input's labels, + // so its value is the sample-value column; a per-series count is + // Float64. + let (sum, count) = match measure.reduction { + Reduction::Reduce(by) => ( + last_as_float64(memo, by.len(), sum)?, + last_as_float64(memo, by.len(), count)?, + ), + Reduction::PerEntity => { + let sum_schema = schema(memo, sum); + let value = resolve_column_ref(&ColumnRef::SampleValue, sum_schema) + .ok() + .and_then(|column| sum_schema.fields.get(column)); + if value.map(|field| &field.dtype) + != Some(&FieldDataType::Plain(DataType::Float64)) + { + return Err(PerComputationCandidateError::Unsupported( + "non-Float64 per-series avg", + )); + } + (sum, count) + } + }; + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + // A per-series window may hold no finite sum to divide. + checked_finite_division: matches!(measure.reduction, Reduction::PerEntity), + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: sum, + rhs: count, + }) + } + }) +} + +/// The accumulator-state group of `kind` for `intent` (`measure`'s own, or +/// one of its states, such as `avg`'s `sum`), reading `read` from +/// `measure`'s input, with its build candidate. +fn accumulator_state( + memo: &mut Memo, + kind: ExactKind, + intent: &AggIntent, + measure: &SingleMeasure<'_>, + read: SummaryInputExpr, +) -> Result { + let build = accumulator_build(memo, kind, intent, measure, read)?; + add_state_group(memo, build, &[]) +} + +/// The accumulator build of `kind` for `intent` over `measure`'s input, +/// filtered and grouped as `measure`. +fn accumulator_build( + memo: &Memo, + kind: ExactKind, + intent: &AggIntent, + measure: &SingleMeasure<'_>, + read: SummaryInputExpr, +) -> Result, PerComputationCandidateError> { + let summary_type = FieldDataType::exact(kind); + Ok(Operator::ASAP(ASAPOp::SummaryAgg { + input: summary_update( + intent, + &summary_type, + measure.reduction, + schema(memo, measure.input), + read, + )?, + child: measure.input, + summary_type, + reduction: measure.reduction.clone(), + filter: measure.filter.cloned(), + })) +} + +/// Adds `output` as a candidate of `result_group`, then the `above` filters +/// over it. When its column names differ from the group's, +/// `output` becomes its own group, read through a `Project` that names its +/// columns as the group does ([`renaming_project`]). With `above` +/// predicates the output (named as the group) is a group, read by one +/// `Filter` group per predicate but the last, whose `Filter` is the +/// candidate. +fn add_output_candidate( + memo: &mut Memo, + result_group: MemoGroupId, + output: Operator, + above: &[Predicate], +) -> Result<(), PerComputationCandidateError> { + let Some((last, filters)) = above.split_last() else { + match add_candidate(memo, result_group, output.clone(), None, Vec::new()) { + Err(PerComputationCandidateError::Candidate( + CandidateError::OutputDiffersFromGroup { .. }, + )) => {} + added => return added.map(|_| ()), + } + let output = add_group(memo, output)?; + let renamed = renaming_project(memo, output, result_group)?; + add_candidate(memo, result_group, renamed, None, Vec::new())?; + return Ok(()); + }; + let mut named = add_group(memo, output)?; + if schema(memo, named) != schema(memo, result_group) { + let renamed = renaming_project(memo, named, result_group)?; + named = add_group(memo, renamed)?; + } + for pred in filters { + named = add_group( + memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: pred.clone(), + child: named, }), - &query + )?; + } + add_candidate( + memo, + result_group, + Operator::NonASAP(NonASAPOp::Filter { + pred: last.clone(), + child: named, + }), + None, + Vec::new(), + )?; + Ok(()) +} + +/// A `Project` over `output` that names its columns as `result_group`'s. `output` must have the group's number of columns, +/// and the group's schema must be closed (list every column its rows carry). +fn renaming_project( + memo: &Memo, + output: MemoGroupId, + result_group: MemoGroupId, +) -> Result, PerComputationCandidateError> { + let (from, to) = (schema(memo, output), schema(memo, result_group)); + // An open schema's rows carry columns it does not list (a PromQL + // series whose identity is not realized as a column): a `Project` lists + // its columns, so it would drop them, and its rows are not the group's. + if !to.closed { + return Err(PerComputationCandidateError::Unsupported( + "a rename drops the columns an open schema does not list", + )); + } + if from.fields.len() != to.fields.len() { + return Err(PerComputationCandidateError::Unsupported( + "substituted output with other columns than the aggregate", )); } + let cols = to + .fields + .iter() + .enumerate() + .map(|(column, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(column), + }) + .collect(); + Ok(Operator::NonASAP(NonASAPOp::Project { + cols, + child: output, + })) +} + +/// A heap's top-k `estimate` group (`[partitions.., items.., topk]`, one +/// row per ranked item) in `result_group`'s columns, as a `Project`: its +/// `[keys.., topk_k]`, with `topk_k` the estimated Float64 weight, then the +/// item columns. This is the exact top-k's physical shape (its ranked rows' +/// value, then the columns naming each ranked item: PromQL labels, SQL +/// group keys), so consumers of `[keys.., topk_k]` find those columns where +/// they were. +fn ranked_as_topk( + memo: &Memo, + reduction: &Reduction, + result_group: MemoGroupId, + estimate: MemoGroupId, +) -> Result, PerComputationCandidateError> { + let ranked_index = reduction.group_keys().map_or(0, |keys| keys.len()); + let fields = &schema(memo, estimate).fields; + let estimated = fields.len() - 1; + let to = schema(memo, result_group); + let cols = to + .fields + .iter() + .enumerate() + .map(|(index, field)| { + let source = if index == ranked_index { + estimated + } else { + fields[..estimated] + .iter() + .position(|input| input.name == field.name) + .ok_or(PerComputationCandidateError::Unsupported( + "top-k estimate without a ranked-row identity column", + ))? + }; + // The estimated weight is Float64; an exact top-k of a SQL + // count ranks Int64 values, so its column is cast to them. + let expr = match (&fields[source].dtype, &field.dtype) { + (FieldDataType::Plain(from), FieldDataType::Plain(to)) if from != to => { + ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Column(source)), + to: to.clone(), + try_cast: false, + } + } + _ => ScalarExpr::Column(source), + }; + Ok(ProjectItem { + alias: Some(field.name.clone()), + expr, + }) + }) + .collect::, PerComputationCandidateError>>()?; + Ok(Operator::NonASAP(NonASAPOp::Project { + cols, + child: estimate, + })) +} + +/// The group of `operator`, whatever adds it, with `operator` as its +/// candidate: an intermediate result is its own group. +fn add_group( + memo: &mut Memo, + operator: Operator, +) -> Result { + let group = memo.find_or_add_group(operator.clone())?; + add_candidate(memo, group, operator, None, Vec::new())?; + Ok(group) +} + +/// The state group of `build` (a `SummaryAgg`) with its build candidates: +/// one per configuration of a sketch algorithm's predefined list +/// (`parameter_list`), or the one exact accumulator build. Each lists +/// `fused_groups`. +fn add_state_group( + memo: &mut Memo, + build: Operator, + fused_groups: &[MemoGroupId], +) -> Result { + let group = memo.find_or_add_group(build.clone())?; + let configurations: Vec> = match &build { + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::Sketch(algorithm), + .. + }) => parameter_list(*algorithm) + .iter() + .cloned() + .map(Some) + .collect(), + _ => vec![None], + }; + for parameters in configurations { + add_candidate( + memo, + group, + build.clone(), + parameters, + fused_groups.to_vec(), + )?; + } + Ok(group) +} + +/// Adds the candidate `operator` with `parameters` to `group`. Its input +/// requirements are empty: Pass 2 sets each readout's required accuracy +/// (`Memo::set_input_requirements`). +fn add_candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, + fused_groups: Vec, +) -> Result<(), PerComputationCandidateError> { + let expression = CandidateExpression { operator }; + let logical_properties = LogicalProperties::new(&expression, parameters)?; + let input_requirements = + vec![LogicalRequirements::default(); expression.operator.children().len()]; + memo.add_candidate( + group, + CandidateForMemoGroup { + expression, + logical_properties, + input_requirements, + fused_groups, + origin: OriginOfCandidateForMemoGroup::Pass1, + }, + )?; + Ok(()) +} + +/// `group`, a grouped aggregate's `keys` group keys and then its value, +/// with the value cast to Float64 unless it already is one (a `Project` +/// group over it). +fn last_as_float64( + memo: &mut Memo, + keys: usize, + group: MemoGroupId, +) -> Result { + if schema(memo, group).fields.last().map(|field| &field.dtype) + == Some(&FieldDataType::Plain(DataType::Float64)) + { + return Ok(group); + } + let mut cols = key_items(keys); + cols.push(ProjectItem { + alias: None, + expr: ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Column(keys)), + to: DataType::Float64, + try_cast: false, + }, + }); + add_group( + memo, + Operator::NonASAP(NonASAPOp::Project { cols, child: group }), + ) +} + +/// SQL `avg` from its finalized `sum` and `count` groups: joined on the +/// group keys (NULL keys match), then `sum / count` as Float64. A group +/// with no row passing the measure filter, or an empty global input, counts +/// 0 and its `avg` is NULL. +fn sql_avg( + memo: &mut Memo, + reduction: &Reduction, + filtered: bool, + sum: MemoGroupId, + count: MemoGroupId, +) -> Result, PerComputationCandidateError> { + let Reduction::Reduce(by) = reduction else { + return Err(PerComputationCandidateError::Unsupported( + "SQL avg without `by` keys", + )); + }; + if by.is_without() { + return Err(PerComputationCandidateError::Unsupported( + "SQL avg without `by` keys", + )); + } + let keys = by.len(); + let float = |column| { + Box::new(ScalarExpr::Cast { + expr: Box::new(ScalarExpr::Column(column)), + to: DataType::Float64, + try_cast: false, + }) + }; + let count_column = keys + 1 + keys; + let mut avg = ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Div, + left: float(keys), + right: float(count_column), + semantics: ExprSemantics::Sql, + }; + if filtered || keys == 0 { + avg = ScalarExpr::Case { + operand: None, + branches: vec![( + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(count_column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(0))), + semantics: ExprSemantics::Sql, + }, + ScalarExpr::Literal(ScalarValue::Null), + )], + else_expr: Some(Box::new(avg)), + }; + } + let mut cols = key_items(keys); + cols.push(ProjectItem { + alias: None, + expr: avg, + }); + let joined = add_group(memo, join_on_keys(memo, keys, sum, count))?; + Ok(Operator::NonASAP(NonASAPOp::Project { + cols, + child: joined, + })) +} + +/// `left` joined with `right` on their leading `keys` columns; NULL keys +/// match each other, as grouping treats them. +fn join_on_keys( + memo: &Memo, + keys: usize, + left: MemoGroupId, + right: MemoGroupId, +) -> Operator { + let width = schema(memo, left).fields.len(); + let right_schema = schema(memo, right); + let column = |id| Box::new(ScalarExpr::Column(id)); + let matches = (0..keys) + .map(|key| { + let equal = ScalarExpr::Compare { + left: column(key), + op: CompareOpKind::Eq, + right: column(width + key), + semantics: ExprSemantics::Sql, + }; + if right_schema.fields[key].nullable { + ScalarExpr::BoolOr(vec![ + equal, + ScalarExpr::BoolAnd(vec![ + ScalarExpr::IsNull(column(key)), + ScalarExpr::IsNull(column(width + key)), + ]), + ]) + } else { + equal + } + }) + .collect::>(); + let (kind, pred) = match matches.len() { + 0 => ( + JoinKind::Cross, + ScalarExpr::Literal(ScalarValue::Boolean(true)), + ), + _ => (JoinKind::Inner, ScalarExpr::BoolAnd(matches)), + }; + Operator::NonASAP(NonASAPOp::Join { + kind, + pred: Predicate(pred), + left, + right, + }) +} + +/// Joins single-measure `parts` (each a group: group keys, then one value) +/// on their group keys, and projects `result_group`'s output columns. Every +/// part has the same groups, so an inner join keeps them all; NULL keys +/// match each other, as grouping treats them. +fn join_measures( + memo: &mut Memo, + reduction: &Reduction, + result_group: MemoGroupId, + parts: Vec, +) -> Result, PerComputationCandidateError> { + let Reduction::Reduce(by) = reduction else { + return Err(PerComputationCandidateError::Unsupported( + "multi-measure aggregate without `by` keys", + )); + }; + if by.is_without() { + return Err(PerComputationCandidateError::Unsupported( + "multi-measure aggregate without `by` keys", + )); + } + let keys = by.len(); + let width = keys + 1; + let mut parts = parts.into_iter(); + let mut joined = parts.next().expect("an aggregate has measures"); + for part in parts { + let join = join_on_keys(memo, keys, joined, part); + joined = add_group(memo, join)?; + } + let to = schema(memo, result_group); + let mut cols = key_items(keys); + cols.extend((0..to.fields.len() - keys).map(|measure| ProjectItem { + alias: None, + expr: ScalarExpr::Column(measure * width + keys), + })); + for (item, field) in cols.iter_mut().zip(&to.fields) { + item.alias = Some(field.name.clone()); + } + Ok(Operator::NonASAP(NonASAPOp::Project { + cols, + child: joined, + })) +} + +fn key_items(keys: usize) -> Vec> { + (0..keys) + .map(|key| ProjectItem { + alias: None, + expr: ScalarExpr::Column(key), + }) + .collect() +} + +/// What a measure's state reads from each row of `input`, resolvable there. +/// A SQL `count` reads [`counted_column`]; any other measure, a PromQL count +/// included, reads its input column (the sample value when it names none). +fn measure_input( + intent: &AggIntent, + memo: &Memo, + input: MemoGroupId, + filter: Option<&Predicate>, +) -> Result { + let schema = schema(memo, input); + let read = match intent { + AggIntent::Count + if memo.groups()[input.0 as usize].group_properties.result_kind + == OperatorResultKind::Relation => + { + SummaryInputExpr::Column(counted_column(schema, filter)?) + } + // A top-k ranks its child's aggregate output, not a column of its own. + AggIntent::TopK { .. } => SummaryInputExpr::Column( + crate::realization::ranked_value(memo, input) + .map_err(|_| { + PerComputationCandidateError::Unsupported("Top-K has no ranked column") + })? + .1, + ), + _ => crate::realization::summarised_input(intent, schema).ok_or( + PerComputationCandidateError::Unsupported("input column outside child schema"), + )?, + }; + if let SummaryInputExpr::Column(column) = &read { + resolve_column_ref(column, schema).map_err(|_| { + PerComputationCandidateError::Unsupported("summary input column does not resolve") + })?; + } + Ok(read) +} + +/// The column a SQL `count` state updates on. The runtime skips NULL updates, +/// so it is a column non-null on every counted row: one the measure filter +/// proves non-null (`count(col)` lowers to the filter `col IS NOT NULL`), else +/// a non-nullable one, Float64 first (the runtime's numeric update). +fn counted_column( + schema: &Schema, + filter: Option<&Predicate>, +) -> Result { + fn not_null(expr: &ScalarExpr) -> Vec { + match expr { + ScalarExpr::IsNotNull(inner) => match **inner { + ScalarExpr::Column(column) => vec![column], + _ => vec![], + }, + ScalarExpr::BoolAnd(conjuncts) => conjuncts.iter().flat_map(not_null).collect(), + _ => vec![], + } + } + let non_nullable = |float: bool| { + schema + .fields + .iter() + .enumerate() + .filter_map(move |(id, field)| { + (!field.nullable + && (!float || field.dtype == FieldDataType::Plain(DataType::Float64))) + .then_some(id) + }) + }; + let reference = |id: usize| { + schema + .fields + .get(id) + .map(asap_types::ir::schema::Field::column_ref) + }; + filter + .map_or_else(Vec::new, |filter| not_null(&filter.0)) + .into_iter() + .chain(non_nullable(true)) + .chain(non_nullable(false)) + .find_map(|id| { + reference(id).filter(|column| resolve_column_ref(column, schema).ok() == Some(id)) + }) + .ok_or(PerComputationCandidateError::Unsupported( + "SQL count over rows with no column known to be non-null", + )) +} + +/// What each input row contributes, given what the measure reads +/// ([`measure_input`]), following the legacy realization rules: heap sketches +/// rank series identities, frequency sketches count values, and every other +/// summary type reads the measure's input. +fn summary_update( + intent: &AggIntent, + summary_type: &FieldDataType, + reduction: &Reduction, + child: &Schema, + weight: SummaryInputExpr, +) -> Result { + let algorithm = match summary_type { + FieldDataType::Sketch(kind) => Some(kind), + _ => None, + }; + Ok(match (intent, algorithm) { + (AggIntent::TopK { .. }, Some(_)) => { + if reduction.group_keys().is_some_and(|keys| keys.is_without()) { + return Err(PerComputationCandidateError::Unsupported( + "Top-K partitions given by `without`", + )); + } + let groups: Vec = reduction + .group_keys() + .into_iter() + .flat_map(|keys| keys.iter().copied()) + .collect(); + let item = if child.has_promql_series_identity() { + asap_types::ir::schema::topk::series_identity_item(child, &groups) + } else if child.closed { + // A closed child (an aggregate's output, say) lists every + // column: the item is its columns outside the partition and + // the ranked value, as the legacy current-series and counter + // rules rank them. + let SummaryInputExpr::Column(ranked) = &weight else { + return Err(PerComputationCandidateError::Unsupported( + "Top-K ranked value is not a column", + )); + }; + let value = resolve_column_ref(ranked, child).map_err(|_| { + PerComputationCandidateError::Unsupported("Top-K ranked value does not resolve") + })?; + let items: Vec<_> = child + .fields + .iter() + .enumerate() + .filter(|(index, _)| *index != value && !groups.contains(index)) + .map(|(_, field)| SummaryInputExpr::Column(field.column_ref())) + .collect(); + if items.is_empty() { + return Err(PerComputationCandidateError::Unsupported( + "Top-K item has no label outside its partition", + )); + } + SummaryInputExpr::Tuple(items) + } else { + let excluding = groups + .iter() + .filter_map(|&index| child.fields.get(index)) + .map(asap_types::ir::schema::Field::column_ref) + .collect(); + SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { excluding }) + }; + SummaryUpdate { + item: Some(item), + // The ranked value ([`measure_input`]). + weight, + // Not proven non-negative; selection decides whether CMS is admissible. + weight_domain: WeightDomain::UnknownOrSigned, + } + } + (_, Some(SketchAlgorithm::UnivMon)) | (AggIntent::Count, Some(SketchAlgorithm::Cms)) => { + SummaryUpdate { + item: Some(weight), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::UnitCount, + }, + } + } + _ => SummaryUpdate { + item: None, + weight, + weight_domain: WeightDomain::UnknownOrSigned, + }, + }) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs index 7111f60f0..98bbaef06 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs @@ -1,13 +1,956 @@ -//! Frontend-to-Pass-1 acceptance: candidate discovery precedes empirical selection. -use crate::per_computation_candidates::computation_candidates_for_intent; +//! Pass 1's tests. Each builds its input memo through `Memo::find_or_add_group` +//! and `Memo::add_query`, as Stage 0 does, and reads the candidates +//! [`pass1_memo`](crate::pass1_memo) adds; no other stage crate is called. +use std::fmt::Write as _; +use std::time::Duration; + +use asap_types::ir::operator::operator_properties::{BinaryOpKind, Reduction, Source}; use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::SketchAlgorithm; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::{ + DataType, ExactKind, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, + SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, PROMQL_SERIES_IDENTITY, +}; +use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, + TimeRangeKind, +}; +use asap_types::stages::{ + parameter_list, CandidateForMemoGroup, HowToComputeAggregateMeasure, LogicalRequirements, Memo, + MemoGroupId, +}; + +use crate::pass1_memo; +use crate::per_computation_candidates::{ + computation_candidates_for_intent, computation_targets, PerComputationCandidateError, +}; +use crate::test_fixtures::{ + aggregate_group, group, metric_schema, query, scan_group, scan_operator, +}; + +/// A compact printout of every group and its candidates: the expression's +/// operator, then per candidate its operator, input groups, θ and fused +/// groups. +fn memo_printout(memo: &Memo) -> String { + let mut out = String::new(); + for (index, group) in memo.groups().iter().enumerate() { + let _ = writeln!(out, "G{index}: {}", label(&group.expression)); + for (position, candidate) in group.candidates.iter().enumerate() { + let inputs: Vec = candidate + .expression + .operator + .children() + .iter() + .map(|g| format!("G{}", g.0)) + .collect(); + let mut line = format!( + " [{position}] {} inputs=[{}]", + label(&candidate.expression.operator), + inputs.join(", ") + ); + if let Some(parameters) = candidate.logical_properties.parameters() { + let _ = write!(line, " θ={parameters:?}"); + } + if !candidate.fused_groups.is_empty() { + let _ = write!(line, " fused_groups={:?}", candidate.fused_groups); + } + let _ = writeln!(out, "{line}"); + } + } + out +} + +fn label(operator: &Operator) -> String { + match operator { + Operator::ASAP(ASAPOp::SummaryAgg { summary_type, .. }) => { + format!("SummaryAgg[{summary_type:?}]") + } + Operator::ASAP(ASAPOp::SummaryEstimate { query, .. }) => { + format!("SummaryEstimate[{query:?}]") + } + Operator::NonASAP(NonASAPOp::Aggregate { measures, .. }) => { + let names: Vec = measures + .iter() + .map(|m| { + format!("{m:?}") + .split([' ', '{']) + .next() + .unwrap_or("") + .to_string() + }) + .collect(); + format!("Aggregate[{}]", names.join(", ")) + } + operator => operator.kind_name().to_string(), + } +} + +fn candidates(memo: &Memo, group: MemoGroupId) -> &[CandidateForMemoGroup] { + &memo.groups()[group.0 as usize].candidates +} + +/// The groups of `memo` whose expression matches `pattern`. +fn groups_where(memo: &Memo, pattern: impl Fn(&Operator) -> bool) -> Vec { + memo.groups() + .iter() + .enumerate() + .filter(|(_, group)| pattern(&group.expression)) + .map(|(index, _)| MemoGroupId(index as u32)) + .collect() +} + +/// The state group of `algorithm` in `memo`, and its build's update. +fn state_group(memo: &Memo, algorithm: SketchAlgorithm) -> (MemoGroupId, SummaryUpdate) { + let found = groups_where( + memo, + |expression| matches!(expression, Operator::ASAP(ASAPOp::SummaryAgg { summary_type: FieldDataType::Sketch(a), .. }) if *a == algorithm), + ); + let [state] = found[..] else { + panic!( + "one {algorithm:?} state group: {found:?}\n{}", + memo_printout(memo) + ) + }; + let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = + &memo.groups()[state.0 as usize].expression + else { + unreachable!() + }; + (state, input.clone()) +} + +fn quantile(q: f64) -> AggIntent { + AggIntent::Quantile { col: None, q } +} + +fn sum() -> AggIntent { + AggIntent::Sum { col: None } +} + +fn topk(k: usize) -> AggIntent { + AggIntent::TopK { k } +} + +/// `metric[5m]` (`Range`) or `metric` (`Instant`, which a cross-series +/// aggregate reads): a selector over `metric{labels}`. +fn selector(memo: &mut Memo, kind: TimeRangeKind, metric: &str, labels: &[&str]) -> MemoGroupId { + let scan = scan_group(memo, metric, labels); + group( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind, + child: scan, + }), + ) +} + +/// `column op literal` over an aggregate's output. +fn compare(column: usize, op: CompareOpKind, value: ScalarValue) -> Predicate { + Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(value)), + semantics: ExprSemantics::Promql, + }) +} + +/// `Filter(pred)` over `child`, added as Stage 0 adds a consumer `Filter`. +fn filter(memo: &mut Memo, child: MemoGroupId, pred: Predicate) -> MemoGroupId { + group(memo, Operator::NonASAP(NonASAPOp::Filter { pred, child })) +} + +// ── Running example (concepts doc, section 1) ────────────────────────────── + +/// The doc's running example: query A reads p95 of `latency`, query B +/// reads p99 / p50 of it. KLL state group holds one build per θ of KLL's +/// list, DDSketch's state group likewise; the p95, p99 and p50 groups each +/// read either state through their own readout groups. The metric rule +/// (B's relative ratio cannot read KLL's rank error) drops edges in PR 4, +/// so Pass 1 offers both algorithms to every quantile group. +#[test] +fn running_example_gives_one_build_per_theta_and_separate_readouts() { + let mut memo = Memo::default(); + let g1 = selector(&mut memo, TimeRangeKind::Instant, "latency", &[]); + let g3 = aggregate_group( + &mut memo, + g1, + Reduction::by(vec![]), + vec![quantile(0.95)], + None, + ); + let g4 = aggregate_group( + &mut memo, + g1, + Reduction::by(vec![]), + vec![quantile(0.99)], + None, + ); + let g5 = aggregate_group( + &mut memo, + g1, + Reduction::by(vec![]), + vec![quantile(0.5)], + None, + ); + let g6 = group( + &mut memo, + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: g4, + rhs: g5, + }), + ); + query(&mut memo, g3); + query(&mut memo, g6); + let memo = pass1_memo(memo, None).unwrap(); + println!("{}", memo_printout(&memo)); + let (kll, _) = state_group(&memo, SketchAlgorithm::Kll); + let (ddsketch, _) = state_group(&memo, SketchAlgorithm::DDSketch); + for (state, algorithm) in [ + (kll, SketchAlgorithm::Kll), + (ddsketch, SketchAlgorithm::DDSketch), + ] { + let thetas: Vec = candidates(&memo, state) + .iter() + .map(|candidate| candidate.logical_properties.parameters().unwrap().clone()) + .collect(); + assert_eq!(thetas, parameter_list(algorithm)); + // θ is a property, never an operator argument: every build has the + // group's own expression. + assert!(candidates(&memo, state).iter().all(|candidate| { + candidate.expression.operator == memo.groups()[state.0 as usize].expression + })); + } + // Each quantile group: its exact aggregate and one readout per algorithm, + // each through a readout group of its own over the state group. + for (result, q) in [(g3, 0.95), (g4, 0.99), (g5, 0.5)] { + let readouts: Vec = candidates(&memo, result)[1..] + .iter() + .map(|candidate| *candidate.expression.operator.children()[0]) + .collect(); + assert_eq!(readouts.len(), 2, "G{}", result.0); + for (readout, state) in readouts.iter().zip([kll, ddsketch]) { + assert_eq!( + memo.groups()[readout.0 as usize].expression, + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q }, + }) + ); + assert!(candidates(&memo, *readout)[0] + .logical_properties + .parameters() + .is_none()); + } + } + // The ratio's group keeps its one candidate, its own expression. + assert_eq!(candidates(&memo, g6).len(), 1); +} + +// ── Series identity ──────────────────────────────────────────────────────── + +/// `quantile_over_time(0.9, latency[5m])` over series whose identity is +/// realized as a column (as the planner does before Stage 0): each readout +/// candidate outputs the per-series aggregate's exact schema, the +/// `$promql_series_identity` column included, so every readout keeps the +/// series it answers for. +#[test] +fn per_series_readouts_keep_the_series_identity() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "latency", &["job"]); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![quantile(0.9)], + None, + ); + query(&mut memo, per_series); + let memo = pass1_memo(memo, None).unwrap(); + let output = &memo.groups()[per_series.0 as usize].group_properties.schema; + assert!(output.closed && output.has_promql_series_identity()); + let readouts = &candidates(&memo, per_series)[1..]; + assert_eq!(readouts.len(), 2, "{}", memo_printout(&memo)); + for readout in readouts { + let Operator::NonASAP(NonASAPOp::Project { child, cols, .. }) = + &readout.expression.operator + else { + panic!("a renaming Project: {:?}", readout.expression.operator) + }; + let estimate = &memo.groups()[child.0 as usize].group_properties.schema; + assert!(estimate.has_promql_series_identity()); + let identity = output + .fields + .iter() + .position(|field| field.name == PROMQL_SERIES_IDENTITY) + .unwrap(); + let read = estimate + .fields + .iter() + .position(|field| field.name == PROMQL_SERIES_IDENTITY) + .unwrap(); + assert_eq!(cols[identity].expr, ScalarExpr::Column(read)); + } +} + +/// Without a realized identity the per-series aggregate's schema is open: +/// its rows carry labels it does not list. A renaming `Project` would drop +/// them, so no readout is offered; the group keeps its own expression. +#[test] +fn an_open_per_series_aggregate_offers_no_renamed_readout() { + let mut memo = Memo::default(); + let scan = group( + &mut memo, + scan_operator("latency", metric_schema(&["job"], vec![])), + ); + let rows = group( + &mut memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(300), + kind: TimeRangeKind::Range, + child: scan, + }), + ); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![quantile(0.9)], + None, + ); + query(&mut memo, per_series); + let memo = pass1_memo(memo, None).unwrap(); + assert!( + !memo.groups()[per_series.0 as usize] + .group_properties + .schema + .closed + ); + assert_eq!(candidates(&memo, per_series).len(), 1); +} + +// ── Target discovery ─────────────────────────────────────────────────────── + +/// Each measure of a multi-measure aggregate is its own target; a measure +/// whose only realization is its exact accumulator is still a target. +#[test] +fn multi_measure_aggregate_has_one_target_per_measure() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let both = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![sum(), quantile(0.9)], + None, + ); + query(&mut memo, both); + let targets = computation_targets(&memo, None).unwrap(); + let measures: Vec<(MemoGroupId, usize)> = targets + .iter() + .map(|target| (target.aggregate_group, target.measure_index)) + .collect(); + assert_eq!(measures, vec![(both, 0), (both, 1)]); +} + +/// An aggregate with no realization besides exact execution (no +/// accumulator, no summary: `stddev`) is not a target. +#[test] +fn aggregate_without_realization_is_not_a_target() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let stddev = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![AggIntent::StdDev { + col: None, + population: true, + }], + None, + ); + query(&mut memo, stddev); + assert!(computation_targets(&memo, None).unwrap().is_empty()); + let memo = pass1_memo(memo, None).unwrap(); + assert_eq!(candidates(&memo, stddev).len(), 1); +} + +/// An aggregate several queries read, or a scalar root reads twice, is one +/// target per measure: Pass 1 walks the groups. +#[test] +fn an_aggregate_read_by_several_roots_is_one_target() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let p = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + query(&mut memo, p); + query(&mut memo, p); + assert_eq!(computation_targets(&memo, None).unwrap().len(), 1); +} + +/// A memo whose groups already hold ASAP operators fails at the stage +/// boundary. +#[test] +fn asap_input_is_rejected() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let state = group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: rows, + summary_type: FieldDataType::exact(ExactKind::Sum), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }), + ); + query(&mut memo, state); + assert!(matches!( + pass1_memo(memo, None), + Err(PerComputationCandidateError::ContainsAsap) + )); +} + +/// The summaries the deployment does not support are not offered. +#[test] +fn unsupported_summaries_are_not_offered() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &[]); + let p = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + query(&mut memo, p); + let memo = pass1_memo(memo, Some(&[])).unwrap(); + assert_eq!(candidates(&memo, p).len(), 1); + assert!(groups_where(&memo, |e| matches!(e, Operator::ASAP(_))).is_empty()); +} + +// ── Candidate composition ────────────────────────────────────────────────── + +/// Every group has its own expression as a candidate, with no θ: plain +/// exact execution. +#[test] +fn every_group_offers_its_own_expression() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); + query(&mut memo, s); + let stage0 = memo.groups().len(); + let memo = pass1_memo(memo, None).unwrap(); + for index in 0..stage0 { + let group = &memo.groups()[index]; + let own = &group.candidates[0]; + assert_eq!(own.expression.operator, group.expression); + assert!(own.logical_properties.parameters().is_none()); + } +} + +/// An exact accumulator is a state group (`SummaryAgg` of the exact +/// summary type, no θ) read by a `FinalizeExactAccumulator` group, which the +/// aggregate's group reads through a renaming `Project`. +#[test] +fn exact_accumulator_is_a_state_group_and_a_finalize() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let s = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); + query(&mut memo, s); + let memo = pass1_memo(memo, None).unwrap(); + let states = groups_where(&memo, |e| { + matches!( + e, + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), + .. + }) + ) + }); + let [state] = states[..] else { + panic!("{}", memo_printout(&memo)) + }; + assert_eq!(candidates(&memo, state).len(), 1); + assert!(candidates(&memo, state)[0] + .logical_properties + .parameters() + .is_none()); + // The aggregate reads it through a `FinalizeExactAccumulator`, renamed + // to the aggregate's columns when its names differ. + let finalize = Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }); + let finalize_groups = groups_where(&memo, |e| *e == finalize); + assert!(candidates(&memo, s).iter().any(|c| { + c.expression.operator == finalize + || c.expression + .operator + .children() + .iter() + .any(|g| finalize_groups.contains(g)) + })); +} + +/// A multi-measure aggregate's candidate joins the measures' own groups +/// (one single-measure aggregate each, with its own candidates) on the +/// group keys. +#[test] +fn multi_measure_candidate_joins_single_measure_groups() { + let mut memo = Memo::default(); + // A SQL table `flows(src Utf8, bytes Float64)`. + let rows = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "flows".into(), + }, + predicates: vec![], + schema: Schema::new(vec![ + Field::plain("src", DataType::Utf8, false), + Field::plain("bytes", DataType::Float64, false), + ]), + }), + ); + let both = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![0]), + vec![ + AggIntent::Sum { col: Some(1) }, + AggIntent::Quantile { + col: Some(1), + q: 0.9, + }, + ], + None, + ); + query(&mut memo, both); + let memo = pass1_memo(memo, None).unwrap(); + let single = groups_where( + &memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Aggregate { measures, .. }) if measures.len() == 1), + ); + assert_eq!(single.len(), 2, "{}", memo_printout(&memo)); + // The quantile measure's group has its exact aggregate and two readouts. + assert!(single.iter().any(|g| candidates(&memo, *g).len() == 3)); + let joins = groups_where(&memo, |e| { + matches!(e, Operator::NonASAP(NonASAPOp::Join { .. })) + }); + assert_eq!(joins.len(), 1); + assert_eq!(candidates(&memo, both).len(), 2); +} + +/// Pass 1 sets no input requirement, a top-k readout's included: each +/// candidate has one empty requirement per input group, and Pass 2 sets +/// the readouts' required accuracy (a top-k's `heap_size` ≥ k among it). +#[test] +fn pass1_sets_no_input_requirement() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_job, + Reduction::by(vec![]), + vec![topk(15)], + None, + ); + let samples = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let p = aggregate_group( + &mut memo, + samples, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + query(&mut memo, ranked); + query(&mut memo, p); + let memo = pass1_memo(memo, None).unwrap(); + // The top-k readout is offered, and has no input requirement either. + assert!(!groups_where(&memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryEstimate { + query: SketchStatistic::TopK { k: 15 }, + .. + }) + )) + .is_empty()); + for group in memo.groups() { + for candidate in &group.candidates { + assert_eq!( + candidate.input_requirements, + vec![ + LogicalRequirements::default(); + candidate.expression.operator.children().len() + ] + ); + } + } +} + +// ── HAVING placement ─────────────────────────────────────────────────────── + +/// The aggregate's input with a key-only predicate pushed below it: a +/// `Filter` group over `rows`. +fn pushed_below(memo: &Memo, rows: MemoGroupId) -> Vec { + groups_where( + memo, + |e| matches!(e, Operator::NonASAP(NonASAPOp::Filter { child, .. }) if *child == rows), + ) +} + +/// A key-only `HAVING` (a consumer `Filter` over the group keys) gets +/// candidates in the `Filter`'s group that push it below the aggregate: a +/// `Filter` on the input rows, which the states read. The aggregate's own +/// group keeps its unfiltered candidates for its other consumers. +#[test] +fn key_only_having_is_pushed_below_into_the_filter_group() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + by_job, + compare(0, CompareOpKind::Eq, ScalarValue::Utf8("a".into())), + ); + query(&mut memo, having); + query(&mut memo, by_job); + let memo = pass1_memo(memo, None).unwrap(); + let below = pushed_below(&memo, rows); + assert_eq!(below.len(), 1, "{}", memo_printout(&memo)); + // The Filter group: its own expression, then one readout per algorithm + // over states built from the filtered rows. + assert_eq!(candidates(&memo, having).len(), 3); + assert_eq!(candidates(&memo, by_job).len(), 3); +} + +/// A `HAVING` over an aggregate value stays above the output: nothing is +/// pushed below, and the `Filter` group keeps its own expression only. +#[test] +fn value_having_stays_above() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + by_job, + compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0)), + ); + query(&mut memo, having); + let memo = pass1_memo(memo, None).unwrap(); + assert!(pushed_below(&memo, rows).is_empty()); + assert_eq!(candidates(&memo, having).len(), 1); +} + +/// A global aggregate's `HAVING` stays above even when it is constant: the +/// one output row exists with no input row. +#[test] +fn global_aggregate_having_stays_above() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &[]); + let p = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![]), + vec![quantile(0.9)], + None, + ); + let having = filter( + &mut memo, + p, + Predicate(ScalarExpr::Literal(ScalarValue::Boolean(false))), + ); + query(&mut memo, having); + let memo = pass1_memo(memo, None).unwrap(); + assert!(pushed_below(&memo, rows).is_empty()); +} + +/// The aggregate's own value `having` is a `Filter` above each readout: +/// the aggregate group's summary candidates are `Filter`s over a readout. +#[test] +fn own_value_having_filters_each_readout() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let by_job = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![quantile(0.9)], + Some(compare(1, CompareOpKind::Gt, ScalarValue::Float64(10.0))), + ); + query(&mut memo, by_job); + let memo = pass1_memo(memo, None).unwrap(); + let kinds: Vec<&str> = candidates(&memo, by_job) + .iter() + .map(|c| c.expression.operator.kind_name()) + .collect(); + assert_eq!(kinds, ["Aggregate", "Filter", "Filter"]); +} + +// ── Fused heaps, whole-expression top-k, ranked items, raw-data signs ────── + +/// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the top-k gets heaps +/// over the range's raw data samples, keyed by `job` (the ranked item), +/// weighted by the sample value with the open requirement that the samples +/// of `m` are non-negative (`RawDataSamplesOfMetric`). Each heap build +/// lists the absorbed sums' groups in `fused_groups`, and the top-k's own +/// heaps over its one-row-per-item input are pruned. +#[test] +fn whole_expression_topk_heaps_read_the_raw_samples() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_job, + Reduction::by(vec![]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked); + let memo = pass1_memo(memo, None).unwrap(); + for algorithm in [ + SketchAlgorithm::CmsWithHeap, + SketchAlgorithm::CountSketchWithHeap, + ] { + let (state, update) = state_group(&memo, algorithm); + assert_eq!( + *memo.groups()[state.0 as usize].expression.children()[0], + rows + ); + assert_eq!( + update.item, + Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) + ); + assert_eq!( + update.weight_domain, + WeightDomain::RawDataSamplesOfMetric { metric: "m".into() } + ); + for build in candidates(&memo, state) { + let mut fused = build.fused_groups.clone(); + fused.sort(); + assert_eq!(fused, vec![per_series, by_job]); + } + assert_eq!( + candidates(&memo, state).len(), + parameter_list(algorithm).len() + ); + } +} -use asap_types::stages::HowToComputeAggregateMeasure; +/// A top-k over a `count` ranks the counted items with unit weight, proven +/// non-negative. +#[test] +fn count_ranked_topk_counts_with_unit_weight() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let counts = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![AggIntent::Count], + None, + ); + let ranked = aggregate_group( + &mut memo, + counts, + Reduction::by(vec![]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked); + let memo = pass1_memo(memo, None).unwrap(); + let (_, update) = state_group(&memo, SketchAlgorithm::CmsWithHeap); + assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); + assert!(matches!( + update.weight_domain, + WeightDomain::NonNegative { .. } + )); +} + +/// The ranked item over several group keys is their tuple; the top-k's own +/// partition keys partition the heap and stay out of the tuple. +#[test] +fn ranked_item_is_the_keys_outside_the_partition() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job", "host"]); + let per_series = aggregate_group(&mut memo, rows, Reduction::PerEntity, vec![sum()], None); + let by_both = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2, 3]), + vec![sum()], + None, + ); + let ranked = aggregate_group( + &mut memo, + by_both, + Reduction::by(vec![0]), + vec![topk(5)], + None, + ); + query(&mut memo, ranked); + let memo = pass1_memo(memo, None).unwrap(); + let (_, update) = state_group(&memo, SketchAlgorithm::CountSketchWithHeap); + assert_eq!( + update.item, + Some(SummaryInputExpr::Column(ColumnRef::Named("host".into()))) + ); +} + +// ── Nested distributive merges ───────────────────────────────────────────── + +/// `sum by (job) (sum_over_time(m[5m]))`, and likewise `min` of `min` and +/// `max` of `max`, gets a merged build: one exact accumulator of that kind +/// over the raw samples grouped by `job`, a candidate of the outer +/// aggregate's own accumulator state group, listing the inner aggregate's +/// group in `fused_groups`. +#[test] +fn nested_distributive_aggregate_merges_into_the_outer_state_group() { + for (intent, kind) in [ + (sum(), ExactKind::Sum), + (AggIntent::Min { col: None }, ExactKind::Min), + (AggIntent::Max { col: None }, ExactKind::Max), + ] { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![intent.clone()], + None, + ); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![intent], + None, + ); + query(&mut memo, by_job); + let memo = pass1_memo(memo, None).unwrap(); + let outer_state = groups_where( + &memo, + |e| matches!(e, Operator::ASAP(ASAPOp::SummaryAgg { child, summary_type: FieldDataType::ExactAggregate(state, _), .. }) if *child == per_series && *state == kind), + ); + let [outer_state] = outer_state[..] else { + panic!("{kind:?}: {}", memo_printout(&memo)) + }; + let builds = candidates(&memo, outer_state); + assert_eq!(builds.len(), 2, "{kind:?}: {}", memo_printout(&memo)); + let merged = &builds[1]; + assert_eq!(merged.expression.operator.children(), vec![&rows]); + assert_eq!(merged.fused_groups, vec![per_series]); + } +} + +/// The merge needs the same distributive aggregate twice: a `sum` over a +/// `max`, or a `min` over a `max`, gets none. +#[test] +fn nested_merge_needs_the_same_distributive_aggregate() { + for outer in [sum(), AggIntent::Min { col: None }] { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Range, "m", &["job"]); + let per_series = aggregate_group( + &mut memo, + rows, + Reduction::PerEntity, + vec![AggIntent::Max { col: None }], + None, + ); + let by_job = aggregate_group( + &mut memo, + per_series, + Reduction::by(vec![2]), + vec![outer], + None, + ); + query(&mut memo, by_job); + let memo = pass1_memo(memo, None).unwrap(); + assert!(memo + .groups() + .iter() + .flat_map(|g| &g.candidates) + .all(|c| c.fused_groups.is_empty())); + } +} + +// ── Stream norms (update weights) ────────────────────────────────────────── + +/// A Count-Min count adds one per row (bounded by the row count); a sum's +/// state reads the raw value column (bounded by the source's norm). +#[test] +fn update_weights_follow_the_measure() { + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let counts = aggregate_group( + &mut memo, + rows, + Reduction::by(vec![2]), + vec![AggIntent::Count], + None, + ); + query(&mut memo, counts); + let memo = pass1_memo(memo, None).unwrap(); + let (_, update) = state_group(&memo, SketchAlgorithm::Cms); + assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); + let mut memo = Memo::default(); + let rows = selector(&mut memo, TimeRangeKind::Instant, "m", &["job"]); + let sums = aggregate_group(&mut memo, rows, Reduction::by(vec![2]), vec![sum()], None); + query(&mut memo, sums); + let memo = pass1_memo(memo, None).unwrap(); + let state = groups_where( + &memo, + |e| matches!(e, Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) if input.weight == SummaryInputExpr::Column(ColumnRef::SampleValue)), + ); + assert_eq!(state.len(), 1); +} + +// ── Intents ──────────────────────────────────────────────────────────────── fn candidates_for(intent: AggIntent) -> Vec { computation_candidates_for_intent(&intent).candidates } + fn algorithms(choices: &[HowToComputeAggregateMeasure]) -> Vec { choices .iter() @@ -51,7 +994,7 @@ fn frequency_statistics_keep_universal_choices() { /// TopK keeps both declared heap substrates without choosing an implementation. #[test] fn topk_keeps_both_specialized_heap_choices() { - let choices = candidates_for(AggIntent::TopK { k: 10 }); + let choices = candidates_for(topk(10)); assert_eq!(choices[0], HowToComputeAggregateMeasure::Exact); assert_eq!( algorithms(&choices), @@ -62,10 +1005,28 @@ fn topk_keeps_both_specialized_heap_choices() { ); } -/// Summary candidates are unsized, so they do not depend on the requested error. +/// Approximate count lists one exact option, its accumulator, beside the +/// summaries (the plain aggregate is the group's own expression). +#[test] +fn approximate_count_lists_its_accumulator_and_summaries() { + let choices = candidates_for(AggIntent::Count); + assert!(!choices.contains(&HowToComputeAggregateMeasure::Exact)); + assert_eq!( + choices[0], + HowToComputeAggregateMeasure::ExactAccumulator(vec![ExactKind::Count]) + ); + assert!(choices.contains(&HowToComputeAggregateMeasure::Summary( + SketchAlgorithm::UnivMon + ))); +} + +/// Count Sketch's signed counters give no total, so Pass 1 offers no Count +/// Sketch for a count. #[test] -fn summary_candidates_are_unsized() { - let loose = candidates_for(AggIntent::Count); - let tight = candidates_for(AggIntent::Count); - assert_eq!(loose, tight); +fn a_count_offers_no_count_sketch() { + assert!( + !candidates_for(AggIntent::Count).contains(&HowToComputeAggregateMeasure::Summary( + SketchAlgorithm::CountSketch + )) + ); } diff --git a/crates/stage1-pass1-per-computation-candidates/src/realization.rs b/crates/stage1-pass1-per-computation-candidates/src/realization.rs index 4885bd334..16ef12ac6 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/realization.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/realization.rs @@ -4,24 +4,26 @@ //! keyed-additive rule for a heap sketch that reads a larger sub-DAG. use asap_types::ir::operator::agg_intent::AggIntent; +use asap_types::ir::operator::non_asap::any_measure_filtered; +use asap_types::ir::operator::operator_properties::Source; use asap_types::ir::operator::Reduction; -use asap_types::ir::scalar::column_resolution::resolve_column_ref; +use asap_types::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use asap_types::ir::scalar::expr_ir::ColumnRef; +use asap_types::ir::schema::topk::ranked_column_of_group; +use asap_types::ir::schema::ColumnId; use asap_types::ir::schema::{ - EntityIdentity, Field, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, + EntityIdentity, Field, FieldDataType, NonNegativeWeightProof, Schema, SketchAlgorithm, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, }; -use asap_types::ir::{NonASAPOp, OperatorNode}; -#[cfg(test)] -use asap_types::ir::{Operator, ScalarExpr}; -use std::rc::Rc; +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId}; use asap_accuracy::sizing::sketch_statistic; -/// The sketch families the planner sizes and offers, in the order Pass 1 -/// offers those that answer an intent's statistic: the specialized families +/// The sketch algorithms the planner sizes and offers, in the order Pass 1 +/// offers those that answer an intent's statistic: the specialized algorithms /// before the universal UnivMon. -const FAMILY_PREFERENCE: &[SketchAlgorithm] = &[ +const ALGORITHM_PREFERENCE: &[SketchAlgorithm] = &[ SketchAlgorithm::Kll, SketchAlgorithm::DDSketch, SketchAlgorithm::Hll, @@ -58,20 +60,78 @@ fn candidates_by( // over a single value stream, and `realize_value_frequency_summary_input` // would feed it one column of the tuple. let tuple = matches!(intent, AggIntent::Cardinality { cols, .. } if cols.len() > 1); - FAMILY_PREFERENCE + ALGORITHM_PREFERENCE .iter() - .filter(|family| capability(family).answers(&statistic)) - .filter(|family| !(tuple && **family == SketchAlgorithm::UnivMon)) + .filter(|algorithm| capability(algorithm).answers(&statistic)) + .filter(|algorithm| !(tuple && **algorithm == SketchAlgorithm::UnivMon)) .cloned() .collect() } +/// The column fed into a *single-column* summary: the intent's leading +/// positional input resolved to a name against the child schema, or the PromQL +/// sample value when it reads none. Callers are responsible for only reaching +/// here with a one-column intent — [`summarised_input`] is the general form. +pub(crate) fn summarised_column(intent: &AggIntent, child_schema: &Schema) -> ColumnRef { + match intent + .input_cols() + .first() + .and_then(|id| child_schema.fields.get(*id)) + { + Some(c) => c.column_ref(), + None => ColumnRef::SampleValue, + } +} + +/// The column a top-k ranks group `child` by ([`ranked_column_of_group`]), +/// also as a reference: the sample value when it is that column, else the +/// column by name. +pub(crate) fn ranked_value( + memo: &Memo, + child: MemoGroupId, +) -> Result<(ColumnId, ColumnRef), ResolveError> { + let ranked = ranked_column_of_group(memo, child)?; + let column = match resolve_column_ref(&ColumnRef::SampleValue, schema(memo, child)) { + Ok(value) if value == ranked => ColumnRef::SampleValue, + _ => schema(memo, child).fields[ranked].column_ref(), + }; + Ok((ranked, column)) +} + +/// What the summary consumes per input row. An intent that reads one column (or +/// none) feeds that column; `COUNT(DISTINCT a, b)` feeds the whole tuple as one +/// item, so the distinct-count sketch hashes `(a, b)` rather than `a` — the +/// difference between tuple cardinality and single-column cardinality. +/// +/// A tuple leg outside the child schema is an error rather than +/// [`summarised_column`]'s sample-value fallback: a leg has no sample-value +/// reading, and silently dropping one would under-count: `None` then. +pub(crate) fn summarised_input( + intent: &AggIntent, + child_schema: &Schema, +) -> Option { + let cols = intent.input_cols(); + if cols.len() < 2 { + return Some(SummaryInputExpr::Column(summarised_column( + intent, + child_schema, + ))); + } + let legs = cols + .iter() + .map(|id| child_schema.fields.get(*id).map(Field::column_ref)) + .collect::>>()?; + Some(SummaryInputExpr::Tuple( + legs.into_iter().map(SummaryInputExpr::Column).collect(), + )) +} + /// The physical input consumed by one summary realization. Most summaries /// consume the logical aggregate's immediate child and summarize its declared /// input value. Composite realizations can instead consume a larger /// logical sub-DAG and bind a different key or value. pub(crate) struct PhysicalSummaryInput { - pub(crate) child: Rc, + pub(crate) child: MemoGroupId, pub(crate) input: SummaryUpdate, } @@ -81,15 +141,16 @@ pub(crate) struct PhysicalSummaryInput { /// when the rule does not apply or the shape is unsupported (each reason is /// noted where it returns). pub(crate) fn realize_keyed_additive_summary_input( + memo: &Memo, intent: &AggIntent, - family: &FieldDataType, + summary_type: &FieldDataType, output_reduction: &Reduction, - child: &Rc, + child: MemoGroupId, ) -> Option { if !matches!(intent, AggIntent::TopK { .. }) { return None; } - let FieldDataType::Sketch(kind) = family else { + let FieldDataType::Sketch(kind) = summary_type else { return None; }; let heap_algorithm = kind; @@ -99,28 +160,29 @@ pub(crate) fn realize_keyed_additive_summary_input( ) { return None; } - let Some(NonASAPOp::Aggregate { + let Operator::NonASAP(NonASAPOp::Aggregate { reduction, measures, filters, having: None, child: raw_child, .. - }) = child.non_asap() + }) = expression(memo, child) else { return None; }; + let raw_child = *raw_child; // The heap sketch consumes every raw row; a filtered measure counts only // some of them (#466). - if asap_types::ir::operator::non_asap::any_measure_filtered(filters) { + if any_measure_filtered(filters) { return None; } // A grouped sum of per-series `rate`/`increase`/`irate` values: the // metric they derive from, whose declared type decides their sign. - let derivative_of = match (measures.as_slice(), raw_child.non_asap()) { + let derivative_of = match (measures.as_slice(), expression(memo, raw_child)) { ( [AggIntent::Sum { .. }], - Some(NonASAPOp::Aggregate { + Operator::NonASAP(NonASAPOp::Aggregate { measures: derivative, child: samples, .. @@ -130,9 +192,7 @@ pub(crate) fn realize_keyed_additive_summary_input( [AggIntent::Rate | AggIntent::Increase | AggIntent::IRate] ) => { - Some(asap_accuracy::propagation::raw_data_samples_of_metric( - samples, - )) + Some(raw_data_samples_of_metric(memo, *samples)) } _ => None, }; @@ -140,7 +200,7 @@ pub(crate) fn realize_keyed_additive_summary_input( // `sum by (G) (sum_over_time(m[W]))` or `... count_over_time ...`: the // grouped sum of per-series sums (counts) of raw data samples is their sum // (count) by `G`, so the heap reads those samples. - let samples = summed_per_series_samples(child); + let samples = summed_per_series_samples(memo, child); // A raw data sample's count or a count's row weighs one. let counted = match &samples { Some((_, inner)) => matches!(inner, AggIntent::Count), @@ -153,7 +213,11 @@ pub(crate) fn realize_keyed_additive_summary_input( } [AggIntent::Sum { col }] => SummaryInputExpr::Column(match col { None => ColumnRef::SampleValue, - Some(index) => match raw_child.schema.fields.get(*index).map(Field::column_ref) { + Some(index) => match schema(memo, raw_child) + .fields + .get(*index) + .map(Field::column_ref) + { Some(column) => column, None => { // Unsupported: sum-ranked Top-K value column is outside the raw input schema. @@ -183,7 +247,12 @@ pub(crate) fn realize_keyed_additive_summary_input( Reduction::PerEntity => vec![], Reduction::Reduce(keys) => keys .iter() - .filter_map(|index| child.schema.fields.get(*index).map(Field::column_ref)) + .filter_map(|index| { + schema(memo, child) + .fields + .get(*index) + .map(Field::column_ref) + }) .collect(), }; let item = match reduction { @@ -193,7 +262,12 @@ pub(crate) fn realize_keyed_additive_summary_input( Reduction::Reduce(keys) if !keys.is_without() && !keys.is_empty() => { let Some(columns) = keys .iter() - .map(|index| raw_child.schema.fields.get(*index).map(Field::column_ref)) + .map(|index| { + schema(memo, raw_child) + .fields + .get(*index) + .map(Field::column_ref) + }) .collect::>>() else { // Unsupported: ranked item column is outside the raw input schema. @@ -223,7 +297,7 @@ pub(crate) fn realize_keyed_additive_summary_input( } }; Some(PhysicalSummaryInput { - child: samples.map_or_else(|| Rc::clone(raw_child), |(rows, _)| rows), + child: samples.map_or(raw_child, |(rows, _)| rows), input: SummaryUpdate { item: Some(item), weight, @@ -238,30 +312,32 @@ pub(crate) fn realize_keyed_additive_summary_input( /// `count` of raw data samples (`sum by (G) (sum_over_time(m[W]))`), neither /// filtered nor with `HAVING`; `None` otherwise. The grouped sum is then the /// sum (count) of those samples by `G`. -fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc, AggIntent)> { - use asap_types::ir::operator::non_asap::any_measure_filtered; - let Some(NonASAPOp::Aggregate { +fn summed_per_series_samples( + memo: &Memo, + grouped: MemoGroupId, +) -> Option<(MemoGroupId, AggIntent)> { + let Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::Reduce(by), measures, filters, having: None, child: per_series, .. - }) = grouped.non_asap() + }) = expression(memo, grouped) else { return None; }; let [AggIntent::Sum { col }] = measures.as_slice() else { return None; }; - let Some(NonASAPOp::Aggregate { + let Operator::NonASAP(NonASAPOp::Aggregate { reduction: Reduction::PerEntity, measures: inner, filters: inner_filters, having: None, child: rows, .. - }) = per_series.non_asap() + }) = expression(memo, *per_series) else { return None; }; @@ -271,76 +347,151 @@ fn summed_per_series_samples(grouped: &OperatorNode) -> Option<(Rc if by.is_without() || any_measure_filtered(filters) || any_measure_filtered(inner_filters) - || matches!(rows.non_asap(), Some(NonASAPOp::Aggregate { .. })) + || matches!( + expression(memo, *rows), + Operator::NonASAP(NonASAPOp::Aggregate { .. }) + ) { return None; } // The grouped sum reads the per-series value. - let value = resolve_column_ref(&ColumnRef::SampleValue, &per_series.schema).ok()?; + let value = resolve_column_ref(&ColumnRef::SampleValue, schema(memo, *per_series)).ok()?; if col.is_some_and(|col| col != value) || by.keys().contains(&value) { return None; } if by.keys().iter().any(|&key| { - per_series - .schema + schema(memo, *per_series) .fields .get(key) .is_none_or(|field| field.name == "__name__") }) { return None; } - Some((Rc::clone(rows), inner.clone())) + Some((*rows, inner.clone())) +} + +/// The expression of group `group`. +pub(crate) fn expression(memo: &Memo, group: MemoGroupId) -> &Operator { + &memo.groups()[group.0 as usize].expression +} + +/// The output schema of group `group`. +pub(crate) fn schema(memo: &Memo, group: MemoGroupId) -> &Schema { + &memo.groups()[group.0 as usize].group_properties.schema +} + +/// The metric whose raw data samples, or plain sums of them, are every value +/// group `group` outputs, or `None`: walking down the groups' expressions +/// through a time range, an `offset`-only time shift (it moves the window, +/// not the samples) and a plain `sum`, to a time-series scan. Any other +/// operator (rate and arithmetic included) ends it. Whether the metric's +/// samples are non-negative is plan selection's question +/// ([`WeightDomain::RawDataSamplesOfMetric`]). +pub(crate) fn raw_data_samples_of_metric(memo: &Memo, group: MemoGroupId) -> Option { + let Operator::NonASAP(operator) = expression(memo, group) else { + return None; + }; + match operator { + NonASAPOp::Scan { + source: Source::TimeSeries { metric }, + .. + } => Some(metric.clone()), + NonASAPOp::TimeRange { child, .. } => raw_data_samples_of_metric(memo, *child), + NonASAPOp::TimeShift { shift, child } if shift.at.is_none() => { + raw_data_samples_of_metric(memo, *child) + } + NonASAPOp::Aggregate { + measures, child, .. + } if matches!(measures.as_slice(), [AggIntent::Sum { col: None }]) => { + raw_data_samples_of_metric(memo, *child) + } + _ => None, + } } #[cfg(test)] mod tests { use super::*; - use crate::test_fixtures::metric_scan; + use crate::test_fixtures::{group, metric_schema, scan_group}; use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::{ExprSemantics, Predicate}; + use asap_types::ir::{ExprSemantics, Predicate, ScalarExpr}; /// A heap sketch consumes every raw row, so the keyed input rule refuses /// a TopK over a filtered count. #[test] fn keyed_heap_input_refuses_a_filtered_count() { - let count_by_job = |filter: Option| { - OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Count], - output_names: vec![], - filters: vec![filter], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap() + let mut memo = Memo::default(); + let rows = scan_group(&mut memo, "m", &["job"]); + let mut count_by_job = |filter: Option>| { + group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![2]), + measures: vec![AggIntent::Count], + output_names: vec![String::new()], + filters: vec![filter], + having: None, + child: rows, + }), + ) }; + let unfiltered = count_by_job(None); + let value_above_one = Predicate(ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(1)), + op: CompareOpKind::Gt, + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + semantics: ExprSemantics::Promql, + }); + let filtered = count_by_job(Some(value_above_one)); let intent = AggIntent::TopK { k: 5 }; - let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); + let summary_type = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); let output = Reduction::Reduce(asap_types::ir::operator::operator_properties::GroupKeys::none()); assert!(realize_keyed_additive_summary_input( + &memo, &intent, - &family, + &summary_type, &output, - &count_by_job(None) + unfiltered ) .is_some()); - let value_above_one = Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(1)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(1.0))), - semantics: ExprSemantics::Promql, - }); assert!(realize_keyed_additive_summary_input( + &memo, &intent, - &family, + &summary_type, &output, - &count_by_job(Some(value_above_one)) + filtered ) .is_none()); } - /// Each approximable intent is offered the families it was offered when + /// A summary reads the intent's own input columns: one column by name, + /// none as the sample value, several as one hashed tuple. + #[test] + fn summarised_input_reads_the_intents_columns() { + // `metric_schema(&["job"], ..)` → columns [ts=0, value=1, job=2]. + let schema = metric_schema(&["job"], vec![]); + let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); + let cases = [ + (vec![2], column("job")), + (vec![1], column("value")), + (vec![], SummaryInputExpr::Column(ColumnRef::SampleValue)), + ( + vec![1, 2], + SummaryInputExpr::Tuple(vec![column("value"), column("job")]), + ), + ]; + for (cols, want) in cases { + let intent = AggIntent::Cardinality { cols: cols.clone() }; + assert_eq!( + summarised_input(&intent, &schema), + Some(want), + "wrong summarised input for cols={cols:?}" + ); + } + } + + /// Each approximable intent is offered the algorithms it was offered when /// the list was written by hand, in the same order; a non-sketch intent /// is offered none. #[test] @@ -369,13 +520,13 @@ mod tests { } } - /// Dropping a readout from a family's table row drops the family from + /// Dropping a readout from an algorithm's table row drops the algorithm from /// the candidates of the intents that read it, with no other edit. #[test] - fn a_dropped_readout_drops_the_family() { + fn a_dropped_readout_drops_the_algorithm() { // Count-Min-with-heap answered from Count-Min's row: no top-k readout. - let heap_without_top_k = |family: &SketchAlgorithm| { - SummaryTypeCapability::of(&FieldDataType::Sketch(match family { + let heap_without_top_k = |algorithm: &SketchAlgorithm| { + SummaryTypeCapability::of(&FieldDataType::Sketch(match algorithm { SketchAlgorithm::CmsWithHeap => SketchAlgorithm::Cms, other => *other, })) @@ -387,12 +538,12 @@ mod tests { ); } - /// The preference order names each family it offers once. + /// The preference order names each algorithm it offers once. #[test] - fn preference_order_names_each_family_once() { + fn preference_order_names_each_algorithm_once() { let mut seen = std::collections::HashSet::new(); - for family in FAMILY_PREFERENCE { - assert!(seen.insert(family), "{family:?} twice"); + for algorithm in ALGORITHM_PREFERENCE { + assert!(seen.insert(algorithm), "{algorithm:?} twice"); } } } diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs index a69c984be..fa648844e 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -1,22 +1,16 @@ -//! Fixtures this crate's own tests share. Every builder returns an -//! `Rc` whose schema is derived by `OperatorNode::new_shared`, -//! so a fixture is exactly what a front end would hand the planner. +//! Fixtures this crate's own tests share. The memo builders add Stage 0's +//! groups through `Memo::find_or_add_group` and `Memo::add_query`, as +//! Stage 0 does, so no test calls another stage crate. -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::Source; +use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, OperatorNode}; - -/// A `TimeSeries("m")` scan over `[ts(0), value(1), labels...]`, time index 0, -/// no unique key. -pub fn metric_scan(labels: &[&str]) -> Rc { - metric_scan_with_keys(labels, vec![]) -} +use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot}; +use asap_types::stages::{Memo, MemoGroupId, MemoQuery}; -/// [`metric_scan`] with explicit `unique_keys` (a `[[0]]` key makes CSE -/// willing to hoist the scan). -pub fn metric_scan_with_keys(labels: &[&str], unique_keys: Vec>) -> Rc { +/// `[ts(0), value(1), labels...]`, time index 0, with `unique_keys`. +pub fn metric_schema(labels: &[&str], unique_keys: Vec>) -> Schema { let mut columns = vec![ Field::plain("ts", DataType::Timestamp, false), Field::plain("value", DataType::Float64, false), @@ -26,24 +20,64 @@ pub fn metric_scan_with_keys(labels: &[&str], unique_keys: Vec>) - .iter() .map(|n| Field::plain(*n, DataType::Utf8, true)), ); - scan("m", Schema::with_time_index(columns, 0, unique_keys)) + Schema::with_time_index(columns, 0, unique_keys) } -/// A predicate-free `TimeSeries(metric)` scan with the given schema. -pub fn scan(metric: &str, schema: Schema) -> Rc { - scan_from( - Source::TimeSeries { +/// A predicate-free `TimeSeries(metric)` scan with `schema`. +pub fn scan_operator(metric: &str, schema: Schema) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: metric.into(), }, + predicates: vec![], schema, + }) +} + +/// The group of `operator`, added as Stage 0 adds it. +pub fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { + memo.find_or_add_group(operator).unwrap() +} + +/// The group of a `TimeSeries(metric)` scan over [`metric_schema`] with its +/// series identity realized as the planner realizes it before Stage 0 +/// (`with_promql_series_identity`): a closed schema whose last column is +/// `$promql_series_identity`, keyed by it and the timestamp. +pub fn scan_group(memo: &mut Memo, metric: &str, labels: &[&str]) -> MemoGroupId { + let open = + OperatorNode::new_shared(scan_operator(metric, metric_schema(labels, vec![]))).unwrap(); + let realized = with_promql_series_identity(&open).unwrap(); + group(memo, scan_operator(metric, realized.schema.clone())) +} + +/// The group of an `Aggregate` of `measures` (named by their intents) over +/// `child`, grouped by `reduction`, unfiltered, with `having`. +pub fn aggregate_group( + memo: &mut Memo, + child: MemoGroupId, + reduction: Reduction, + measures: Vec, + having: Option>, +) -> MemoGroupId { + let count = measures.len(); + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction, + measures, + output_names: vec![String::new(); count], + filters: vec![None; count], + having, + }), ) } -pub fn scan_from(source: Source, schema: Schema) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source, - predicates: vec![], - schema, - })) - .unwrap() +/// Adds a query whose root is group `root`. +pub fn query(memo: &mut Memo, root: MemoGroupId) -> usize { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(root), + accuracy_target: None, + values_per_evaluation: None, + }) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_support.rs b/crates/stage1-pass1-per-computation-candidates/src/test_support.rs deleted file mode 100644 index f44495593..000000000 --- a/crates/stage1-pass1-per-computation-candidates/src/test_support.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! The Pass 1 internals other crates' tests call (feature `test-support`): -//! Pass 1's working form and single-candidate composition. - -pub use crate::per_computation_candidates::generate_per_computation_candidates; diff --git a/crates/types/src/ir/operator/asap.rs b/crates/types/src/ir/operator/asap.rs index 1cf4992d3..a496416f8 100644 --- a/crates/types/src/ir/operator/asap.rs +++ b/crates/types/src/ir/operator/asap.rs @@ -1052,10 +1052,10 @@ fn topk_schema_of_build( let complete_item_identity = matches!( input.item.as_ref()?, SummaryInputExpr::Column(_) | SummaryInputExpr::Tuple(_) - ) || child_schema.closed - || columns.iter().any(|&index| { - child_schema.fields[index].name == crate::ir::schema::PROMQL_SERIES_IDENTITY - }); + ) || crate::ir::schema::topk::item_identity_is_complete( + child_schema, + columns.iter().copied(), + ); let mut fields: Vec = state_schema .fields .iter() @@ -1072,12 +1072,12 @@ fn topk_schema_of_build( } // One row per (partition, item), not one row per partition. Preserve this // key through the ranked-row projection used by the logical TopK result. - let unique_keys = if complete_item_identity && !fields.is_empty() { - vec![(0..fields.len()).collect()] + fields.push(Field::plain("topk", DataType::Float64, false)); + let unique_keys = if complete_item_identity { + crate::ir::schema::topk::ranked_rows_unique_keys(fields.len(), fields.len() - 1) } else { Vec::new() }; - fields.push(Field::plain("topk", DataType::Float64, false)); Some(Schema { fields, unique_keys, diff --git a/crates/types/src/ir/operator/non_asap.rs b/crates/types/src/ir/operator/non_asap.rs index a3ba976af..60a21ad68 100644 --- a/crates/types/src/ir/operator/non_asap.rs +++ b/crates/types/src/ir/operator/non_asap.rs @@ -588,11 +588,17 @@ impl NonASAPOp { let name = crate::ir::schema::topk::topk_output_name(*k, output_names); let cols = crate::ir::schema::topk::ranked_row_projection(child, reduction, &name)?; - return NonASAPOp::Project { + let mut output = NonASAPOp::Project { cols, child: child.clone(), } - .output_schema(); + .output_schema()?; + output.unique_keys = crate::ir::schema::topk::exact_topk_unique_keys( + child, + reduction.group_keys().map_or(0, |keys| keys.len()), + &output, + ); + return Ok(output); } let mut output = aggregate_output_schema(child.schema(), reduction, measures, output_names)?; @@ -1326,7 +1332,7 @@ fn default_proj_name( } /// Whether any aggregate measure has its own input predicate. -pub fn any_measure_filtered(filters: &[Option]) -> bool { +pub fn any_measure_filtered(filters: &[Option>]) -> bool { filters.iter().any(Option::is_some) } diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index de89631f5..89d9f819d 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -5,6 +5,7 @@ use crate::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use crate::ir::scalar::{ColumnRef, ProjectItem}; use crate::ir::schema::{ColumnId, DataType, FieldDataType, PROMQL_SERIES_IDENTITY}; use crate::ir::{ASAPOp, NonASAPOp, Operator, ScalarExpr}; +use crate::stages::{Memo, MemoGroupAsInput, MemoGroupId}; /// The column a top-k ranks `child` by. Over a one-measure grouped /// aggregate (the SQL heavy-hitter shape and PromQL `topk(k, sum by ..)`) it @@ -13,9 +14,40 @@ use crate::ir::{ASAPOp, NonASAPOp, Operator, ScalarExpr}; /// `FinalizeExactAccumulator` or `SummaryEstimate`, under the `Project` /// restoring its names (a renaming) and a value `HAVING`'s `Filter`. Over /// per-series rows (PromQL `topk by (..) (k, rows)`) it is the sample value. -pub fn ranked_column(child: &I) -> Result { +pub(crate) fn ranked_column(child: &I) -> Result { + match ranked_source(child) { + RankedSource::TopK(reduction) => Ok(reduction.group_keys().map_or(0, |keys| keys.len())), + RankedSource::OneRowPerItem(Reduction::Reduce(_)) => Ok(child.schema().fields.len() - 1), + RankedSource::OneRowPerItem(Reduction::PerEntity) | RankedSource::Rows => { + resolve_column_ref(&ColumnRef::SampleValue, child.schema()) + } + } +} + +/// [`ranked_column`] of memo group `group`: the column a top-k reading the +/// group ranks it by. +pub fn ranked_column_of_group(memo: &Memo, group: MemoGroupId) -> Result { + ranked_column(&MemoGroupAsInput { + memo, + group_id: group, + }) +} + +/// What a top-k's input rows are, seen through a renaming `Project`, a +/// value `HAVING`'s `Filter`, a finalize or a summary readout. +enum RankedSource { + /// Another top-k's ranked rows, grouped by this reduction. + TopK(Reduction), + /// One row per item: a one-measure aggregate (or a summary build), per + /// group of this reduction or per series. + OneRowPerItem(Reduction), + /// Any other rows, possibly several per item. + Rows, +} + +fn ranked_source(child: &I) -> RankedSource { let mut node = child.clone(); - let reduction = loop { + loop { match node.operator() { // Only a renaming keeps the measure last. Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) @@ -35,23 +67,73 @@ pub fn ranked_column(child: &I) -> Result node = child, - Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => break Some(reduction), + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => { + return RankedSource::OneRowPerItem(reduction) + } Operator::NonASAP(NonASAPOp::Aggregate { reduction, measures, .. }) if measures.len() == 1 => { if matches!(measures[0], crate::ir::operator::AggIntent::TopK { .. }) { - return Ok(reduction.group_keys().map_or(0, |keys| keys.len())); + return RankedSource::TopK(reduction); } - break Some(reduction); + return RankedSource::OneRowPerItem(reduction); } - _ => break None, + _ => return RankedSource::Rows, } - }; - match reduction { - Some(Reduction::Reduce(_)) => Ok(child.schema().fields.len() - 1), - _ => resolve_column_ref(&ColumnRef::SampleValue, child.schema()), + } +} + +/// Whether the item columns `item_columns` of `input` identify each item +/// completely: `input` lists every column its rows carry (it is closed), or +/// the items include the realized PromQL series identity. An open PromQL +/// label set may carry unlisted labels, so its listed columns cannot. +pub(crate) fn item_identity_is_complete( + input: &crate::ir::schema::Schema, + item_columns: impl IntoIterator, +) -> bool { + input.closed + || item_columns + .into_iter() + .any(|index| input.fields[index].name == PROMQL_SERIES_IDENTITY) +} + +/// The unique key of ranked rows (one row per partition and item) with +/// `columns` columns, the ranked value at `value`: every other column. The +/// one rule for a top-k's ranked rows, exact or a heap's readout. +pub(crate) fn ranked_rows_unique_keys(columns: usize, value: ColumnId) -> Vec> { + let key: Vec = (0..columns).filter(|&column| column != value).collect(); + if key.is_empty() { + Vec::new() + } else { + vec![key] + } +} + +/// The unique keys of an exact top-k's ranked rows (`output`, the +/// projection [`ranked_row_projection`] gives, its ranked value after the +/// `partitions` keys) over `child`: one row per (partition, item) when +/// `child` has one row per item whose identity is complete +/// ([`item_identity_is_complete`]); else the projection's own keys. +pub(crate) fn exact_topk_unique_keys( + child: &I, + partitions: usize, + output: &crate::ir::schema::Schema, +) -> Vec> { + let item_columns = (0..child.schema().fields.len()).filter(|&index| { + output + .fields + .iter() + .skip(partitions + 1) + .any(|field| field.name == child.schema().fields[index].name) + }); + if matches!(ranked_source(child), RankedSource::OneRowPerItem(_)) + && item_identity_is_complete(child.schema(), item_columns) + { + ranked_rows_unique_keys(output.fields.len(), partitions) + } else { + output.unique_keys.clone() } } @@ -152,3 +234,129 @@ pub fn series_identity_item( SummaryInputExpr::Tuple(items) } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use crate::ir::operator::operator_properties::{Reduction, Source}; + use crate::ir::operator::AggIntent; + use crate::ir::scalar::ColumnRef; + use crate::ir::schema::schema_support::with_promql_series_identity; + use crate::ir::schema::{ + DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchStatistic, SummaryUpdate, + WeightDomain, + }; + use crate::ir::{ + ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, ScalarExpr, TimeRangeKind, + }; + use crate::stages::{Memo, MemoGroupId}; + + fn schema_of(memo: &Memo, group: MemoGroupId) -> &Schema { + &memo.groups()[group.0 as usize].group_properties.schema + } + + /// `topk by (job) (10, sum_over_time(m[1m]))` over series whose identity + /// is realized: the exact top-k's group and a heap's ranked readout of + /// the same rows (a `Project` naming the estimate's columns as the + /// top-k's) derive the same unique keys, one row per (job, series). + #[test] + fn exact_topk_and_heap_readout_derive_the_same_unique_keys() { + let open = Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, true), + ], + 0, + vec![], + ); + fn scan(schema: Schema) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema, + }) + } + let realized = with_promql_series_identity(&OperatorNode::new_shared(scan(open)).unwrap()) + .unwrap() + .schema + .clone(); + let mut memo = Memo::default(); + let rows = memo.find_or_add_group(scan(realized)).unwrap(); + let window = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: rows, + })) + .unwrap(); + let per_series = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Aggregate { + child: window, + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + })) + .unwrap(); + let topk = AggIntent::TopK { k: 10 }; + let exact = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Aggregate { + child: per_series, + reduction: Reduction::by(vec![2]), + measures: vec![topk], + output_names: vec![String::new()], + filters: vec![None], + having: None, + })) + .unwrap(); + let heap = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: window, + summary_type: FieldDataType::Sketch(SketchAlgorithm::CountSketchWithHeap), + input: SummaryUpdate { + item: Some(super::series_identity_item(schema_of(&memo, window), &[2])), + weight: crate::ir::schema::SummaryInputExpr::Column(ColumnRef::SampleValue), + weight_domain: WeightDomain::UnknownOrSigned, + }, + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let estimate = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: heap, + query: SketchStatistic::TopK { k: 10 }, + })) + .unwrap(); + let estimated = schema_of(&memo, estimate).fields.len() - 1; + let cols = schema_of(&memo, exact) + .fields + .iter() + .enumerate() + .map(|(index, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(if index == 1 { + estimated + } else { + schema_of(&memo, estimate) + .fields + .iter() + .position(|input| input.name == field.name) + .unwrap() + }), + }) + .collect(); + let readout = memo + .find_or_add_group(Operator::NonASAP(NonASAPOp::Project { + cols, + child: estimate, + })) + .unwrap(); + let exact_keys = &schema_of(&memo, exact).unique_keys; + assert!(!exact_keys.is_empty()); + assert_eq!(exact_keys, &schema_of(&memo, readout).unique_keys); + } +} diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index e80abf027..8d4768da8 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -416,8 +416,12 @@ pub struct CandidateForMemoGroup { /// `expression.operator`, in [`Operator::children`] order. Changed only /// through [`Memo::set_input_requirements`]. pub input_requirements: Vec, - /// The groups a fused computation also implements, beyond the group - /// whose candidate it is; empty otherwise. + /// A fused candidate computes, as one operator, a sub-DAG pattern of + /// several operators in one appearance of the sub-expression (a top-k + /// over an aggregate as one heap summary; a sum over a sum as one + /// aggregate): the groups of the operators it replaces. Empty otherwise. + /// Reuse across appearances is Pass 2's, not fusion (concepts doc §3, + /// "Fused groups"). pub fused_groups: Vec, /// Which rule added it. Not in the formulation: kept for /// `stage_pipeline --text` and the viewer (approved 2026-11-05). diff --git a/crates/types/src/stages/mod.rs b/crates/types/src/stages/mod.rs index f200f995d..68c997549 100644 --- a/crates/types/src/stages/mod.rs +++ b/crates/types/src/stages/mod.rs @@ -29,9 +29,7 @@ pub use memo::{ queries_rooted_at, root_groups, ColumnLineage, GroupProperties, MemoGroup, MemoGroupId, ValueSign, }; -pub use per_computation_logical_memo_candidates::{ - ComputationTarget, FusedComputation, HavingEntry, HavingPlacement, HowToComputeAggregateMeasure, -}; +pub use per_computation_logical_memo_candidates::HowToComputeAggregateMeasure; pub use physical::{ output_references, producer_satisfies_physical_requirements, EvaluationGrid, ExecutionContext, ExecutionContextId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, diff --git a/crates/types/src/stages/per_computation_logical_memo_candidates.rs b/crates/types/src/stages/per_computation_logical_memo_candidates.rs index 137360946..1a6d77f2c 100644 --- a/crates/types/src/stages/per_computation_logical_memo_candidates.rs +++ b/crates/types/src/stages/per_computation_logical_memo_candidates.rs @@ -1,135 +1,25 @@ -//! The computation-target types Stage 1, Pass 1 builds its candidates from. +//! How Stage 1, Pass 1 computes an aggregate measure. -use std::rc::Rc; - -use super::MemoGroupId; use crate::ir::schema::{ExactKind, SketchAlgorithm}; -use crate::ir::{OperatorNode, Predicate}; /// How one Pass 1 target is computed: exact or a summary. /// -/// Each target has exactly one exact option: `ExactAccumulator` when the -/// intent has an exact mergeable state and it can be built, otherwise -/// `Exact`. The plain form would duplicate "accumulator, not materialized"; -/// Stage 2/3 decide materialization and sharing. Which computation is -/// computed is the target's ([`ComputationTarget`]) or its fused -/// computation's ([`FusedComputation`]). +/// Each target lists one exact option: `ExactAccumulator` when the intent +/// has an exact mergeable state and it can be built, otherwise `Exact`. +/// The plain exact computation is always offered too, as the aggregate +/// group's own expression. #[derive(Debug, Clone, PartialEq)] pub enum HowToComputeAggregateMeasure { - /// Plain exact execution of the original sub-DAG; the exact option only - /// when the intent has no exact accumulator form or that form is pruned. + /// Plain exact execution: the aggregate group's own expression; the + /// listed exact option only when the intent has no exact accumulator + /// form or that form is pruned. Exact, /// One or more exact mergeable accumulator states, by accumulator kind, /// finalized and combined into the measure: one state for `sum`, /// `count`, `min`, `max`, `rate`, `irate` and `increase`; `[Sum, Count]`, /// divided, for `avg`. ExactAccumulator(Vec), - /// An unsized summary family; sized for the measure's accuracy when Pass 1 - /// attaches it to its memo group. + /// A sketch algorithm: its state group gets one build candidate per + /// configuration of the algorithm's predefined list (`parameter_list`). Summary(SketchAlgorithm), } - -/// Where a target's `HAVING` is applied. Composition (#575) applies it. -/// -/// Only the adjacent form is recorded for push-down: the aggregate's own -/// `having`, or a `Filter` directly above the aggregate (the form the SQL -/// frontend lowers `HAVING` to). A value predicate anywhere above an -/// approximate output gets `guarantee = None` through guarantee propagation -/// (#575); multi-level push-down is tracked in #664. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum HavingPlacement { - /// The predicate reads only group keys and every consumer of the - /// aggregate applies it, so it is pushed below the aggregate as a row - /// predicate (a `Filter` on its input, which drops the group). - BelowAggregate, - /// The predicate reads aggregate values, so it stays a `Filter` above the - /// finalize/estimate output; approximate candidates of this target carry - /// `guarantee = None`, since membership near the threshold may change. - AboveOutput, -} - -/// One `HAVING` over a target's aggregate output. -#[derive(Debug, Clone, PartialEq)] -pub struct HavingEntry { - /// The consumer `Filter` that applies it; `None` for the aggregate's own - /// `having`, which every consumer reads. - pub filter: Option>, - pub predicate: Predicate, - pub placement: HavingPlacement, -} - -/// One measure of an aggregate with at least one realization besides exact -/// execution. The target retains source, grouping, filters, input -/// expressions and evaluation context. -#[derive(Debug, Clone)] -pub struct ComputationTarget { - /// The memo group of the `Aggregate` the measure belongs to. - pub aggregate_group: MemoGroupId, - /// Index into the aggregate's measures. - pub measure_index: usize, - /// The measure's unranked ways to compute it, the exact option first. - pub ways_to_compute: Vec, - /// The measure's own filter; only rows that pass it update the state - /// (`SummaryAgg.filter`). - pub measure_filter: Option, - /// The aggregate's own `having`, then one entry per consumer `Filter` - /// directly above the aggregate. - pub having_filters: Vec, - /// The fused computation of this measure together with the target - /// directly beneath it, offered beside this target's own candidates; - /// `None` when no fusion applies. - pub fused_computation: Option, -} - -/// A target's measure computed together with the targets beneath it, which -/// it absorbs (they are not computed): the whole-expression top-k over a -/// per-item `sum` or `count` (`Summary` of a heap family), or the nested -/// sum, a grouped `sum` over a `sum` (`ExactAccumulator([Sum])`). It has the -/// outer target's aggregate, measure, filter and `HAVING`. -#[derive(Debug, Clone, PartialEq)] -pub struct FusedComputation { - /// Indices, into Pass 1's computation-target list, of the targets it - /// absorbs: the one directly beneath, then, for a whole-expression - /// heap over the raw data samples beneath a grouped sum of per-series sums or - /// counts (`topk(k, sum by (G) (sum_over_time(m[W])))`), the per-series - /// target beneath that, then the targets any of these targets' own fused - /// computations absorb. None of them is computed when this is picked. - pub absorbed_targets: Vec, - /// How the fused computation is computed: exact or a summary. Empty - /// when Pass 1 pruned every one (today only for a family the deployment - /// does not support); the fusion is then not offered. - pub ways_to_compute: Vec, -} - -impl ComputationTarget { - /// How many values this target's digit of a whole-workload choice - /// takes: its own candidates, then its fused computation's. - pub fn choice_count(&self) -> usize { - self.ways_to_compute.len() - + self - .fused_computation - .as_ref() - .map_or(0, |fused| fused.ways_to_compute.len()) - } - - /// How the measure is computed when this target's digit of a - /// whole-workload choice is `index`: the candidate, and the targets it - /// absorbs when the candidate is the fused computation's (past this - /// target's own candidates). `None` past [`Self::choice_count`]. - pub fn computation_for( - &self, - index: usize, - ) -> Option<(&HowToComputeAggregateMeasure, Option<&[usize]>)> { - let own = self.ways_to_compute.len(); - match self.ways_to_compute.get(index) { - Some(candidate) => Some((candidate, None)), - None => { - let fused = self.fused_computation.as_ref()?; - Some(( - fused.ways_to_compute.get(index - own)?, - Some(fused.absorbed_targets.as_slice()), - )) - } - } - } -} diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 5f983defc..b5ed3ca5a 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -522,8 +522,8 @@ candidates is in `crates/stage3-plan-selection/src/` (`select.rs`, `groups/`, `m | Step | Entry point | Path | |---|---|---| | Stage 0 | `memo_of_queries` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | -| Pass 1 | `pass1_memo` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's working type, in `asap_types::stages`), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | -| Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, Pass 1's working types (`ComputationTarget`, `FusedComputation`, `HavingEntry`, `HavingPlacement`), `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | +| Pass 1 | `pass1_memo` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's private working type), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | +| Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | | Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | | Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/select.rs` | From e98e80b4d1d2f71f0dcf1a1b91c23cd26c44f6cd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:11:15 +0000 Subject: [PATCH 29/30] refactor(stage1-pass2): rollup from the finest common group, readout requirements, dead-candidate elimination, chain groups (#1207 PR 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass 2 runs, in order: the general rollup rule with one synthesized finest common group per cluster of equal definitions (replaces scan widening, finer-aggregate reuse and the rollup rules); top-k reuse; reach, caps and readout requirements (one readout candidate per requirement level, written with Memo::set_input_requirements); dead-candidate elimination with every root cause (PrunedLogicalCandidate); chain groups SummaryTumblingWindowChain[w](X) for the surviving state groups, widths from the gcd closure of the reaching queries' grids, the window and the offsets, then elimination again; and the logical edges between survivors. A merge, and a Filter/TimeRange/TimeShift over states, carries the θ of the states it reads (one candidate per θ, reading only producers with that θ). The metric rule cuts edges from readouts whose metric the consumer's input does not accept (accepted metrics pushed down from the targeted queries), not candidates; a candidate cut off from an input is eliminated too. A whole-result δ with n_q unknown prunes the readouts it reaches (ValuesPerEvaluationUnknown). The summary-input and window-rollup rules and the local-window-composition step are deleted. - types: LogicalRequirements::error_propagation(memo, consumer, metric) with the constant contracts (key-only Filter by column lineage, abs / clamp / negation, min / max / avg / count, + - / *), b = 0.1 private; AccuracyMetric::ALL; RootPruneReason::{HeapSmallerThanTopK, ValuesPerEvaluationUnknown}; MetricRule.accepted_metrics; PruneCause.through_pruned_candidates; Memo::producers; add_candidate checks θ against the group's state (CandidateError::LogicalProperties); a candidate's input requirements are part of its identity in add_candidate; summary coverage accessors no longer test-only; origins SummaryInput and WidenedScans deleted. - accuracy: theoretical_guarantee at a failure probability (KLL and KMV evaluated at it, others checked against their own), producer_satisfies_logical_requirements returns the failing root reason, readout_metric; node-tree composition, propagation, budget split, function rules, PropagationStats, test_support and the test-only sizing deleted. - devtools: one pass2 stage. --- crates/accuracy/Cargo.toml | 14 +- crates/accuracy/src/budget.rs | 106 -- crates/accuracy/src/composition.rs | 1339 -------------- crates/accuracy/src/estimators/cardinality.rs | 115 +- crates/accuracy/src/estimators/cms.rs | 45 - .../accuracy/src/estimators/count_sketch.rs | 111 +- crates/accuracy/src/estimators/ddsketch.rs | 5 - crates/accuracy/src/estimators/hll.rs | 57 +- crates/accuracy/src/estimators/kll.rs | 108 +- crates/accuracy/src/estimators/mod.rs | 318 ++-- crates/accuracy/src/estimators/univmon.rs | 246 +-- crates/accuracy/src/evidence.rs | 39 - crates/accuracy/src/function_rules.rs | 41 - crates/accuracy/src/lib.rs | 35 +- crates/accuracy/src/model.rs | 249 ++- crates/accuracy/src/propagation.rs | 1640 ----------------- crates/accuracy/src/sizing.rs | 37 +- crates/accuracy/src/test_support.rs | 82 - crates/devtools/src/bin/asap_planner.rs | 14 +- crates/devtools/src/planning_stages.rs | 19 +- crates/devtools/src/planning_stages_text.rs | 12 +- crates/devtools/tests/asap_planner.rs | 5 +- .../devtools/tests/asap_planner_acceptance.rs | 5 +- crates/integration-tests/Cargo.toml | 1 - crates/planner/Cargo.toml | 1 - .../Cargo.toml | 1 - .../src/per_computation_candidates_tests.rs | 2 +- .../stage1-pass2-reuse-candidates/Cargo.toml | 3 + .../src/dead_candidates.rs | 192 ++ .../stage1-pass2-reuse-candidates/src/lib.rs | 30 +- .../src/logical_memo_candidates.rs | 169 +- .../stage1-pass2-reuse-candidates/src/memo.rs | 6 - .../src/memo/rules.rs | 19 - .../src/memo/rules/finer_aggregate.rs | 12 - .../src/memo/rules/rollup.rs | 18 - .../src/memo/rules/scan_labels.rs | 11 - .../src/memo/rules/summary_inputs.rs | 12 - .../src/memo/rules/topk.rs | 11 - .../src/memo/window_candidates.rs | 21 - .../src/requirements.rs | 322 ++++ .../src/rules.rs | 105 ++ .../src/rules/chains.rs | 251 +++ .../src/rules/rollup.rs | 446 +++++ .../src/rules/topk.rs | 87 + .../src/tests.rs | 1077 +++++++++++ .../src/tests/decided_examples.rs | 515 ++++++ crates/types/src/ir/properties/accuracy.rs | 8 + .../src/ir/properties/summary_coverage.rs | 3 +- crates/types/src/stages/candidates.rs | 664 ++++++- .../src/stages/logical_memo_candidates.rs | 31 +- crates/types/tests/memo_summary_coverage.rs | 7 +- .../design_docs/proposals/planner-layering.md | 12 +- 52 files changed, 4250 insertions(+), 4429 deletions(-) delete mode 100644 crates/accuracy/src/budget.rs delete mode 100644 crates/accuracy/src/composition.rs delete mode 100644 crates/accuracy/src/evidence.rs delete mode 100644 crates/accuracy/src/function_rules.rs delete mode 100644 crates/accuracy/src/propagation.rs delete mode 100644 crates/accuracy/src/test_support.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/rules.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs delete mode 100644 crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/requirements.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/rules.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/rules/chains.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/rules/topk.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/tests.rs create mode 100644 crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs diff --git a/crates/accuracy/Cargo.toml b/crates/accuracy/Cargo.toml index 03965917f..fe5fcdec5 100644 --- a/crates/accuracy/Cargo.toml +++ b/crates/accuracy/Cargo.toml @@ -3,20 +3,14 @@ name = "asap-accuracy" version = "0.1.0" edition = "2021" -# The accuracy model shared by the planning stages: per-family estimators, -# guarantee composition and propagation through a DAG, sizing, budget -# split, the target check and the -# deployment's accuracy model and evidence traits. Depends only on -# `asap-types`, never on a stage crate, so every stage can use it. +# The accuracy rules shared by the planning stages: per-algorithm +# estimators (the theoretical guarantee of θ), the target check, the +# deployment's measured accuracy and α^log of a logical edge. Depends only +# on `asap-types`, never on a stage crate, so every stage can use it. [dependencies] asap_sketchlib = { workspace = true } asap-types = { path = "../types" } serde_json = "1" -[features] -# Exposes `test_support`: estimator, sizing, composition and propagation -# helpers other crates' tests call. -test-support = [] - [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/accuracy/src/budget.rs b/crates/accuracy/src/budget.rs deleted file mode 100644 index 562529301..000000000 --- a/crates/accuracy/src/budget.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! Split an end-to-end accuracy target equally among the approximate inputs -//! of one composition (#669, part b). -//! -//! When `n` approximate values compose into one output, each is sized for an -//! equal share of the output's target, so the composed bound meets the -//! target: `ε_i = ε / n` for an additively composed metric (no slack), -//! `ε_i = 1 − (1 + ε)^{−1/n}` for a relative one, and `δ_i = δ / n` (the -//! union bound). Relative errors compose through products and quotients: -//! each input contributes a factor of at most `1 + ε_i` as a factor and -//! `1/(1 − ε_i)` as a divisor, so the true error of any mix stays within -//! `(1 − ε_i)^{−n} − 1 = ε`. The composed bound Stage 3 checks meets it for -//! one product or quotient of two inputs (a ratio's `(a + b)/(1 − b)`), not -//! for a nested quotient such as `x / (y / z)`, whose bound composes each -//! step's slack. A lone approximate denominator over an exact numerator is -//! sized for the whole target, so its quotient's bound `ε/(1 − ε)` misses -//! it. Only equal splits are made. -use super::*; - -/// The target of one of `approximate_inputs` approximate inputs that -/// compose into a value with `target`, split equally in `metric`'s -/// composition (see the module doc). `None` for an exact target or an ε -/// that is not positive and finite: no approximate input meets it. One -/// input (or none) gets `target` itself. -pub fn per_input_target( - target: &AccuracyTarget, - metric: AccuracyMetric, - approximate_inputs: usize, -) -> Option { - if target.is_exact() || !(target.max_error.is_finite() && target.max_error > 0.0) { - return None; - } - if approximate_inputs <= 1 { - return Some(target.clone()); - } - let fraction = 1.0 / approximate_inputs.max(1) as f64; - let max_error = match metric { - AccuracyMetric::Relative => 1.0 - (1.0 + target.max_error).powf(-fraction), - _ => target.max_error * fraction, - }; - Some(AccuracyTarget { - max_error, - failure_probability: target.failure_probability.map(|delta| delta * fraction), - ..target.clone() - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Additive metrics split ε and δ by `n`; the `n` shares add up to the - /// target. - #[test] - fn an_additive_target_splits_epsilon_and_delta_equally() { - let target = AccuracyTarget::max_error(0.1).failure_probability(0.02); - let share = per_input_target(&target, AccuracyMetric::ABSOLUTE, 2).unwrap(); - assert_eq!( - share, - AccuracyTarget::max_error(0.05).failure_probability(0.01) - ); - assert_eq!( - per_input_target(&target, AccuracyMetric::Rank, 1), - Some(target.clone()) - ); - assert_eq!( - per_input_target(&AccuracyTarget::max_error(0.1), AccuracyMetric::Rank, 2) - .unwrap() - .failure_probability, - None - ); - } - - /// A relative target splits so that a quotient meets it too: - /// `(1 − ε_i)^{−n} = 1 + ε`. For ε = 0.05 over two inputs each gets - /// 0.0241, and both a ratio `(a + b)/(1 − b)` and a product - /// `(1 + a)(1 + b) − 1` of them stay within 0.05; the product-only split - /// `(1 + ε)^{1/2} − 1 = 0.0247` would give the ratio 0.0507. - #[test] - fn a_relative_target_splits_so_a_quotient_meets_it() { - let target = AccuracyTarget::max_error(0.05); - let share = per_input_target(&target, AccuracyMetric::Relative, 2) - .unwrap() - .max_error; - assert!((share - (1.0 - 1.05f64.powf(-0.5))).abs() < 1e-12); - let ratio = (share + share) / (1.0 - share); - let product = (1.0 + share) * (1.0 + share) - 1.0; - assert!(ratio <= 0.05 && product <= 0.05, "{ratio} {product}"); - assert_eq!( - per_input_target(&target, AccuracyMetric::Relative, 1), - Some(target) - ); - } - - /// No approximate input meets an exact target. - #[test] - fn an_exact_target_has_no_share() { - assert_eq!( - per_input_target(&AccuracyTarget::exact(), AccuracyMetric::ABSOLUTE, 2), - None - ); - assert_eq!( - per_input_target(&AccuracyTarget::max_error(0.0), AccuracyMetric::ABSOLUTE, 2), - None - ); - } -} diff --git a/crates/accuracy/src/composition.rs b/crates/accuracy/src/composition.rs deleted file mode 100644 index 6c3d249d2..000000000 --- a/crates/accuracy/src/composition.rs +++ /dev/null @@ -1,1339 +0,0 @@ -//! Registered propagation rules for computations over uncertain inputs. -//! Stage 1 composes each candidate's guarantees with them -//! (`per_computation_candidates`); a rule that needs a statistic leaves an -//! `Unknown` leaf named by one of the constants below, which Stage 3 binds. -use super::*; - -/// Rows an exact aggregation folds (per group): bounded by the series -/// count of the sources the query reads. -pub const INPUT_ROW_COUNT: &str = "input_row_count"; -/// Rows one group of a SQL fold over an approximate relation folds: not -/// bounded by the series count of the sources it reads, so plan selection -/// leaves it unbound. -pub(crate) const GROUP_ROW_COUNT: &str = "group_row_count"; -/// Whether every input value is non-negative. -pub(crate) const VALUES_NON_NEGATIVE: &str = "values_non_negative"; -/// The absolute bound of a product of two estimates, -/// `|x|·e_y + |y|·e_x + e_x·e_y`, given enforced operand domains. -pub const PRODUCT_OPERAND_DOMAINS: &str = "product_operand_domains"; -/// The absolute bound of a quotient with an approximate denominator, -/// `(|x|·e_y + |y|·e_x) / (|y|·(|y| − e_y))`, given enforced operand domains -/// with the denominator's bounded away from zero. -pub(crate) const QUOTIENT_OPERAND_DOMAINS: &str = "quotient_operand_domains"; -/// `‖f‖₁` of the stream a Count-Min bound is normalized by. -pub const STREAM_L1_NORM: &str = "stream_l1_norm"; -/// `‖f‖₂` of the stream a Count Sketch bound is normalized by. -pub(crate) const STREAM_L2_NORM: &str = "stream_l2_norm"; -/// Rows of one source's stream a summary folds: with a constant update -/// weight `c`, `‖f‖₁ = |c| · rows` and `‖f‖₂ ≤ ‖f‖₁`. -pub(crate) const STREAM_ROW_COUNT: &str = "stream_row_count"; - -/// Which stream norm bounds a Count-Min or Count Sketch estimate, given its -/// summary's update weight. -#[derive(Debug, Clone, PartialEq)] -pub enum StreamNorm { - /// The weight is the source's raw value column: the source's norm - /// (`source_norm_statistic`). - Source(String), - /// A constant weight (a count): `|weight|` times the source's row count. - ConstantWeight { weight: f64, source: String }, -} - -/// The statistic for `norm` ([`STREAM_L1_NORM`] or [`STREAM_L2_NORM`]) of -/// one source's stream, e.g. `stream_l1_norm[m]`. -pub(crate) fn source_norm_statistic(norm: &str, source: &str) -> String { - format!("{norm}[{source}]") -} - -/// Whether every value folded is non-negative, for values that are raw -/// samples of `metric` or plain sums of them, e.g. -/// `values_non_negative[m]`: an indicator, 1 when proven. Plan selection -/// binds it to 1 when the data workload declares `metric` a counter or -/// non-negative, and leaves it unbound otherwise. -pub(crate) fn raw_data_samples_sign_statistic(metric: &str) -> String { - format!("{VALUES_NON_NEGATIVE}[{metric}]") -} - -/// The metric a `raw_data_samples_sign_statistic` names. -pub fn parse_raw_data_samples_sign_statistic(statistic: &str) -> Option<&str> { - statistic - .strip_prefix(VALUES_NON_NEGATIVE)? - .strip_prefix('[')? - .strip_suffix(']') -} - -/// The norm and source a `source_norm_statistic` names. -pub fn parse_source_norm_statistic(statistic: &str) -> Option<(&'static str, &str)> { - [STREAM_L1_NORM, STREAM_L2_NORM] - .into_iter() - .find_map(|norm| { - let source = statistic - .strip_prefix(norm)? - .strip_prefix('[')? - .strip_suffix(']')?; - Some((norm, source)) - }) -} - -fn additive( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, -) -> AccuracyGuarantee { - let mut terms: Vec = inputs.iter().map(|g| g.spec.error.clone()).collect(); - terms.push(local.spec.error.clone()); - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - local.spec.accuracy_metric, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, rule), - ) -} - -/// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors -/// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all -/// cross-products so the expression DAG is exact for any input count. -fn multiplicative( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, -) -> AccuracyGuarantee { - let factors: Vec<&BoundExpr> = inputs - .iter() - .map(|g| &g.spec.error) - .chain(std::iter::once(&local.spec.error)) - .collect(); - // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. - let mut terms = Vec::new(); - for mask in 1..(1u32 << factors.len()) { - let subset: Vec = factors - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, b)| (*b).clone()) - .collect(); - terms.push(if subset.len() == 1 { - subset.into_iter().next().expect("one element") - } else { - BoundExpr::Product { factors: subset } - }); - } - let mut deltas: Vec = inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(); - deltas.push(local.spec.failure_probability.clone()); - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Sum { terms }, - ProbabilityExpr::UnionBound { terms: deltas }, - composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), - ) -} - -fn lipschitz( - op: &CompositionOperator, - constant: f64, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, -) -> AccuracyGuarantee { - let input = &inputs[0]; - let scaled = BoundExpr::Scaled { - factor: constant, - inner: Box::new(input.spec.error.clone()), - }; - let (bound, delta) = match local { - Some(local) => ( - BoundExpr::Sum { - terms: vec![scaled, local.spec.error.clone()], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - input.spec.failure_probability.clone(), - local.spec.failure_probability.clone(), - ], - }, - ), - None => (scaled, input.spec.failure_probability.clone()), - }; - let exact_local = AccuracyGuarantee::exact("deterministic Lipschitz transformation"); - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - bound, - delta, - composed_provenance( - op, - inputs, - local.unwrap_or(&exact_local), - "lipschitz_union_bound", - ), - ) -} - -/// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The -/// planner composes one *per-value* child guarantee over an unknown -/// number of input rows, so both the bound and the union bound scale by -/// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is -/// not supplied. Each input's normalized bound is first converted to -/// absolute units via the statistic its metric is normalized by (also -/// unknown unless supplied); a `Rank` input has no such conversion. -fn exact_sum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, true, "exact_sum_relative"); - } - let mut terms = Vec::with_capacity(inputs.len()); - let mut deltas = Vec::with_capacity(inputs.len()); - let mut provenance = Vec::new(); - for (i, input) in inputs.iter().enumerate() { - let absolute = - absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: format!( - "input {i} carries a {:?} guarantee, which has no registered \ - conversion to an absolute value error", - input.spec.accuracy_metric - ), - })?; - if let BoundExpr::Product { factors } = &absolute { - for f in factors { - if let BoundExpr::Unknown { statistic } = f { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - } - } - } - terms.push(absolute); - deltas.push(input.spec.failure_probability.clone()); - } - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Sum)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_sum_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Product { - factors: vec![count.clone(), BoundExpr::Sum { terms }], - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), - }, - provenance, - )) -} - -/// Exact arithmetic mean over values with absolute-error guarantees. -/// Averaging cannot amplify the largest absolute input error. The event -/// that every row respects its bound is still protected conservatively -/// by a union bound over the input row count. -fn exact_average( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, true, "exact_average_relative"); - } - if inputs - .iter() - .any(|input| input.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact average requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Average)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_average_union_bound", - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// Exact `max`/`min` over approximate inputs of one shared metric: the -/// returned value's error is at most the largest input bound (order -/// statistics are monotone under a uniform perturbation), with -/// probability by the union bound over every input row. This bounds the -/// returned *value*; it does not identify the true winning key. -fn exact_extremum( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, -) -> Result { - if all_relative(inputs) { - return relative_fold(op, inputs, stats, false, "exact_extremum_relative"); - } - let metric = inputs[0].spec.accuracy_metric; - if inputs - .iter() - .any(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "exact max/min requires absolute-error input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = AccuracyGuarantee::exact("ExactAggregate(Max)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_extremum_union_bound", - )); - Ok(AccuracyGuarantee::new( - metric, - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// An exact sum, mean, max or min of values each within relative `α_i` -/// is within relative `max α_i`. A max or min holds whatever the signs; -/// a sum or mean only over non-negative values (no cancellation), so -/// with `needs_sign` and the sign unknown the bound is the symbolic -/// [`VALUES_NON_NEGATIVE`]. The failure probability is the union bound -/// over every input row. -fn relative_fold( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - stats: &PropagationStats, - needs_sign: bool, - rule: &str, -) -> Result { - if needs_sign && stats.values_non_negative == Some(false) { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason: "relative error of a sum over signed values is unbounded".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let bound = if needs_sign && stats.values_non_negative.is_none() { - match &stats.sign_statistic { - // The sign of one metric's samples: the bound, gated on it. - Some(statistic) => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - BoundExpr::Product { - factors: vec![ - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - }, - BoundExpr::Unknown { - statistic: statistic.clone(), - }, - ], - } - } - None => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - } - } - } - } else { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - }; - provenance.extend(composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact aggregation"), - rule, - )); - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - bound, - ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }), - }, - provenance, - )) -} - -/// Exact division of two relative-value estimates (an exact operand has -/// bound 0). If the numerator is within relative `a` and the denominator -/// within `b < 1`, the ratio is within -/// `max((1 + a)/(1 − b) − 1, 1 − (1 − a)/(1 + b)) = (a + b)/(1 − b)`, and -/// fails with the union bound of their failure probabilities. No operand -/// domain is needed: a relative bound below 1 keeps a zero denominator at -/// zero and a numerator's sign, so the true and estimated ratios are then -/// the same non-finite value. Both bounds must be constants: a symbolic one -/// has no rule. -/// Float64 overflow and underflow of the quotient are not bounded, as for a -/// product. -fn exact_division( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], -) -> Result { - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.spec.accuracy_metric).collect(), - local_metric: None, - reason, - }; - if inputs.len() != 2 - || inputs.iter().any(|input| { - input.spec.accuracy_metric != AccuracyMetric::Relative && !input.is_exact() - }) - { - return Err(unsupported( - "division needs two relative-value or exact guarantees".into(), - )); - } - let Some(numerator) = inputs[0].spec.error.evaluate() else { - return Err(unsupported( - "numerator relative bound is unavailable".into(), - )); - }; - let Some(denominator) = inputs[1].spec.error.evaluate() else { - return Err(unsupported( - "denominator relative bound is unavailable".into(), - )); - }; - if !(numerator.is_finite() && numerator >= 0.0 && (0.0..1.0).contains(&denominator)) { - return Err(unsupported( - "division needs finite non-negative bounds and a denominator bound below one".into(), - )); - } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (numerator + denominator) / (1.0 - denominator), - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|input| input.spec.failure_probability.clone()) - .collect(), - }, - composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("exact division"), - "relative_division_union_bound", - ), - )) -} - -/// Whether `guarantee` is a sketch's distinct-count estimate. -fn estimates_distinct_count(guarantee: &AccuracyGuarantee) -> bool { - let cardinality = format!("{:?}", asap_types::ir::schema::SketchStatistic::Cardinality); - guarantee.provenance.iter().any(|source| { - matches!(source, GuaranteeSource::SketchEvaluation { query, .. } if *query == cardinality) - }) -} - -/// Every input carries a relative-value guarantee. -fn all_relative(inputs: &[AccuracyGuarantee]) -> bool { - !inputs.is_empty() - && inputs - .iter() - .all(|g| g.spec.accuracy_metric == AccuracyMetric::Relative) -} - -/// `stats.input_row_count` as a bound factor, or an `Unknown` leaf named by -/// `stats.row_count_statistic` (recorded in `provenance`) when absent. -fn row_count(stats: &PropagationStats, provenance: &mut Vec) -> BoundExpr { - match stats.input_row_count { - Some(n) => BoundExpr::Constant { value: n as f64 }, - None => { - let statistic = stats.row_count_statistic.unwrap_or(INPUT_ROW_COUNT); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }); - BoundExpr::Unknown { - statistic: statistic.into(), - } - } - } -} - -/// `input` with a Count-Min ([`AccuracyMetric::ABSOLUTE_STREAM_L1`]) or Count Sketch -/// ([`AccuracyMetric::ABSOLUTE_STREAM_L2`]) bound restated as an additive absolute -/// error: the normalized bound times its stream norm, a symbolic statistic -/// until bound. The norm is the one `norm` names ([`StreamNorm`]), which -/// plan selection can bind; without one (a weight computed from the -/// values) it is a statistic nothing binds. Any other guarantee is -/// returned unchanged. -pub(crate) fn additive_as_absolute( - input: AccuracyGuarantee, - norm: Option<&StreamNorm>, -) -> AccuracyGuarantee { - if !matches!( - input.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE_STREAM_L1 | AccuracyMetric::ABSOLUTE_STREAM_L2 - ) || input.is_exact() - { - return input; - } - let mut bound = absolute_bound(&input).expect("additive metrics convert to absolute errors"); - if let (Some(norm), BoundExpr::Product { factors }) = (norm, &mut bound) { - for factor in factors { - if let BoundExpr::Unknown { statistic } = factor { - *factor = match norm { - StreamNorm::Source(source) => BoundExpr::Unknown { - statistic: source_norm_statistic(statistic, source), - }, - StreamNorm::ConstantWeight { weight, source } => BoundExpr::Product { - factors: vec![ - BoundExpr::Constant { - value: weight.abs(), - }, - BoundExpr::Unknown { - statistic: source_norm_statistic(STREAM_ROW_COUNT, source), - }, - ], - }, - }; - } - } - } - let mut provenance = input.provenance; - unknown_statistics(&bound, &mut |statistic| { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }) - }); - let mut absolute = AccuracyGuarantee { - provenance, - ..input - }; - absolute.spec.accuracy_metric = AccuracyMetric::ABSOLUTE; - absolute.spec.error = bound; - absolute -} - -/// Every `Unknown` statistic in `bound`. -fn unknown_statistics(bound: &BoundExpr, found: &mut dyn FnMut(&str)) { - match bound { - BoundExpr::Unknown { statistic } => found(statistic), - BoundExpr::Sum { terms } - | BoundExpr::Product { factors: terms } - | BoundExpr::Max { terms } => { - for term in terms { - unknown_statistics(term, found); - } - } - BoundExpr::Scaled { inner, .. } => unknown_statistics(inner, found), - _ => {} - } -} - -/// `input`'s bound converted to absolute value units, multiplying a -/// normalized metric by the (unknown) statistic it is normalized by. `None` -/// for a metric with no such conversion (`Rank`). -fn absolute_bound(input: &AccuracyGuarantee) -> Option { - let normalizer = match input.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => return Some(input.spec.error.clone()), - AccuracyMetric::Relative => "true_value_magnitude", - AccuracyMetric::ABSOLUTE_STREAM_L1 => STREAM_L1_NORM, - AccuracyMetric::ABSOLUTE_STREAM_L2 => STREAM_L2_NORM, - // `Rank` has no distribution-free conversion to a value error; a - // metric this crate does not know has no registered conversion. - AccuracyMetric::Rank | _ => return None, - }; - if input.spec.error.is_zero() { - return Some(BoundExpr::Zero); - } - Some(BoundExpr::Product { - factors: vec![ - input.spec.error.clone(), - BoundExpr::Unknown { - statistic: normalizer.into(), - }, - ], - }) -} - -fn composed_provenance( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: &AccuracyGuarantee, - rule: &str, -) -> Vec { - let mut provenance: Vec = inputs - .iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - }) - .collect(); - provenance.extend(local.provenance.iter().cloned()); - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: rule.into(), - }); - provenance -} - -/// The built-in composition rule: `inputs`' guarantees (in the parent's -/// child order) composed with the parent's own `local` guarantee under `op`, -/// given the statistics in `stats` (all unknown in Stage 1; plan selection -/// binds some). `Err` is the fail-closed answer: no registered rule, or a -/// missing input guarantee. It means "no rule", not "infeasible": Pass 1 -/// keeps the candidate with `guarantee = None`. -pub(crate) fn propagate( - op: &CompositionOperator, - inputs: &[AccuracyGuarantee], - local: Option<&AccuracyGuarantee>, - stats: &PropagationStats, -) -> Result { - // Exact input: only the local guarantee remains (or the value is exact). - if inputs.iter().all(AccuracyGuarantee::is_exact) - && !matches!(op, CompositionOperator::TopKSelection) - { - return Ok(match local { - Some(local) => { - let mut out = local.clone(); - out.provenance - .extend(inputs.iter().enumerate().map(|(input_index, g)| { - GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - } - })); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - None => { - let mut out = AccuracyGuarantee::exact(format!("{op:?} over exact inputs")); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - }); - } - - let input_metrics: Vec = - inputs.iter().map(|g| g.spec.accuracy_metric).collect(); - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: input_metrics.clone(), - local_metric: local.map(|g| g.spec.accuracy_metric), - reason, - }; - // An exact input is compatible with every metric; only approximate - // inputs constrain the rule. - let approximate: Vec<&AccuracyGuarantee> = inputs.iter().filter(|g| !g.is_exact()).collect(); - let same_metric = - |metric: AccuracyMetric| approximate.iter().all(|g| g.spec.accuracy_metric == metric); - - match op { - CompositionOperator::CheckedRelativeDivision => { - if inputs.len() != 2 || local.is_some() || !same_metric(AccuracyMetric::Relative) { - return Err(unsupported( - "checked division requires two exact/relative-value operands".into(), - )); - } - let a = inputs[0] - .spec - .error - .evaluate() - .ok_or_else(|| unsupported("unknown numerator bound".into()))?; - let b = inputs[1] - .spec - .error - .evaluate() - .ok_or_else(|| unsupported("unknown denominator bound".into()))?; - if !(0.0..1.0).contains(&b) || a < 0.0 || !a.is_finite() { - return Err(unsupported("invalid relative division bounds".into())); - } - Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { - value: (a + b) / (1.0 - b) + 4.0 * f64::EPSILON, - }, - ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.spec.failure_probability.clone()) - .collect(), - }, - composed_provenance( - op, - inputs, - &AccuracyGuarantee::exact("checked floating-point division"), - "checked_relative_division_union_bound", - ), - )) - } - CompositionOperator::ApproximateAggregate => { - let local = local.ok_or_else(|| { - unsupported("approximate operator has no local guarantee to compose".into()) - })?; - if !same_metric(local.spec.accuracy_metric) { - return Err(unsupported(format!( - "no registered cross-metric rule from {input_metrics:?} to {:?}", - local.spec.accuracy_metric - ))); - } - // A distinct count over approximate values is not within the - // inputs' relative error: no rule, as for any summary whose - // estimate is not a value of its inputs. - if estimates_distinct_count(local) { - return Err(unsupported( - "no registered rule for a distinct count over approximate values".into(), - )); - } - match local.spec.accuracy_metric { - AccuracyMetric::ABSOLUTE => Ok(additive(op, inputs, local, "additive_union_bound")), - AccuracyMetric::Relative => { - if stats.values_non_negative == Some(false) { - return Err(unsupported( - "relative-error composition cannot use a known signed input".into(), - )); - } - if stats.values_non_negative.is_none() { - let mut provenance = composed_provenance( - op, - inputs, - local, - "relative_value_sign_unverified", - ); - provenance.extend(stats.evidence_provenance.clone()); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: VALUES_NON_NEGATIVE.into(), - }); - return Ok(AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - }, - ProbabilityExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into(), - }, - provenance, - )); - } - Ok(multiplicative(op, inputs, local)) - } - AccuracyMetric::Rank - | AccuracyMetric::ABSOLUTE_STREAM_L1 - | AccuracyMetric::ABSOLUTE_STREAM_L2 - | _ => Err(unsupported(format!( - "no registered same-metric composition rule for {:?} over {:?}", - local.spec.accuracy_metric, local.spec.accuracy_metric - ))), - } - } - CompositionOperator::Lipschitz { constant } => { - if !(constant.is_finite() && *constant >= 0.0) { - return Err(unsupported(format!( - "Lipschitz constant {constant} is not a finite non-negative number" - ))); - } - if inputs.len() != 1 || !same_metric(AccuracyMetric::ABSOLUTE) { - return Err(unsupported( - "Lipschitz rule is registered for exactly one absolute-error input".into(), - )); - } - if local.is_some_and(|g| g.spec.accuracy_metric != AccuracyMetric::ABSOLUTE) { - return Err(unsupported( - "Lipschitz rule needs an absolute-error local guarantee".into(), - )); - } - Ok(lipschitz(op, *constant, inputs, local)) - } - CompositionOperator::ExactSum => exact_sum(op, inputs, stats), - CompositionOperator::ExactAverage => exact_average(op, inputs, stats), - CompositionOperator::ExactExtremum => exact_extremum(op, inputs, stats), - CompositionOperator::ExactDivision => exact_division(op, inputs), - CompositionOperator::CounterRate - | CompositionOperator::InstantCounterRate - | CompositionOperator::CounterIncrease => Err(unsupported( - "counter reset detection and boundary extrapolation have no distribution-free \ - accuracy bound over approximate samples; exact samples remain exact" - .into(), - )), - CompositionOperator::TopKSelection => { - let selected = stats.topk_selected_lower_bound; - let excluded = stats.topk_excluded_upper_bound; - let delta = stats.topk_interval_failure_probability; - if selected.is_some_and(|value| !value.is_finite()) - || excluded.is_some_and(|value| !value.is_finite()) - || delta.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) - || selected - .zip(excluded) - .is_some_and(|(lower, upper)| lower <= upper) - { - return Err(unsupported( - "top-k confidence intervals overlap or contain invalid evidence".into(), - )); - } - let mut provenance = inputs - .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .collect::>(); - provenance.extend(stats.evidence_provenance.clone()); - if let Some(local) = local { - provenance.extend(local.provenance.clone()); - } - for (name, missing) in [ - ("topk_selected_lower_bound", selected.is_none()), - ("topk_excluded_upper_bound", excluded.is_none()), - ("topk_interval_failure_probability", delta.is_none()), - ] { - if missing { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: name.into(), - }); - } - } - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "topk_membership_margin_certificate".into(), - }); - let certified = selected.is_some() && excluded.is_some() && delta.is_some(); - // A certified margin makes the returned key set the true one, so - // each returned value keeps its score's error; without one, - // membership and hence the values are unknown. - let metric = approximate - .first() - .map_or(AccuracyMetric::ABSOLUTE, |score| score.spec.accuracy_metric); - if !same_metric(metric) { - return Err(unsupported( - "top-k selection needs scores of one metric".into(), - )); - } - let error = if certified { - BoundExpr::Max { - terms: inputs.iter().map(|g| g.spec.error.clone()).collect(), - } - } else { - BoundExpr::Unknown { - statistic: "topk_membership_margin".into(), - } - }; - Ok(AccuracyGuarantee::new( - metric, - error, - delta.map_or_else( - || ProbabilityExpr::Unknown { - statistic: "topk_interval_failure_probability".into(), - }, - |value| ProbabilityExpr::Constant { value }, - ), - provenance, - )) - } - // An operator this crate does not know has no registered rule. - _ => Err(unsupported("no registered rule for this operator".into())), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A source-norm statistic names its norm and source, whatever - /// characters the source holds; any other statistic is not one. - #[test] - fn source_norm_statistics_round_trip() { - for source in ["m", "a[b]", "]x[", "t.c", ""] { - for norm in [STREAM_L1_NORM, STREAM_L2_NORM] { - let statistic = source_norm_statistic(norm, source); - assert_eq!( - parse_source_norm_statistic(&statistic), - Some((norm, source)) - ); - } - } - for other in [ - INPUT_ROW_COUNT, - STREAM_L1_NORM, - "stream_l3_norm[m]", - "stream_l1_norm[m", - ] { - assert_eq!(parse_source_norm_statistic(other), None, "{other}"); - } - } - fn abs(bound: f64, delta: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::ABSOLUTE, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Constant { value: delta }, - vec![], - ) - } - fn rel(bound: f64) -> AccuracyGuarantee { - AccuracyGuarantee::new( - AccuracyMetric::Relative, - BoundExpr::Constant { value: bound }, - ProbabilityExpr::Zero, - vec![], - ) - } - fn with_metric(metric: AccuracyMetric, bound: f64) -> AccuracyGuarantee { - let mut g = abs(bound, 0.0); - g.spec.accuracy_metric = metric; - g - } - #[test] - fn checked_division_propagates_value_bounds_and_rejects_rank_bounds() { - let op = CompositionOperator::CheckedRelativeDivision; - let inputs = [rel(0.01), rel(0.01)]; - let g = propagate(&op, &inputs, None, &Default::default()).unwrap(); - assert!((g.spec.error.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-14); - let mut rank = inputs[0].clone(); - rank.spec.accuracy_metric = AccuracyMetric::Rank; - assert!(propagate(&op, &[rank.clone(), rank], None, &Default::default()).is_err()); - assert!(propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()).is_err()); - } - /// A ratio of relative bounds `a`, `b` is within `(a + b)/(1 − b)`, - /// the larger of its over- and under-estimate bounds, with the union - /// bound on failure; no operand domain is needed. A denominator bound of - /// 1 or more, or a bound that is not finite and non-negative, has no - /// rule. - #[test] - fn relative_division_bounds_the_ratio_without_operand_domains() { - let mut numerator = rel(0.02); - numerator.spec.failure_probability = ProbabilityExpr::Constant { value: 0.001 }; - let got = propagate( - &CompositionOperator::ExactDivision, - &[numerator, rel(0.03)], - None, - &PropagationStats::default(), - ) - .unwrap(); - let (a, b): (f64, f64) = (0.02, 0.03); - let over = (1.0 + a) / (1.0 - b) - 1.0; - let under = 1.0 - (1.0 - a) / (1.0 + b); - assert!((got.spec.error.evaluate().unwrap() - over.max(under)).abs() < 1e-14); - assert!((got.spec.failure_probability.evaluate().unwrap() - 0.001).abs() < 1e-15); - for bound in [1., f64::NAN, f64::INFINITY, -0.1] { - assert!(propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(bound)], - None, - &PropagationStats::default() - ) - .is_err()); - } - let rank = with_metric(AccuracyMetric::Rank, 0.01); - assert!(propagate( - &CompositionOperator::ExactDivision, - &[rank.clone(), rank], - None, - &PropagationStats::default() - ) - .is_err()); - } - #[test] - fn exact_child_contributes_zero_error() { - let local = abs(0.05, 0.01); - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[AccuracyGuarantee::exact("sum")], - Some(&local), - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.spec.error.evaluate(), Some(0.05)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.01)); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - } - #[test] - fn additive_bounds_and_delta_union_bound_compose() { - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[abs(0.02, 0.01)], - Some(&abs(0.03, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.05).abs() < 1e-12); - // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::CompositionStep { rule, .. } if rule == "additive_union_bound" - ))); - } - #[test] - fn relative_error_includes_the_cross_term() { - let stats = PropagationStats { - values_non_negative: Some(true), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &stats, - ) - .unwrap(); - // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. - assert!((out.spec.error.evaluate().unwrap() - 0.32).abs() < 1e-12); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); - } - #[test] - fn relative_error_without_sign_knowledge_remains_symbolic() { - let unknown = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - let signed = propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats { - values_non_negative: Some(false), - ..Default::default() - }, - ); - assert!(signed.is_err()); - } - #[test] - fn incompatible_metrics_are_rejected_not_treated_as_exact() { - // HLL cardinality error under a CMS frequency guarantee. - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Relative, 0.01)], - Some(&with_metric(AccuracyMetric::ABSOLUTE_STREAM_L1, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!( - err, - AccuracyError::UnsupportedComposition { - input_metrics, - local_metric: Some(AccuracyMetric::ABSOLUTE_STREAM_L1), - .. - } if input_metrics == vec![AccuracyMetric::Relative] - )); - // Quantile rank error under value-additive logic. - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&abs(0.01, 0.0)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn same_metric_rank_over_rank_has_no_registered_rule() { - let err = propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(AccuracyMetric::Rank, 0.01)], - Some(&with_metric(AccuracyMetric::Rank, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn lipschitz_scales_the_input_bound() { - let out = propagate( - &CompositionOperator::Lipschitz { constant: 3.0 }, - &[abs(0.1, 0.01)], - Some(&abs(0.05, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.35).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - } - #[test] - fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { - let out = propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!( - out.spec.error.evaluate(), - None, - "unknown row count stays unknown" - ); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" - ))); - assert!(!meets(&out, 1.0, None)); - - let known = PropagationStats { - input_row_count: Some(4), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.4).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - #[test] - fn exact_extremum_takes_the_max_bound() { - let known = PropagationStats { - input_row_count: Some(2), - ..Default::default() - }; - let out = propagate( - &CompositionOperator::ExactExtremum, - &[abs(0.1, 0.01), abs(0.3, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.spec.error.evaluate().unwrap() - 0.3).abs() < 1e-12); - assert!((out.spec.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - /// `max`/`min` of values each within relative `α_i` is within relative - /// `max α_i` of the true extremum, whatever the signs. - #[test] - fn exact_extremum_keeps_a_relative_value_bound() { - let out = propagate( - &CompositionOperator::ExactExtremum, - &[rel(0.01), rel(0.02)], - None, - &PropagationStats { - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(out.spec.error.evaluate(), Some(0.02)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.0)); - } - /// A sum or mean of relative-error values keeps the relative bound only - /// over values known non-negative (cancellation otherwise); with the sign - /// unknown, the bound stays symbolic, and a known sign change rejects. - #[test] - fn exact_sum_and_average_keep_a_relative_bound_only_without_a_sign_change() { - for op in [ - CompositionOperator::ExactSum, - CompositionOperator::ExactAverage, - ] { - let with = |values_non_negative| { - propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - values_non_negative, - input_row_count: Some(3), - ..Default::default() - }, - ) - }; - let known = with(Some(true)).unwrap(); - assert_eq!(known.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(known.spec.error.evaluate(), Some(0.01), "{op:?}"); - let unknown = with(None).unwrap(); - assert_eq!(unknown.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!( - unknown.spec.error, - BoundExpr::Unknown { - statistic: VALUES_NON_NEGATIVE.into() - } - ); - assert!(with(Some(false)).is_err(), "{op:?}"); - } - } - /// Over the samples of one metric, an unknown sign is named by that - /// metric ([`raw_data_samples_sign_statistic`]) and gates the bound: binding it - /// to 1 (proven) gives the relative bound, and left unbound it stays - /// unknown. - #[test] - fn a_sign_over_one_metrics_samples_gates_the_relative_bound() { - let statistic = raw_data_samples_sign_statistic("m"); - assert_eq!(parse_raw_data_samples_sign_statistic(&statistic), Some("m")); - assert_eq!( - parse_raw_data_samples_sign_statistic(VALUES_NON_NEGATIVE), - None - ); - for op in [ - CompositionOperator::ExactSum, - CompositionOperator::ExactAverage, - ] { - let out = propagate( - &op, - &[rel(0.01)], - None, - &PropagationStats { - sign_statistic: Some(statistic.clone()), - input_row_count: Some(3), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.error.evaluate(), None, "{op:?}"); - let proven = out.bind(&|s| (s == statistic).then_some(1.0)); - assert_eq!(proven.spec.error.evaluate(), Some(0.01), "{op:?}"); - let other = out.bind(&|s| (s == raw_data_samples_sign_statistic("n")).then_some(1.0)); - assert_eq!(other.spec.error.evaluate(), None, "{op:?}"); - } - } - - #[test] - fn exact_average_has_its_own_absolute_error_rule() { - let out = propagate( - &CompositionOperator::ExactAverage, - &[abs(0.25, 0.01)], - None, - &PropagationStats { - input_row_count: Some(4), - ..PropagationStats::default() - }, - ) - .unwrap(); - assert_eq!(out.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(out.spec.error.evaluate(), Some(0.25)); - assert_eq!(out.spec.failure_probability.evaluate(), Some(0.04)); - } - #[test] - fn topk_selection_requires_a_separated_margin_certificate() { - let unknown = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - - let certified = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(certified.spec.accuracy_metric, AccuracyMetric::ABSOLUTE); - assert_eq!(certified.spec.error.evaluate(), Some(0.1)); - assert_eq!(certified.spec.failure_probability.evaluate(), Some(0.005)); - - let overlapping = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(100.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ); - assert!(overlapping.is_err()); - - let partial = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert!(partial.has_unknown()); - assert_eq!(partial.spec.failure_probability.evaluate(), Some(0.005)); - - let invalid_partial = propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(f64::NAN), - ..Default::default() - }, - ); - assert!(invalid_partial.is_err()); - } -} diff --git a/crates/accuracy/src/estimators/cardinality.rs b/crates/accuracy/src/estimators/cardinality.rs index 5e03bd0b6..56073d7ae 100644 --- a/crates/accuracy/src/estimators/cardinality.rs +++ b/crates/accuracy/src/estimators/cardinality.rs @@ -32,33 +32,76 @@ use super::*; /// The smallest `k` `asap_sketchlib`'s KMV estimate `(k-1)/U` is defined for. const KMV_MIN_K: u32 = 2; -/// The largest `k` sizing returns. -#[cfg(test)] -const KMV_MAX_K: u32 = 1 << 26; - pub(super) fn guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, ) -> Option { - match params { - SketchParams::Kmv { k } => { - // Its relative standard error. - let eps = 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(); - Some(super::bounded_guarantee( - algorithm, - params, - query, - AccuracyMetric::Relative, - eps, - ProbabilityExpr::Constant { - value: kmv_failure_probability(*k, eps), - }, - "kmv_exact_below_k_chernoff_v1", - )) + let SketchParams::Kmv { k } = params else { + return None; + }; + // Its relative standard error. + let eps = 1.0 / f64::from(k.saturating_sub(2).max(1)).sqrt(); + Some(kmv_guarantee( + algorithm, + params, + query, + eps, + kmv_failure_probability(*k, eps), + )) +} + +/// The smallest relative error `k` misses with probability at most +/// `failure_probability` ([`kmv_failure_probability`], by bisection); +/// `None` when no error up to 1 does. +pub(super) fn guarantee_at( + algorithm: &SketchAlgorithm, + params: &SketchParams, + query: &SketchStatistic, + failure_probability: f64, +) -> Option { + let SketchParams::Kmv { k } = params else { + return None; + }; + if kmv_failure_probability(*k, 1.0) > failure_probability { + return None; + } + let (mut met, mut missed) = (1.0, 0.0); + for _ in 0..60 { + let eps = (met + missed) / 2.0; + if kmv_failure_probability(*k, eps) <= failure_probability { + met = eps; + } else { + missed = eps; } - _ => None, } + Some(kmv_guarantee( + algorithm, + params, + query, + met, + failure_probability, + )) +} + +fn kmv_guarantee( + algorithm: &SketchAlgorithm, + params: &SketchParams, + query: &SketchStatistic, + eps: f64, + failure_probability: f64, +) -> AccuracyGuarantee { + super::bounded_guarantee( + algorithm, + params, + query, + AccuracyMetric::Relative, + eps, + ProbabilityExpr::Constant { + value: failure_probability, + }, + "kmv_exact_below_k_chernoff_v1", + ) } /// The probability that `asap_sketchlib`'s KMV with `k` misses relative @@ -79,38 +122,6 @@ pub(crate) fn kmv_failure_probability(k: u32, eps: f64) -> f64 { ((over + under) * (1.0 + 1e-12)).min(1.0) } -/// The smallest `k` whose KMV failure probability at relative error `eps` -/// is at most `delta` ([`kmv_failure_probability`]), up to `2^26`; a -/// degenerate `eps` or `delta` saturates to `2^26`. -#[cfg(test)] -pub(crate) fn kmv_k(eps: f64, delta: f64) -> u32 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return KMV_MAX_K; - } - // The overestimate term alone at most `δ` bounds `k` below; both terms - // at most `δ/2` bound it above. - let k_at = |ln_inverse: f64| { - saturating_ceil( - 1.0 + (1.0 + eps) * (2.0 + eps) * ln_inverse / (eps * eps), - KMV_MIN_K, - KMV_MAX_K, - ) - }; - let (mut lo, mut hi) = (k_at((1.0 / delta).ln()), k_at((2.0 / delta).ln())); - if kmv_failure_probability(hi, eps) > delta { - return KMV_MAX_K; - } - while lo < hi { - let mid = lo + (hi - lo) / 2; - if kmv_failure_probability(mid, eps) <= delta { - hi = mid; - } else { - lo = mid + 1; - } - } - hi -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/accuracy/src/estimators/cms.rs b/crates/accuracy/src/estimators/cms.rs index 77d4dcd56..2af2e9f4b 100644 --- a/crates/accuracy/src/estimators/cms.rs +++ b/crates/accuracy/src/estimators/cms.rs @@ -24,40 +24,10 @@ pub(super) fn guarantee( )) } -/// CMS: over-count ≤ ε·N with width `w = ⌈e/ε⌉` columns. -#[cfg(test)] -pub(crate) fn cms_width(eps: f64) -> u32 { - saturating_ceil(std::f64::consts::E / eps, 2, 1 << 26) -} - -/// CMS: failure probability ≤ δ with depth `d = ⌈ln(1/δ)⌉` rows. -/// δ = 0.01 → depth 5. -pub(crate) fn cms_depth(delta: f64) -> u32 { - saturating_ceil((1.0 / delta).ln(), 1, 32) -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn local_guarantee_inverts_frequency_sizing() { - use crate::estimators::size_params; - let c = asap_types::ir::operator::agg_intent::default_cardinality(); - let params = size_params(SketchAlgorithm::Cms, &c, 0.01, 0.001); - let g = sketch_guarantee( - &SketchAlgorithm::Cms, - ¶ms, - &SketchStatistic::PointCount { - key: asap_types::ir::scalar::expr_ir::ColumnRef::SampleValue, - value: None, - }, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::ABSOLUTE_STREAM_L1); - assert!(meets(&g, 0.01, Some(0.001))); - } - #[test] fn heap_evaluation_retains_frequency_metric() { let cms_heap = SketchParams::CmsWithHeap { @@ -76,19 +46,4 @@ mod tests { AccuracyMetric::ABSOLUTE_STREAM_L1 ); } - - /// A top-k heap holds `max(k, ⌈1/ε⌉)` entries over a CMS sized as for counts. - #[test] - fn topk_heap_capacity_respects_accuracy_and_output_count() { - use crate::estimators::size_params; - let intent = AggIntent::TopK { k: 25 }; - assert_eq!( - size_params(SketchAlgorithm::CmsWithHeap, &intent, 0.01, 0.01), - SketchParams::CmsWithHeap { - width: 272, // ⌈e/0.01⌉ - depth: 5, - heap_size: 100, - } - ); - } } diff --git a/crates/accuracy/src/estimators/count_sketch.rs b/crates/accuracy/src/estimators/count_sketch.rs index 9e027377c..2d9cab7ee 100644 --- a/crates/accuracy/src/estimators/count_sketch.rs +++ b/crates/accuracy/src/estimators/count_sketch.rs @@ -56,47 +56,10 @@ fn count_sketch_failure_probability(width: u32, depth: u32) -> Option { )) } -/// CountSketch `L2` point-query width: ε = sqrt(3/w). -#[cfg(test)] -pub(crate) fn count_sketch_width(eps: f64) -> u32 { - saturating_ceil(3.0 / (eps * eps), 2, 1 << 26) -} - -/// The smallest positive odd depth whose binomial median failure -/// probability at `width` is at most `delta`, capped at 255. -#[cfg(test)] -pub(crate) fn count_sketch_depth(width: u32, delta: f64) -> u32 { - if !(delta.is_finite() && delta > 0.0 && delta < 1.0) { - return 255; - } - (1..=255) - .step_by(2) - .find(|&depth| { - count_sketch_failure_probability(width, depth).is_some_and(|tail| tail <= delta) - }) - .unwrap_or(255) -} - #[cfg(test)] mod tests { use super::*; - /// The executor's `CountSketch<_, RegularPath>` takes a column as the - /// low 32 hash bits modulo the width, so a row fails with probability - /// `1/3` (to within `2^-32`) at any width, not `3/8` off powers of two: - /// ε = 0.05 (width 1200) needs depth 47 at δ = 0.01, 81 at 1e-3 and - /// 117 at 1e-4, not 85, 149 and 215. - #[test] - fn depth_follows_the_libraries_modulo_column_hash() { - let width = count_sketch_width(0.05); - assert_eq!(width, 1200); - assert!((row_failure_probability(width) - 1.0 / 3.0).abs() < 1e-6); - assert_eq!( - [0.01, 1e-3, 1e-4].map(|delta| count_sketch_depth(width, delta)), - [47, 81, 117] - ); - } - /// Measured on the library's own `CountSketch<_, RegularPath>`: two /// keys share a column of a 1536-wide row about `1/1536` of the time. /// A `⌈log2 w⌉`-bit hash slice modulo `w` would give `1.125/1536`, 6 @@ -119,48 +82,13 @@ mod tests { ); } - /// Seeded end to end: the sized sketch (ε = 0.05, δ = 0.01, width 1200, - /// depth 47) answers each key of a skewed stream within `ε ‖f‖₂`, at - /// most a δ share of keys missing. - #[test] - fn sized_sketch_meets_its_guarantee_on_a_seeded_stream() { - use asap_sketchlib::{CountSketch, DataInput, RegularPath, Vector2D}; - let (eps, delta) = (0.05, 0.01); - let width = count_sketch_width(eps); - let depth = count_sketch_depth(width, delta); - let mut sketch = CountSketch::, RegularPath>::with_dimensions( - depth as usize, - width as usize, - ); - // 20 000 keys, key i weighted 1e6 / (i + 1) (Zipf, s = 1), in a - // seeded order. - let keys = 20_000u64; - let weight = |key: u64| (1e6 / (key + 1) as f64).round(); - let mut state = 0x9E37_79B9_7F4A_7C15u64; - let mut order: Vec = (0..keys).collect(); - for i in (1..order.len()).rev() { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - order.swap(i, (state % (i as u64 + 1)) as usize); - } - for &key in &order { - sketch.update_weighted(&DataInput::U64(key), weight(key)); - } - let l2 = (0..keys).map(|k| weight(k).powi(2)).sum::().sqrt(); - let bound = (3.0 / f64::from(width)).sqrt() * l2; - let misses = (0..keys) - .filter(|&k| (sketch.estimate(&DataInput::U64(k)) - weight(k)).abs() > bound) - .count(); - assert!(misses as f64 <= delta * keys as f64, "{misses} misses"); - } - #[test] fn count_sketch_uses_an_l2_guarantee() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - let intent = default_cardinality(); - let count_sketch = size_params(SketchAlgorithm::CountSketch, &intent, 0.01, 0.01); + // ε = √(3/30 000) = 0.01; 47 rows fail together with probability ≤ 0.01. + let count_sketch = SketchParams::CountSketch { + width: 30_000, + depth: 47, + }; let guarantee = sketch_guarantee( &SketchAlgorithm::CountSketch, &count_sketch, @@ -176,33 +104,4 @@ mod tests { ); assert!(meets(&guarantee, 0.01, Some(0.01))); } - - /// The depth is the smallest odd one whose binomial tail meets δ. Each - /// is at most Hoeffding's `ln(1/δ) / (2 (1/2 − p)²)` at the row's - /// failure probability `p ≈ 1/3`: `18 ln(1/δ)` at any width. - #[test] - fn count_sketch_depth_is_the_smallest_meeting_the_binomial_tail() { - for width in [1024, 3000, 30_000] { - let hoeffding = 18.0 * (1.0 + 1e-4); - for delta in [0.1, 0.01, 0.001] { - let depth = count_sketch_depth(width, delta); - assert!(depth % 2 == 1); - assert!(count_sketch_failure_probability(width, depth).unwrap() <= delta); - if depth > 1 { - assert!(count_sketch_failure_probability(width, depth - 2).unwrap() > delta); - } - assert!( - f64::from(depth) <= hoeffding * (1.0 / delta).ln(), - "{width} {delta}" - ); - } - } - // Any width fails a row with probability 1/3 (to within 2^-32 w/3): - // P(Bin(3, 1/3) ≥ 2) = 7/27. - for width in [1024, 1000] { - let tail = count_sketch_failure_probability(width, 3).unwrap(); - assert!((tail - 7.0 / 27.0).abs() < 1e-6, "{width}"); - } - assert_eq!(count_sketch_failure_probability(1000, 4), None); - } } diff --git a/crates/accuracy/src/estimators/ddsketch.rs b/crates/accuracy/src/estimators/ddsketch.rs index 5f1a7310c..9184de771 100644 --- a/crates/accuracy/src/estimators/ddsketch.rs +++ b/crates/accuracy/src/estimators/ddsketch.rs @@ -19,8 +19,3 @@ pub(super) fn guarantee( "ddsketch_relative_error_alpha_v1", )) } - -#[cfg(test)] -pub(super) fn size_params(epsilon: f64) -> SketchParams { - SketchParams::DDSketch { alpha: epsilon } -} diff --git a/crates/accuracy/src/estimators/hll.rs b/crates/accuracy/src/estimators/hll.rs index 066a991b5..064480ff5 100644 --- a/crates/accuracy/src/estimators/hll.rs +++ b/crates/accuracy/src/estimators/hll.rs @@ -81,7 +81,7 @@ pub fn parse_classic_failure_statistic(statistic: &str) -> Option<(u8, f64, &str /// distinct bound ([`classic_failure_statistic`]). The failure probability /// is per output row: each group's estimate fails with at most it. `None` /// for a precision outside the contract's. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. +#[allow(dead_code)] // Read again in #1207 PR 8: δ of a distinct count over a source's own rows, with θ. pub(crate) fn classic_guarantee( precision: u8, relative_error: Option, @@ -246,27 +246,6 @@ pub fn min_precision( .unwrap_or(at_least) } -/// Classic HLL's precision for relative error `eps` with failure -/// probability `delta`, without data (#694). Below `1/ε` distinct items a -/// single bucket collision already exceeds the error, and the chance of -/// one is about `N²/(2m)`; a certificate for any bound of at least `1/ε` -/// therefore needs `m ≥ 1/(2ε²δ)`. This is the smallest such precision at -/// which [`ClassicHllConfidence`] certifies `delta` at `eps` for every -/// bound up to `⌈1/ε⌉` (18 when none up to 18 does). Pass 1 then raises -/// the precision by the same rule ([`min_precision`]) when the source's own -/// bound or a smaller failure probability needs it. -#[cfg(test)] -pub(crate) fn hll_precision(eps: f64, delta: f64) -> u8 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return 18; - } - let start = saturating_ceil((1.0 / (2.0 * eps * eps * delta.min(1.0))).log2(), 4, 18) as u8; - let reference = (1.0 / eps).ceil().min(f64::from(MAX_CONTRACT_DISTINCT)) as u64; - (start..=18) - .find(|&p| classic_failure_probability(p, eps, reference) <= delta) - .unwrap_or(18) -} - #[cfg(test)] mod tests { use super::*; @@ -456,38 +435,4 @@ mod tests { assert_eq!(left.estimate(), expected); assert!((expected as f64 - 128.0).abs() / 128.0 <= 0.05); } - /// Pass 1 sizes classic HLL from (ε, δ) without data (#694): the - /// smallest precision with `m ≥ 1/(2ε²δ)` that the tail formula - /// certifies at the target ε, which its guarantee states as its error - /// bound. - #[test] - fn precision_is_sized_from_epsilon_and_delta() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - for (eps, delta, precision) in [(0.05, 0.01, 15), (0.02, 0.01, 17)] { - let params = size_params(SketchAlgorithm::Hll, &default_cardinality(), eps, delta); - let SketchParams::Hll { precision: sized } = params else { - panic!("{params:?}") - }; - assert_eq!(sized, precision); - } - } - - #[test] - fn generic_rse_sizing_does_not_certify_confidence() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_cardinality; - let c = default_cardinality(); - let params = size_params(SketchAlgorithm::Hll, &c, 0.01, 0.01); - let g = sketch_guarantee( - &SketchAlgorithm::Hll, - ¶ms, - &SketchStatistic::Cardinality, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Relative); - assert_eq!(g.spec.failure_probability.evaluate(), None); - assert!(meets(&g, 0.01, None)); - assert!(!meets(&g, 0.01, Some(0.01))); - } } diff --git a/crates/accuracy/src/estimators/kll.rs b/crates/accuracy/src/estimators/kll.rs index f7f76ff28..55c79d96f 100644 --- a/crates/accuracy/src/estimators/kll.rs +++ b/crates/accuracy/src/estimators/kll.rs @@ -4,18 +4,31 @@ use super::*; use asap_types::ir::schema::sketch::KLL_MAX_K; +/// The calibrated guarantee, at failure probability 0.01. pub(super) fn guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, query: &SketchStatistic, +) -> Option { + guarantee_at(algorithm, params, query, KLL_CALIBRATED_FAILURE_PROBABILITY) +} + +/// The rank error of `k` at failure probability `failure_probability`: the +/// calibration at 0.01 and above, scaled by [`delta_scale`] below. `None` +/// for a `k` above the library's cap or a probability outside (0, 1]. +pub(super) fn guarantee_at( + algorithm: &SketchAlgorithm, + params: &SketchParams, + query: &SketchStatistic, + failure_probability: f64, ) -> Option { let SketchParams::Kll { k } = params else { return None; }; - if *k > KLL_MAX_K { + if *k > KLL_MAX_K || !(failure_probability > 0.0 && failure_probability <= 1.0) { return None; } - let delta = KLL_CALIBRATED_FAILURE_PROBABILITY; + let delta = failure_probability.min(KLL_CALIBRATED_FAILURE_PROBABILITY); let scale = delta_scale(delta)?; let contract = if delta < KLL_CALIBRATED_FAILURE_PROBABILITY { KLL_DELTA_SCALED_CONTRACT @@ -48,19 +61,6 @@ pub(crate) fn kll_rank_error_99(k: u32) -> f64 { KLL_RANK_ERROR_COEFFICIENT_99 / f64::from(k).powf(KLL_RANK_ERROR_EXPONENT_99) } -/// Invert Apache DataSketches' empirical 99th-percentile, single-sided KLL -/// normalized rank-error fit: `epsilon = 2.296 / k^0.9723`. Capped at -/// [`KLL_MAX_K`], the largest `k` `asap_sketchlib` builds; an `eps` that -/// needs more gets that `k`, whose guarantee then misses the target. -#[cfg(test)] -pub(crate) fn kll_k(eps: f64) -> u32 { - saturating_ceil( - (KLL_RANK_ERROR_COEFFICIENT_99 / eps).powf(1.0 / KLL_RANK_ERROR_EXPONENT_99), - 8, - KLL_MAX_K, - ) -} - /// How much the rank error at failure probability `delta` exceeds the error /// at the calibrated 0.01, modelling the error as Gaussian: the ratio of the /// two-sided standard normal quantiles, `z(1 − delta/2) / z(1 − 0.01/2)`. @@ -131,55 +131,10 @@ fn normal_quantile(p: f64) -> f64 { } } -/// `k` for rank error `eps` at failure probability `delta`: the empirical -/// 99% calibration for `delta >= 0.01`; below, the calibration at -/// `eps / delta_scale(delta)`, recording `delta` so the guarantee reports it. -#[cfg(test)] -pub(crate) fn size_params(eps: f64, delta: f64) -> SketchParams { - if delta.is_nan() || delta <= 0.0 || delta >= KLL_CALIBRATED_FAILURE_PROBABILITY { - return SketchParams::Kll { k: kll_k(eps) }; - } - SketchParams::Kll { - k: kll_k(eps / delta_scale(delta).unwrap_or(1.0)), - } -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn local_guarantee_inverts_rank_sizing() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - let q = default_quantile(0.99); - let params = size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01); - let g = sketch_guarantee( - &SketchAlgorithm::Kll, - ¶ms, - &SketchStatistic::Quantile { q: 0.99 }, - ) - .unwrap(); - assert_eq!(g.spec.accuracy_metric, AccuracyMetric::Rank); - assert!(meets(&g, 0.01, None)); - assert_eq!(g.spec.failure_probability.evaluate(), Some(0.01)); - assert!(meets(&g, 0.01, Some(0.01))); - assert_eq!(g.approximate_layer_count(), 1); - assert!(g.provenance.iter().any(|source| matches!( - source, - GuaranteeSource::SketchEvaluation { contract, .. } - if contract == "apache_datasketches_kll_empirical_99_a9b42755072b" - ))); - } - - /// `k` never exceeds the 26 602 `asap_sketchlib` silently caps it at, so - /// a guarantee never claims a larger `k` than runs. - #[test] - fn kll_k_is_capped_at_the_library_maximum() { - assert_eq!(kll_k(1e-6), KLL_MAX_K); - assert_eq!(KLL_MAX_K, 26_602); - } - /// A hand-built `k` above the library's cap gives no guarantee. #[test] fn a_k_above_the_library_cap_gives_no_guarantee() { @@ -212,38 +167,5 @@ mod tests { let scale = delta_scale(delta).unwrap(); assert!((scale - expected).abs() < 1e-8, "{delta}: {scale}"); } - assert_eq!( - size_params(0.01, 0.001), - SketchParams::Kll { - k: kll_k(0.01 / ratio), - } - ); - } - - /// KLL `k` inverts the rank-error fit: ε = 0.01 → 269, ε = 0.05 → 52. - #[test] - fn accuracy_target_drives_kll_k() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - let q = default_quantile(0.99); - assert_eq!( - size_params(SketchAlgorithm::Kll, &q, 0.01, 0.01), - SketchParams::Kll { k: 269 } - ); - assert_eq!( - size_params(SketchAlgorithm::Kll, &q, 0.05, 0.01), - SketchParams::Kll { k: 52 } - ); - } - - /// A zero ε saturates KLL `k` to the tightest buildable value. - #[test] - fn degenerate_epsilon_saturates_to_tightest_kll_params() { - use crate::estimators::size_params; - use asap_types::ir::operator::agg_intent::default_quantile; - assert_eq!( - size_params(SketchAlgorithm::Kll, &default_quantile(0.99), 0.0, 0.01), - SketchParams::Kll { k: 26_602 } - ); } } diff --git a/crates/accuracy/src/estimators/mod.rs b/crates/accuracy/src/estimators/mod.rs index 315a64ff8..9530009f1 100644 --- a/crates/accuracy/src/estimators/mod.rs +++ b/crates/accuracy/src/estimators/mod.rs @@ -1,7 +1,5 @@ //! Dispatch committed estimator parameters to their accuracy models. use super::*; -#[cfg(test)] -use asap_types::ir::operator::AggIntent; use asap_types::ir::schema::SummaryTypeCapability; pub(crate) mod cardinality; @@ -12,7 +10,6 @@ pub mod hll; pub(crate) mod kll; pub mod univmon; -#[allow(dead_code)] // Read again for the error bound of each listed θ (`parameter_list`), in #1207 PRs 3–4. pub(super) fn sketch_guarantee( algorithm: &SketchAlgorithm, params: &SketchParams, @@ -62,7 +59,11 @@ fn bounded_guarantee( /// `summary_type` with parameters `parameters` at failure probability /// `failure_probability` (the guarantee's own failure probability): the /// algorithm's error formula (concepts doc, section 2, "Where the guarantee -/// comes from"). `None` when this θ cannot fail that rarely, when the +/// comes from"). Where one parameter sets both the error and the failure +/// probability (KLL's and KMV's `k`), the error is the one that holds at +/// `failure_probability`; where structural parameters set them apart (a +/// Count-Min's width and depth), the bound holds only when θ fails at most +/// that rarely. `None` when this θ cannot fail that rarely, when the /// summary type does not answer `statistic` (its capability row) or has no /// error model (`SummarySample`, `Wavelet`, `StatModel`, and sketch /// readouts without a certified bound). @@ -72,95 +73,26 @@ pub fn theoretical_guarantee( statistic: &SketchStatistic, failure_probability: f64, ) -> Option { - match summary_type { - FieldDataType::Sketch(_) if SummaryTypeCapability::of(summary_type).answers(statistic) => { - let _ = (parameters, failure_probability); - todo!("#1207 PR 4: the algorithm's error formula at θ, at failure_probability") - } - _ => None, + let FieldDataType::Sketch(algorithm) = summary_type else { + return None; + }; + if !SummaryTypeCapability::of(summary_type).answers(statistic) { + return None; } -} - -/// Today's sizing: the parameters of algorithm `kind` for error `eps` at failure -/// probability `delta`. Test-only: the predefined lists (`parameter_list`) -/// are this sizing over an error grid, which a test keeps equal. -#[cfg(test)] -pub(crate) fn size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, -) -> SketchParams { - match kind { - SketchAlgorithm::UnivMon => univmon::size_params(intent, eps, delta), - SketchAlgorithm::Kll => kll::size_params(eps, delta), - SketchAlgorithm::Cms => SketchParams::Cms { - width: cms::cms_width(eps), - depth: cms::cms_depth(delta), - }, - SketchAlgorithm::Hll => SketchParams::Hll { - precision: hll::hll_precision(eps, delta), - }, - SketchAlgorithm::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: cms::cms_width(eps), - depth: cms::cms_depth(delta), - heap_size: topk_capacity(k, eps), - } - } - // Non-preferred candidates (DDSketch / Kmv / CountSketch / - // CountSketchWithHeap) are only reachable once a cost model picks - // them; sized here so that wiring is local. - SketchAlgorithm::DDSketch => ddsketch::size_params(eps), - SketchAlgorithm::Kmv => SketchParams::Kmv { - k: cardinality::kmv_k(eps, delta), - }, - SketchAlgorithm::CountSketch => { - let width = count_sketch::count_sketch_width(eps); - SketchParams::CountSketch { - width, - depth: count_sketch::count_sketch_depth(width, delta), - } - } - SketchAlgorithm::CountSketchWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CountSketchWithHeap is only a TopK candidate"), - }; - let width = count_sketch::count_sketch_width(eps); - SketchParams::CountSketchWithHeap { - width, - depth: count_sketch::count_sketch_depth(width, delta), - heap_size: topk_capacity(k, eps), - } + match parameters { + SketchParams::Kll { .. } => { + kll::guarantee_at(algorithm, parameters, statistic, failure_probability) } - // Listed, not `_`: a family `asap_sketchlib` adds fails to compile - // here until the planner sizes it or lists it as not offered. - SketchAlgorithm::CountSketchWithL2Norm - | SketchAlgorithm::CmsWithSpaceSaving - | SketchAlgorithm::CountSketchWithSpaceSaving - | SketchAlgorithm::CmsFold - | SketchAlgorithm::CountSketchFold - | SketchAlgorithm::HllHip - | SketchAlgorithm::KllDynamic - | SketchAlgorithm::SpaceSaving - | SketchAlgorithm::Elastic - | SketchAlgorithm::Coco - | SketchAlgorithm::Bloom - | SketchAlgorithm::SetAggregator - | SketchAlgorithm::UnivMonTwoTier - | SketchAlgorithm::UnivMonQ - | SketchAlgorithm::HydraCms - | SketchAlgorithm::HydraCountSketch - | SketchAlgorithm::HydraKll - | SketchAlgorithm::HydraHll - | SketchAlgorithm::HydraUnivMon => { - unreachable!("Pass 1 offers only the families in FAMILY_PREFERENCE, not {kind:?}") + SketchParams::Kmv { .. } => { + cardinality::guarantee_at(algorithm, parameters, statistic, failure_probability) } + _ => sketch_guarantee(algorithm, parameters, statistic).filter(|guarantee| { + guarantee + .spec + .failure_probability + .evaluate() + .is_some_and(|own| own <= target_limit(failure_probability)) + }), } } @@ -187,130 +119,110 @@ pub(crate) fn binomial_upper_tail(n: u32, p: f64, k: u32) -> f64 { (tail * (1.0 + 1e-12)).min(1.0) } -/// Accuracy-dependent candidate budget, not a completeness theorem. The -/// membership model must still certify the selected set independently. -#[cfg(test)] -pub(crate) fn topk_capacity(k: usize, eps: f64) -> u32 { - u32::try_from(k) - .unwrap_or(u32::MAX) - .max(saturating_ceil(1.0 / eps, 1, 1 << 26)) -} - -/// `⌈x⌉` clamped to `[lo, hi]`; NaN / non-positive x saturate to `hi` -/// (a degenerate ε means "as accurate as this family goes"). -pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { - if !x.is_finite() || x <= 0.0 { - return hi; - } - (x.ceil() as u32).clamp(lo, hi) -} - #[cfg(test)] mod tests { use super::*; - /// Each algorithm's predefined list (`parameter_list`) is today's sizing - /// over the error grid ε = 0.1 … 0.001 at failure probability 0.01, - /// smallest first, without repeats; KMV's and UnivMon's distinct counts - /// stop at ε = 0.01. Count-Min lists each sized width with depths 5, 7, - /// 10 and 12, Count Sketch with 47, 81, 117 and 155, and a heap variant - /// each of those with - /// every heap size of the 1-2-5 series 10 … 10 000. UnivMon's list also - /// combines each L2 shape with each distinct-count shape (28 entries). + /// Every listed θ has a theoretical guarantee for the statistic its + /// algorithm is listed for, at failure probability 0.01, in the + /// algorithm's metric; along a list of one structure the error never + /// grows (the lists run weakest first). #[test] - fn parameter_list_is_the_sizing_over_the_error_grid() { - use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; + fn every_listed_parameters_has_a_guarantee_at_one_percent() { + use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::stages::parameter_list; - let full_grid = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; - let distinct_grid = [0.1, 0.05, 0.02, 0.01]; - let cms_depths = [5, 7, 10, 12]; - let count_sketch_depths = [47, 81, 117, 155]; - let heap_sizes = [10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000]; - let l2 = AggIntent::FrequencyL2 { col: None }; - let top_1 = AggIntent::TopK { k: 1 }; - let quantile = default_quantile(0.5); - let distinct = default_cardinality(); - for (algorithm, intents) in [ - (SketchAlgorithm::Kll, vec![&quantile]), - (SketchAlgorithm::DDSketch, vec![&quantile]), - (SketchAlgorithm::Hll, vec![&distinct]), - (SketchAlgorithm::Kmv, vec![&distinct]), - (SketchAlgorithm::Cms, vec![&distinct]), - (SketchAlgorithm::CountSketch, vec![&distinct]), - (SketchAlgorithm::CmsWithHeap, vec![&top_1]), - (SketchAlgorithm::CountSketchWithHeap, vec![&top_1]), - (SketchAlgorithm::UnivMon, vec![&l2, &distinct, &quantile]), + let point = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: Some("1".into()), + }; + for (algorithm, statistic, metric) in [ + ( + SketchAlgorithm::Kll, + SketchStatistic::Quantile { q: 0.5 }, + AccuracyMetric::Rank, + ), + ( + SketchAlgorithm::DDSketch, + SketchStatistic::Quantile { q: 0.5 }, + AccuracyMetric::Relative, + ), + ( + SketchAlgorithm::Kmv, + SketchStatistic::Cardinality, + AccuracyMetric::Relative, + ), + ( + SketchAlgorithm::Cms, + point.clone(), + AccuracyMetric::ABSOLUTE_STREAM_L1, + ), + ( + SketchAlgorithm::CountSketch, + point, + AccuracyMetric::ABSOLUTE_STREAM_L2, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchStatistic::TopK { k: 10 }, + AccuracyMetric::ABSOLUTE_STREAM_L1, + ), ] { - let mut sized: Vec = Vec::new(); - for intent in intents { - let grid: &[f64] = match (algorithm, intent) { - ( - SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon, - AggIntent::Cardinality { .. }, - ) => &distinct_grid, - _ => &full_grid, - }; - for &eps in grid { - let listed: Vec = match size_params(algorithm, intent, eps, 0.01) - { - SketchParams::Cms { width, .. } => cms_depths - .iter() - .map(|&depth| SketchParams::Cms { width, depth }) - .collect(), - SketchParams::CountSketch { width, .. } => count_sketch_depths - .iter() - .map(|&depth| SketchParams::CountSketch { width, depth }) - .collect(), - SketchParams::CmsWithHeap { width, .. } => cms_depths - .iter() - .flat_map(|&depth| { - heap_sizes - .iter() - .map(move |&heap_size| SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }) - }) - .collect(), - SketchParams::CountSketchWithHeap { width, .. } => count_sketch_depths - .iter() - .flat_map(|&depth| { - heap_sizes.iter().map(move |&heap_size| { - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } - }) - }) - .collect(), - other => vec![other], - }; - for params in listed { - if !sized.contains(¶ms) { - sized.push(params); - } - } - } + let summary_type = FieldDataType::Sketch(algorithm); + let errors: Vec = parameter_list(algorithm) + .iter() + .map(|parameters| { + let guarantee = + theoretical_guarantee(&summary_type, parameters, &statistic, 0.01) + .unwrap_or_else(|| panic!("{algorithm:?} {parameters:?}")); + assert_eq!(guarantee.spec.accuracy_metric, metric, "{algorithm:?}"); + guarantee.spec.error.evaluate().unwrap() + }) + .collect(); + assert!(!errors.is_empty(), "{algorithm:?}"); + if matches!( + algorithm, + SketchAlgorithm::Kll | SketchAlgorithm::DDSketch | SketchAlgorithm::Kmv + ) { + assert!( + errors.windows(2).all(|pair| pair[1] <= pair[0]), + "{algorithm:?}: {errors:?}" + ); } - let list = parameter_list(algorithm); - let (listed_sized, combined): (Vec<_>, Vec<_>) = - list.iter().partition(|params| sized.contains(params)); - assert_eq!( - format!("{listed_sized:?}"), - format!("{:?}", sized.iter().collect::>()), - "{algorithm:?}" - ); - let expected_combined = if algorithm == SketchAlgorithm::UnivMon { - 28 - } else { - 0 - }; - assert_eq!(combined.len(), expected_combined, "{algorithm:?}"); } } + /// One parameter sets both (KLL's k): its rank error grows as the + /// required failure probability shrinks. Separate parameters (CMS width + /// and depth): the bound holds only when the depth fails rarely enough. + #[test] + fn a_guarantee_is_evaluated_at_the_required_failure_probability() { + use asap_types::ir::scalar::expr_ir::ColumnRef; + let quantile = SketchStatistic::Quantile { q: 0.5 }; + let kll = FieldDataType::Sketch(SketchAlgorithm::Kll); + let k = SketchParams::Kll { k: 547 }; + let error_at = |delta| { + theoretical_guarantee(&kll, &k, &quantile, delta) + .unwrap() + .spec + .error + .evaluate() + .unwrap() + }; + assert_eq!(error_at(1.0), error_at(0.01)); + assert!(error_at(1e-6) > error_at(0.01)); + let cms = FieldDataType::Sketch(SketchAlgorithm::Cms); + let point = SketchStatistic::PointCount { + key: ColumnRef::SampleValue, + value: None, + }; + let depth = |depth| SketchParams::Cms { width: 272, depth }; + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-2).is_some()); + assert!(theoretical_guarantee(&cms, &depth(5), &point, 1e-4).is_none()); + assert!(theoretical_guarantee(&cms, &depth(10), &point, 1e-4).is_some()); + // A statistic the summary type does not answer has no guarantee. + assert!(theoretical_guarantee(&kll, &k, &SketchStatistic::Cardinality, 0.01).is_none()); + } + /// Classic HLL's precision grows until its failure probability over the /// source's distinct bound meets the target, up to 18. #[test] diff --git a/crates/accuracy/src/estimators/univmon.rs b/crates/accuracy/src/estimators/univmon.rs index c0545c20c..25c3ad86e 100644 --- a/crates/accuracy/src/estimators/univmon.rs +++ b/crates/accuracy/src/estimators/univmon.rs @@ -17,23 +17,12 @@ //! (OctoSketch aggregators fed promoted deltas) are not covered: the //! executor rejects them. use super::*; -#[cfg(test)] -use asap_types::ir::operator::AggIntent; /// Layers `asap_sketchlib` builds: layer `i` hashes at seed index `i`, and /// seed 19 picks the layers a key reaches. #[cfg(test)] pub(crate) const MAX_LAYERS: u8 = asap_sketchlib::sketch_framework::univmon::UNIVMON_MAX_LAYERS as u8; -/// The most rows an L2 sketch is sized with: the deepest odd shape within -/// the 20 rows the executor builds ([`hash_independent`]), so the row median -/// is one row's estimate, and within the hash budget at the more than 20 -/// columns an L2 guarantee needs (5 column bits: 19 · 6 = 114 ≤ 128). -#[cfg(test)] -const MAX_L2_ROWS: u32 = 19; -/// Each row of a layer reads its column and sign bits from one hash. -#[cfg(test)] -const HASH_BITS: u32 = 128; /// The per-row failure probability the L2 guarantee fixes: a row misses by /// more than `ε₂ F2` with probability at most `2q/ε₂² = 0.1`. Fixing it /// lets a shape state one `(ε, δ)`: the columns give `ε`, the rows `δ`. @@ -43,9 +32,6 @@ const L2_ROW_FAILURE: f64 = 0.1; const DISTINCT_VARIANCE: f64 = 1.35; /// The smallest heap the distinct-count variance bound holds for. const DISTINCT_MIN_HEAP: u32 = 8; -#[cfg(test)] -const CARD_CAP: u32 = 1 << 26; - const L2_CONTRACT: &str = "univmon_layer0_f2_median_chebyshev_binomial_v1"; const DISTINCT_CONTRACT: &str = "univmon_distinct_complete_top_layer_chebyshev_v1"; /// Names the probability that the top layer is incomplete, left symbolic @@ -149,79 +135,15 @@ fn l2_failure_probability(rows: u32) -> f64 { super::binomial_upper_tail(rows, L2_ROW_FAILURE, rows.div_ceil(2)) } -/// The smallest rows and columns whose L2 guarantee meets `(eps, delta)`, -/// clamped to the hash budget: with more rows than [`MAX_L2_ROWS`], or more -/// columns than `rows` leave bits for, the shape is the largest one built -/// and its guarantee misses the target. -#[cfg(test)] -fn l2_shape(eps: f64, delta: f64) -> (u32, u32) { - let rows = (1..=MAX_L2_ROWS) - .find(|&rows| l2_failure_probability(rows) <= delta) - .unwrap_or(MAX_L2_ROWS); - let max_bits = (HASH_BITS / rows - 1).min(26); - let max_cols = 1u32 << max_bits; - let meets = |cols: u32| l2_relative_error(cols).is_some_and(|error| error <= eps); - if !(eps.is_finite() && eps > 0.0) || !meets(max_cols) { - return (rows, max_cols); - } - // The norm's error `ε` needs `ε₂ ≤ 1 − (1 − ε)²`; a power-of-two width - // needs `2/(0.1 ε₂²)` columns and any other `9/(4 · 0.1 ε₂²)`. - let f2_error = 1.0 - (1.0 - eps.min(1.0)).powi(2); - let general = 9.0 / (4.0 * L2_ROW_FAILURE * f2_error * f2_error); - // Off powers of two the error falls as the width grows, and a power of - // two meets the target whenever its non-power-of-two neighbours do, so - // walking down from the estimate stops at the smallest such width. - let mut general = saturating_ceil(general, 1, max_cols); - while general > 1 && meets(general - 1) { - general -= 1; - } - while !meets(general) { - general += 1; - } - let power_of_two = (0..=max_bits) - .map(|bits| 1u32 << bits) - .find(|&cols| meets(cols)) - .unwrap_or(max_cols); - (rows, general.min(power_of_two)) -} - /// `(e/4)^(k/2)`: the chance the top layer of a heap-`k` pyramid misses a /// key, once the expected keys reaching it are at most `k/2` (Chernoff). fn incomplete_top_layer_bound(heap_size: u32) -> f64 { (f64::from(heap_size) / 2.0 * (1.0 - 2.0 * std::f64::consts::LN_2)).exp() } -/// The Chebyshev share of the distinct count's failure probability at -/// heap `k` and relative error `eps`: `1.35/(k ε²)`. -#[cfg(test)] -fn distinct_chebyshev(heap_size: u32, eps: f64) -> f64 { - DISTINCT_VARIANCE / (f64::from(heap_size) * eps * eps) -} - -/// The smallest heap `k ≥ 8` whose distinct count misses by more than -/// `eps · F0` with probability at most `delta`, top-layer completeness -/// included: `1.35/(k ε²) + (e/4)^(k/2) ≤ δ`. Capped; a capped heap's -/// guarantee misses the target. -#[cfg(test)] -fn distinct_heap(eps: f64, delta: f64) -> u32 { - if !(eps.is_finite() && eps > 0.0 && delta.is_finite() && delta > 0.0) { - return CARD_CAP; - } - let meets = |k: u32| distinct_chebyshev(k, eps) + incomplete_top_layer_bound(k) <= delta; - let mut k = saturating_ceil( - DISTINCT_VARIANCE / (delta * eps * eps), - DISTINCT_MIN_HEAP, - CARD_CAP, - ); - while !meets(k) && k < CARD_CAP { - k += 1; - } - k -} - /// The statistic a UnivMon distinct count over `source` leaves its /// top-layer failure probability as. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. +#[allow(dead_code)] // Read again in #1207 PR 8: δ of a distinct count over a source's own rows, with θ. pub(crate) fn incomplete_top_layer_statistic(heap_size: u32, layers: u8, source: &str) -> String { format!("{INCOMPLETE_TOP_LAYER}[k={heap_size},layers={layers}]({MAX_DISTINCT}[{source}])") } @@ -266,7 +188,7 @@ pub fn incomplete_top_layer_probability(heap_size: u32, layers: u8, max_distinct /// probability `δ_c` plus the chance of an incomplete top layer, left /// symbolic in the source's distinct bound. `None` for a heap under 8, a /// `delta` too small for the heap, or a shape the library would not build. -#[allow(dead_code)] // Read again in #1207 PR 4, with θ from `LogicalProperties::parameters`. +#[allow(dead_code)] // Read again in #1207 PR 8: δ of a distinct count over a source's own rows, with θ. pub(crate) fn distinct_guarantee( params: &SketchParams, delta: f64, @@ -319,46 +241,9 @@ pub(crate) fn distinct_guarantee( )) } -/// Size a UnivMon for one readout's `(eps, delta)`: -/// - L2: the smallest rows and columns meeting the target ([`l2_shape`]), -/// one layer (L2 reads only layer 0) and a one-key heap; -/// - distinct count: the smallest heap meeting it ([`distinct_heap`]) and -/// the most layers, since the layers the top layer needs depend on the -/// source's distinct bound, which plan selection binds. The bound holds -/// whatever the Count Sketch keeps in the heaps, so each layer's is one -/// counter; -/// - total and entropy: the baseline shape (the total is exact; entropy -/// has no certified bound). -#[cfg(test)] -pub(super) fn size_params(intent: &AggIntent, eps: f64, delta: f64) -> SketchParams { - match intent { - AggIntent::FrequencyL2 { .. } => { - let (sketch_rows, sketch_cols) = l2_shape(eps, delta); - SketchParams::UnivMon { - heap_size: 1, - sketch_rows, - sketch_cols, - layers: 1, - } - } - AggIntent::Cardinality { .. } => SketchParams::UnivMon { - heap_size: distinct_heap(eps, delta), - sketch_rows: 1, - sketch_cols: 1, - layers: MAX_LAYERS, - }, - _ => BASELINE, - } -} - #[cfg(test)] mod tests { use super::*; - use asap_types::ir::operator::agg_intent::default_cardinality; - - fn l2_intent() -> AggIntent { - AggIntent::FrequencyL2 { col: None } - } fn univmon(heap_size: u32, rows: u32, cols: u32, layers: u8) -> SketchParams { SketchParams::UnivMon { @@ -377,55 +262,6 @@ mod tests { ) } - fn with(params: &SketchParams, f: impl FnOnce(&mut u32, &mut u32, &mut u32)) -> SketchParams { - let mut params = params.clone(); - let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - .. - } = &mut params - else { - unreachable!() - }; - f(heap_size, sketch_rows, sketch_cols); - params - } - - const TARGETS: [(f64, f64); 6] = [ - (0.01, 0.01), - (0.02, 0.01), - (0.05, 0.01), - (0.05, 0.001), - (0.1, 0.05), - (0.2, 0.2), - ]; - - /// Each L2 shape meets its target as a relative error, and one fewer - /// row or column does not. - #[test] - fn l2_shape_is_the_smallest_meeting_its_target() { - for (eps, delta) in TARGETS { - let params = size_params(&l2_intent(), eps, delta); - let guarantee = l2(¶ms).expect("an L2 guarantee"); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!( - meets(&guarantee, eps, Some(delta)), - "({eps}, {delta}): {params:?}" - ); - let fewer_rows = with(¶ms, |_, rows, _| *rows -= 1); - assert!( - l2(&fewer_rows).is_none_or(|g| !meets(&g, eps, Some(delta))), - "({eps}, {delta}): {fewer_rows:?}" - ); - let fewer_cols = with(¶ms, |_, _, cols| *cols -= 1); - assert!( - l2(&fewer_cols).is_none_or(|g| !meets(&g, eps, Some(delta))), - "({eps}, {delta}): {fewer_cols:?}" - ); - } - } - /// The general collision bound `9/(8w)` applies off powers of two: a /// width just past a power of two is worse than that power of two. #[test] @@ -447,37 +283,6 @@ mod tests { assert!(l2_failure_probability(4) > l2_failure_probability(3)); } - /// Shapes past the hash budget or the layer limit certify nothing, and - /// L2 sizing never builds one: a target it cannot reach gets the - /// largest shape it can build, whose guarantee misses it. - #[test] - fn l2_shapes_stay_within_the_hash_budget() { - // 8 · (16 + 1) = 136 > 128. - assert!(l2(&univmon(1, 8, 1 << 16, 1)).is_none()); - assert!(l2(&univmon(1, 7, 1 << 16, 1)).is_some()); - assert!(l2(&univmon(1, 5, 1024, 20)).is_none()); - for (eps, delta) in [(0.001, 1e-6), (1e-5, 0.01), (0.01, 1e-12)] { - let params = size_params(&l2_intent(), eps, delta); - let SketchParams::UnivMon { - sketch_rows, - sketch_cols, - layers, - .. - } = params - else { - unreachable!() - }; - assert!( - hash_independent(sketch_rows, sketch_cols, layers), - "{params:?}" - ); - assert!( - !meets(&l2(¶ms).unwrap(), eps, Some(delta)), - "{params:?}" - ); - } - } - /// The executor builds at most 20 rows, the rows a UnivMon state /// encodes, so a 21-row UnivMon certifies nothing even though its hashes /// fit the budget (21 · (5 + 1) = 126 ≤ 128 at 32 columns). @@ -489,53 +294,12 @@ mod tests { assert!(l2(&univmon(1, 21, 32, 1)).is_none()); } - /// Each distinct-count heap meets its target once the source's bound - /// is bound, and one fewer key does not. - #[test] - fn distinct_heap_is_the_smallest_meeting_its_target() { - for (eps, delta) in TARGETS { - let params = size_params(&default_cardinality(), eps, delta); - let bind = |params: &SketchParams, max_distinct| { - let guarantee = - distinct_guarantee(params, delta, "m").expect("a distinct guarantee"); - guarantee.bind(&|statistic| { - let (k, layers, source) = parse_incomplete_top_layer_statistic(statistic)?; - (source == "m") - .then(|| incomplete_top_layer_probability(k, layers, max_distinct)) - }) - }; - for max_distinct in [9, 10_000_000] { - let guarantee = bind(¶ms, max_distinct); - assert_eq!(guarantee.spec.accuracy_metric, AccuracyMetric::Relative); - assert!( - meets(&guarantee, eps, Some(delta)), - "({eps}, {delta}): {params:?}" - ); - let smaller = with(¶ms, |heap, _, _| *heap -= 1); - assert!( - !meets(&bind(&smaller, max_distinct), eps, Some(delta)), - "({eps}, {delta}): {smaller:?}" - ); - } - } - } - - /// Example 2's Q1 (ε = 0.02, δ = 0.01) needs a heap of 337,500 keys, - /// `⌈1.35/(δ ε²)⌉`: the top layer's term is negligible at that heap. - /// At (0.5, 0.5) it is not: 1.35/(12 · 0.25) + (e/4)^6 > 0.5, while 13 - /// keys give 0.415 + 0.081. - #[test] - fn distinct_heap_for_example2_q1() { - assert_eq!(distinct_heap(0.02, 0.01), 337_500); - assert_eq!(distinct_heap(0.5, 0.5), 13); - } - /// Without the source's bound the failure probability stays unknown /// and no target is met; a bound the pyramid is too shallow for gives /// probability 1. #[test] fn distinct_guarantee_needs_the_source_bound_and_enough_layers() { - let params = size_params(&default_cardinality(), 0.05, 0.01); + let params = univmon(54_000, 1, 1, MAX_LAYERS); let guarantee = distinct_guarantee(¶ms, 0.01, "m").unwrap(); assert_eq!(guarantee.spec.failure_probability.evaluate(), None); assert!(!meets(&guarantee, 0.05, Some(0.01))); @@ -563,7 +327,7 @@ mod tests { assert!(distinct_guarantee(&univmon(7, 1, 1, 19), 0.5, "m").is_none()); assert!(distinct_guarantee(&univmon(8, 1, 1, 20), 0.5, "m").is_none()); assert!(distinct_guarantee(&univmon(8, 1, 1, 19), 0.5, "m").is_some()); - let sized = size_params(&default_cardinality(), 0.05, 0.01); + let sized = univmon(54_000, 1, 1, MAX_LAYERS); assert!(guarantee( &SketchAlgorithm::UnivMon, &sized, @@ -591,7 +355,7 @@ mod tests { /// The total stays exact; entropy stays uncertified. #[test] fn total_is_exact_and_entropy_uncertified() { - let params = size_params(&default_cardinality(), 0.05, 0.01); + let params = univmon(54_000, 1, 1, MAX_LAYERS); let total = guarantee( &SketchAlgorithm::UnivMon, ¶ms, diff --git a/crates/accuracy/src/evidence.rs b/crates/accuracy/src/evidence.rs deleted file mode 100644 index 802589c91..000000000 --- a/crates/accuracy/src/evidence.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Scoped source contracts and statistics the accuracy rules consult. -use super::*; - -/// Statistics a propagation rule may consult. Every field is optional and -/// defaults to "unknown": a rule that needs a missing statistic emits a -/// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. -#[derive(Debug, Clone, Default, PartialEq)] -pub struct PropagationStats { - /// Provenance for supplied evidence (source, observation identity, etc.). - pub(crate) evidence_provenance: Vec, - /// Whether every input value is known to be non-negative — required by - /// the multiplicative relative-error rule, which is unsound across a - /// sign change. - pub(crate) values_non_negative: Option, - /// The statistic an unknown sign is named by when the folded values are - /// one metric's raw data samples or plain sums of them - /// ([`raw_data_samples_sign_statistic`](crate::composition::raw_data_samples_sign_statistic)): - /// the relative bound is then gated on it, and plan selection binds it - /// to 1 when the metric is declared a counter. `None`: the bare - /// `values_non_negative`, which nothing binds. - pub(crate) sign_statistic: Option, - /// Number of input rows an exact aggregation consumes (e.g. the number - /// of groups a function folds), for exact aggregate union bounds - /// bound over per-input failures. - pub(crate) input_row_count: Option, - /// The statistic an unknown row count is named by (default - /// `input_row_count`), so a fold whose rows the data workload's input - /// cardinality does not bound can leave a statistic nothing binds. - pub(crate) row_count_statistic: Option<&'static str>, - /// Lower confidence bound of the kth selected TopK item, after widening - /// the interval by the sketch's own estimation error. - pub(crate) topk_selected_lower_bound: Option, - /// Greatest upper confidence bound among excluded TopK items, after - /// widening the interval by the sketch's own estimation error. - pub(crate) topk_excluded_upper_bound: Option, - /// Union-bound failure probability of all intervals used by the margin - /// certificate. - pub(crate) topk_interval_failure_probability: Option, -} diff --git a/crates/accuracy/src/function_rules.rs b/crates/accuracy/src/function_rules.rs deleted file mode 100644 index fd5eb81d4..000000000 --- a/crates/accuracy/src/function_rules.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! The accuracy composition rule of each exact function applied to an -//! approximate input, shared by Pass 1's guarantee composition and the -//! accuracy algebra. -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::CompositionOperator; - -/// The composition operator that propagates an approximate input's guarantee -/// through `intent`; `None` for a function with no registered rule over -/// approximate input, whose callers must fail closed. -pub(crate) fn approximate_input_composition(intent: &AggIntent) -> Option { - Some(match intent { - AggIntent::Sum { .. } => CompositionOperator::ExactSum, - AggIntent::Min { .. } | AggIntent::Max { .. } => CompositionOperator::ExactExtremum, - AggIntent::Avg { .. } => CompositionOperator::ExactAverage, - AggIntent::Rate => CompositionOperator::CounterRate, - AggIntent::IRate => CompositionOperator::InstantCounterRate, - AggIntent::Increase => CompositionOperator::CounterIncrease, - _ => return None, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn counter_functions_have_distinct_definition_rules() { - assert_eq!( - approximate_input_composition(&AggIntent::Rate), - Some(CompositionOperator::CounterRate) - ); - assert_eq!( - approximate_input_composition(&AggIntent::IRate), - Some(CompositionOperator::InstantCounterRate) - ); - assert_eq!( - approximate_input_composition(&AggIntent::Increase), - Some(CompositionOperator::CounterIncrease) - ); - } -} diff --git a/crates/accuracy/src/lib.rs b/crates/accuracy/src/lib.rs index 9c480976d..962cde967 100644 --- a/crates/accuracy/src/lib.rs +++ b/crates/accuracy/src/lib.rs @@ -1,38 +1,29 @@ //! `asap-accuracy` — the accuracy rules the planning stages share: the -//! planner's built-in analytical rules (estimators, composition, guarantee -//! propagation through a DAG, sizing, budget split and a target's comparison -//! limit [`target_limit`]), the deployment's measured accuracy ([`AccuracyEvidence`]) and $\alpha^{\log}$ -//! of a logical edge ([`producer_satisfies_logical_requirements`]). -//! -//! Estimator models derive theoretical guarantees, composition propagates -//! them, `propagation` composes them through every node of a DAG, `sizing` -//! sizes a summary for its target, the budget split sizes each approximate -//! input of a composition for an equal share of its target, and `evidence` -//! holds the scoped contracts and statistics the rules consult. +//! theoretical guarantee of reading a statistic from a summary with given +//! parameters ([`theoretical_guarantee`], from the per-algorithm estimators), +//! the deployment's measured accuracy ([`AccuracyEvidence`]), a target's +//! comparison limit [`target_limit`], and $\alpha^{\log}$ of a logical edge +//! ([`producer_satisfies_logical_requirements`]). How an error propagates +//! through an operator is its error contract +//! (`LogicalRequirements::error_propagation` in `asap-types`), applied per +//! edge. //! See `docs/design_docs/concepts/accuracy-models.md` for the design. //! //! It depends only on `asap-types`, never on a stage crate, so every stage //! can use it. -mod budget; -pub mod composition; pub mod estimators; -mod evidence; -mod function_rules; mod model; -pub mod propagation; pub mod sizing; -#[cfg(feature = "test-support")] -pub mod test_support; -pub use budget::per_input_target; pub use estimators::theoretical_guarantee; -pub(crate) use evidence::PropagationStats; -pub use model::{producer_satisfies_logical_requirements, AccuracyEvidence, NoMeasuredGuarantees}; +pub use model::{ + accepted_metrics, producer_satisfies_logical_requirements, readout_metric, AccuracyEvidence, + NoMeasuredGuarantees, +}; use asap_types::ir::properties::{ - AccuracyError, AccuracyGuarantee, AccuracyMetric, AccuracyTarget, BoundExpr, - CompositionOperator, GuaranteeSource, ProbabilityExpr, + AccuracyGuarantee, AccuracyMetric, BoundExpr, GuaranteeSource, ProbabilityExpr, }; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; diff --git a/crates/accuracy/src/model.rs b/crates/accuracy/src/model.rs index 26249ce64..ec0aa3a04 100644 --- a/crates/accuracy/src/model.rs +++ b/crates/accuracy/src/model.rs @@ -1,8 +1,16 @@ //! The deployment's measured accuracy, and α^log of a logical edge. -use asap_types::ir::properties::AccuracyGuarantee; -use asap_types::ir::schema::{FieldDataType, SketchParams, SketchStatistic}; -use asap_types::stages::{CandidateForMemoGroupId, Memo}; +use crate::{target_limit, theoretical_guarantee}; +use asap_types::ir::operator::asap::summary_type_of_operator; +use asap_types::ir::properties::{AccuracyGuarantee, AccuracyMetric}; +use asap_types::ir::schema::{ + FieldDataType, MergedError, SketchParams, SketchStatistic, SummaryTypeCapability, +}; +use asap_types::ir::{ASAPOp, Operator, QueryRoot}; +use asap_types::stages::{ + parameter_list, CandidateForMemoGroupId, LogicalRequirements, Memo, MemoGroupId, + RootPruneReason, +}; /// The deployment's measured accuracy, per summary configuration. A /// producer's guarantee for a readout is its measurement when there is one, @@ -40,17 +48,238 @@ impl AccuracyEvidence for NoMeasuredGuarantees { } } -/// $\alpha^{\log}$ of one edge (concepts doc, section 2): for a readout's -/// input, the producer's guarantee for the readout's statistic at the -/// required failure probability is within the required error, and a -/// top-k's heap holds k; for any other input, the metric rule (section 4). +/// $\alpha^{\log}$ of one edge (concepts doc, section 2): `Ok` when +/// `producer` may feed input `input_index_of_consumer` of `consumer`, else +/// why not, for Pass 2's pruned list. `accepted` is +/// [`accepted_metrics`] of `memo`. +/// - A readout's input (`SummaryEstimate`) from a sketch state: the +/// readout's metric is one its output accepts (the metric rule; no rule +/// where no query with a target reaches it), and the guarantee of the +/// producer's θ for the readout's statistic, at the required failure +/// probability, is within the required error and in its metric: the +/// deployment's measurement (`evidence`) when it has one, else +/// [`theoretical_guarantee`]; a merged state (any producer but a build) +/// must keep the guarantee (`MergedError::Exact` or +/// `AsIfBuiltOverUnion`). A top-k's heap holds its `k` whatever the +/// requirement. +/// - Any other input: the metric rule, when the producer is a sketch +/// readout: the consumer accepts the readout's metric on that input. +/// Only readouts introduce error, so an edge from any other producer +/// holds; a consumer reading an approximate value through it is cut +/// below, on the readout's edge. pub fn producer_satisfies_logical_requirements( memo: &Memo, + accepted: &[Option>], producer: CandidateForMemoGroupId, consumer: CandidateForMemoGroupId, input_index_of_consumer: usize, evidence: &dyn AccuracyEvidence, -) -> bool { - let _ = (memo, producer, consumer, input_index_of_consumer, evidence); - todo!("#1207 PR 4: the readout rule against RequiredAccuracy, else the metric rule") +) -> Result<(), RootPruneReason> { + let consumer_candidate = memo.candidate(consumer); + let input = input_index_of_consumer; + let metric_rule = |producer_metric: AccuracyMetric, accepted_metrics: Vec| { + if accepted_metrics.contains(&producer_metric) { + Ok(()) + } else { + Err(RootPruneReason::MetricRule { + consumer, + input, + producer_metric, + accepted_metrics, + }) + } + }; + let operator = &consumer_candidate.expression.operator; + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query: statistic, + }) = operator + else { + let (Some(producer_metric), Some(accepted)) = ( + sketch_readout_metric(memo, producer), + &accepted[consumer.group.0 as usize], + ) else { + return Ok(()); + }; + let on_input = accepted + .iter() + .copied() + .filter(|&metric| { + LogicalRequirements::error_propagation(memo, operator, metric).is_some() + }) + .collect(); + return metric_rule(producer_metric, on_input); + }; + let summary_type = + summary_type_of_operator(&memo.groups()[summary_input.0 as usize].expression); + let producer_candidate = memo.candidate(producer); + let (Some(summary_type @ FieldDataType::Sketch(_)), Some(parameters)) = ( + summary_type, + producer_candidate.logical_properties.parameters(), + ) else { + // An exact state reads exactly. + return Ok(()); + }; + if let (SketchStatistic::TopK { k }, Some(heap_size)) = (statistic, heap_size(parameters)) { + if (heap_size as usize) < *k { + return Err(RootPruneReason::HeapSmallerThanTopK { + consumer, + input, + k: *k, + }); + } + } + if let (Some(metric), Some(accepted)) = ( + readout_metric(summary_type, statistic), + &accepted[consumer.group.0 as usize], + ) { + metric_rule(metric, accepted.clone())?; + } + let Some(required) = consumer_candidate.input_requirements[input].required_accuracy else { + return Ok(()); + }; + let merged = !matches!( + producer_candidate.expression.operator, + Operator::ASAP(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) + ); + let keeps = !merged + || matches!( + SummaryTypeCapability::of(summary_type).merged_error(statistic), + Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) + ); + let guarantee = evidence + .measured_guarantee( + summary_type, + parameters, + statistic, + required.failure_probability, + ) + .or_else(|| { + theoretical_guarantee( + summary_type, + parameters, + statistic, + required.failure_probability, + ) + }); + let meets = guarantee.is_some_and(|guarantee| { + guarantee.spec.accuracy_metric == required.metric + && guarantee + .spec + .error + .evaluate() + .is_some_and(|error| error <= target_limit(required.max_error)) + }); + if keeps && meets { + Ok(()) + } else { + Err(RootPruneReason::BelowRequiredAccuracy { + consumer, + input, + required, + }) + } +} + +/// Per group, by group id, the metrics its output may be off in for the +/// queries with an accuracy target that reach it (concepts doc, section 4, +/// "Metric rule"): a query root accepts its target's metric (every metric +/// when the target names none); a candidate's input accepts the metrics +/// its output accepts that its error contract +/// (`LogicalRequirements::error_propagation`) carries; a group accepts the +/// union over its consumers. `None` for a group no such query reaches: no +/// metric rule applies there (accuracy not enforced). +pub fn accepted_metrics(memo: &Memo) -> Vec>> { + let mut accepted: Vec>> = vec![None; memo.groups().len()]; + let mut pending: Vec = Vec::new(); + for query in &memo.queries { + let (Some(target), QueryRoot::Operator(root)) = (&query.accuracy_target, &query.root) + else { + continue; + }; + let metrics: Vec = AccuracyMetric::ALL + .into_iter() + .filter(|&metric| target.accuracy_metric.is_none_or(|named| named == metric)) + .collect(); + if add_metrics(&mut accepted[root.0 as usize], &metrics) { + pending.push(*root); + } + } + while let Some(group) = pending.pop() { + let metrics = accepted[group.0 as usize].clone().unwrap_or_default(); + for candidate in &memo.groups()[group.0 as usize].candidates { + let operator = &candidate.expression.operator; + let carried: Vec = metrics + .iter() + .copied() + .filter(|&metric| { + LogicalRequirements::error_propagation(memo, operator, metric).is_some() + }) + .collect(); + for &child in operator.children() { + if add_metrics(&mut accepted[child.0 as usize], &carried) { + pending.push(child); + } + } + } + } + accepted +} + +/// Adds `metrics` to a group's accepted metrics (`None`: not yet reached); +/// whether they changed. +fn add_metrics(known: &mut Option>, metrics: &[AccuracyMetric]) -> bool { + let mut changed = known.is_none(); + let known = known.get_or_insert_with(Vec::new); + for &metric in metrics { + if !known.contains(&metric) { + known.push(metric); + changed = true; + } + } + changed +} + +/// The metric of the error reading `statistic` from a state of +/// `summary_type` has: its first listed θ's guarantee's. `None` for an +/// exact state and for a summary type with no listed θ or no guarantee +/// for the statistic. +pub fn readout_metric( + summary_type: &FieldDataType, + statistic: &SketchStatistic, +) -> Option { + let FieldDataType::Sketch(algorithm) = summary_type else { + return None; + }; + let parameters = parameter_list(*algorithm).first()?; + Some( + theoretical_guarantee(summary_type, parameters, statistic, 1.0)? + .spec + .accuracy_metric, + ) +} + +/// A heap variant's heap size. +fn heap_size(parameters: &SketchParams) -> Option { + match parameters { + SketchParams::CmsWithHeap { heap_size, .. } + | SketchParams::CountSketchWithHeap { heap_size, .. } => Some(*heap_size), + _ => None, + } +} + +/// The metric of `producer`'s error when it is a readout of a sketch state +/// ([`readout_metric`]); `None` for any other candidate. +fn sketch_readout_metric(memo: &Memo, producer: CandidateForMemoGroupId) -> Option { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query, + }) = &memo.candidate(producer).expression.operator + else { + return None; + }; + readout_metric( + summary_type_of_operator(&memo.groups()[summary_input.0 as usize].expression)?, + query, + ) } diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs deleted file mode 100644 index d63a81c3e..000000000 --- a/crates/accuracy/src/propagation.rs +++ /dev/null @@ -1,1640 +0,0 @@ -//! Guarantee propagation: the theoretical guarantee of every node of a -//! composed DAG, composed with the built-in analytical rules -//! (`crate::composition::propagate`) through operators, and the helpers -//! Pass 2 and plan selection (Stage 3) use to read, bind and recompose -//! those guarantees and to resize the summaries they depend on. -use std::collections::{HashMap, HashSet}; -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction, Source}; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::guarantee::{ - AccuracyGuarantee, AccuracyMetric, BoundExpr, CompositionOperator, GuaranteeSource, - ProbabilityExpr, -}; -use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; -use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::{DataType, Schema}; -use asap_types::ir::schema::{ - ExactKind, FieldDataType, MergedError, NonNegativeWeightProof, SketchAlgorithm, - SketchStatistic, SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, -}; -use asap_types::ir::{ - ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, ProjectItem, QueryRoot, ScalarExpr, - SchemaDerivationError, -}; - -use crate::composition::{ - additive_as_absolute, propagate, raw_data_samples_sign_statistic, StreamNorm, GROUP_ROW_COUNT, - PRODUCT_OPERAND_DOMAINS, QUOTIENT_OPERAND_DOMAINS, -}; - -use crate::function_rules::approximate_input_composition; -use crate::PropagationStats; - -/// A rebuilt node per original node, keyed by the original's address. -type NodeToRewrittenNodeCache = HashMap<*const OperatorNode, Rc>; - -/// The metric whose raw data samples, or plain sums of them, are every value -/// `node` outputs, or `None`. Only a time-series scan, a time range over it, -/// an `offset`-only time shift (it moves the window, not the samples) and a -/// plain `sum` keep that; any other operator (rate and arithmetic included) -/// ends it. The metric's type is not read here: whether its -/// samples are non-negative is plan selection's question -/// ([`WeightDomain::RawDataSamplesOfMetric`]). -pub fn raw_data_samples_of_metric(node: &OperatorNode) -> Option { - match node.non_asap()? { - NonASAPOp::Scan { - source: Source::TimeSeries { metric }, - .. - } => Some(metric.clone()), - NonASAPOp::TimeRange { child, .. } => raw_data_samples_of_metric(child), - NonASAPOp::TimeShift { shift, child } if shift.at.is_none() => { - raw_data_samples_of_metric(child) - } - NonASAPOp::Aggregate { - measures, child, .. - } if matches!(measures.as_slice(), [AggIntent::Sum { col: None }]) => { - raw_data_samples_of_metric(child) - } - _ => None, - } -} - -/// The approximate inputs a query's value composes: every sketch summary -/// state reachable from `root`, each once, when the value carries a -/// guarantee that is not exact ([`root_guarantee`]), including compositions -/// conditional on source-sign facts supplied later. Empty when neither the -/// current nor conditional contract composes the inputs, or the result is -/// exact. Pass 1 sizes each for its share of failure probability too. -pub fn approximate_inputs(root: &QueryRoot) -> Vec> { - // Sizing may depend on a source-domain proof supplied only at Stage 3. - // Count the inputs of that conditional composition too, without attaching - // the hypothetical proof to the candidate or accepting its accuracy. - let guarantee = root_guarantee(root).or_else(|| { - bind_guarantees(root, &|guarantee| { - guarantee.bind(&|statistic| { - crate::composition::parse_raw_data_samples_sign_statistic(statistic).map(|_| 1.0) - }) - }) - }); - if guarantee.is_none_or(|g| g.is_exact()) { - return Vec::new(); - } - let operators = match root { - QueryRoot::Operator(node) => vec![node], - QueryRoot::Scalar(expr) => expr.operator_refs(), - }; - let mut seen = HashSet::new(); - operators - .into_iter() - .flat_map(OperatorNode::reachable) - .filter(|node| { - matches!( - node.asap(), - Some( - ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(..), - .. - } | ASAPOp::SummaryTumblingWindowChain { - summary_type: FieldDataType::Sketch(..), - .. - } - ) - ) - }) - .filter(|node| seen.insert(Rc::as_ptr(node))) - .collect() -} - -/// The theoretical guarantee of a query's output, as composed: the root -/// node's own, or a scalar root's composed over the operators it reads -/// (`scalar_guarantee`). -pub fn root_guarantee(root: &QueryRoot) -> Option { - match root { - QueryRoot::Operator(node) => node.guarantee.clone(), - QueryRoot::Scalar(expr) => scalar_guarantee(expr, &|_| None), - } -} - -/// Bind known statistics in existing contracts, then recover compositions -/// that previously required constant child bounds. Physical state contracts, -/// schemas and sharing are preserved. This does not -/// infer facts: `bind` must leave unsupported statistics unknown. -pub fn bind_guarantees( - root: &QueryRoot, - bind: &dyn Fn(&AccuracyGuarantee) -> AccuracyGuarantee, -) -> Option { - fn walk( - node: &Rc, - bind: &dyn Fn(&AccuracyGuarantee) -> AccuracyGuarantee, - cache: &mut NodeToRewrittenNodeCache, - ) -> Rc { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return done.clone(); - } - let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); - rebuilt.guarantee = node - .guarantee - .as_ref() - .map(bind) - .or_else(|| composed_guarantee(&rebuilt).as_ref().map(bind)); - let rebuilt = Rc::new(rebuilt); - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - rebuilt - } - let mut cache = HashMap::new(); - let bound = match root { - QueryRoot::Operator(node) => QueryRoot::Operator(walk(node, bind, &mut cache)), - QueryRoot::Scalar(expr) => { - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| walk(node, bind, &mut cache))) - } - }; - root_guarantee(&bound) -} - -/// The one source a summary state counts the distinct items of its own -/// rows of, as `source_distinct_guarantee` reads it: the source whose -/// distinct bound gives its failure probability. `None` for a state over -/// computed rows or several sources. -pub fn distinct_bound_source(state: &OperatorNode) -> Option { - let Some(ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. }) = - state.asap() - else { - return None; - }; - source_scan(child)?; - source(child) -} - -/// `roots` with every node's guarantee composed again -/// ([`composed_guarantee`]), children first; a sub-DAG shared between roots -/// stays shared. Pass 2 calls it after re-sizing a shared summary, so each -/// consumer's guarantee is the shared state's. -pub fn recompose_guarantees(roots: &[QueryRoot]) -> Result, SchemaDerivationError> { - let mut cache = HashMap::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(with_guarantees(node, &mut cache)?) - } - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - with_guarantees(node, &mut cache)?; - } - QueryRoot::Scalar( - expr.map_operator_refs(&mut |node| cache[&Rc::as_ptr(node)].clone()), - ) - } - }) - }) - .collect() -} - -/// `node` and everything below it, each node with its [`composed_guarantee`]. -pub fn with_guarantees( - node: &Rc, - cache: &mut NodeToRewrittenNodeCache, -) -> Result, SchemaDerivationError> { - if let Some(done) = cache.get(&Rc::as_ptr(node)) { - return Ok(done.clone()); - } - for child in node.children() { - with_guarantees(child, cache)?; - } - let mut rebuilt = node.with_new_children(|child| cache[&Rc::as_ptr(child)].clone())?; - rebuilt.guarantee = composed_guarantee(&rebuilt); - let rebuilt = Rc::new(rebuilt); - cache.insert(Rc::as_ptr(node), rebuilt.clone()); - Ok(rebuilt) -} - -/// An extremum over at most one row per group is an identity on its value: -/// an aggregate whose measures are all `MIN`/`MAX` over a (possibly -/// filtered) grouped estimate, where every column of a unique key of the -/// estimate is a group key or pinned to one value by the filter -/// (Pass 2's finer-aggregate same-level read). Unlike a general -/// aggregate over rank estimates, this does not change the population to -/// which the input's error bound refers. -fn selected_group_guarantee(node: &OperatorNode) -> Option { - let NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - filters, - having: None, - child, - .. - } = node.non_asap()? - else { - return None; - }; - if by.is_without() - || !filters.iter().all(Option::is_none) - || measures.is_empty() - || !measures.iter().all(|measure| { - matches!( - measure, - AggIntent::Min { col: Some(_) } | AggIntent::Max { col: Some(_) } - ) - }) - { - return None; - } - let (pred, grouped) = match child.non_asap()? { - NonASAPOp::Filter { pred, child } => (Some(&pred.0), child), - _ => (None, child), - }; - let pinned = pred.map_or_else(Vec::new, pinned_columns); - // The filter may read only key columns: a predicate on an estimate - // would decide by an approximate value which groups are kept. - let reads = pred.map_or_else(Vec::new, ScalarExpr::columns_referenced); - if !grouped.schema.unique_keys.iter().any(|key| { - key.iter() - .all(|column| by.keys().contains(column) || pinned.contains(column)) - && reads.iter().all(|column| key.contains(column)) - }) { - return None; - } - let mut source = grouped; - while let Some(NonASAPOp::Project { cols, child, .. }) = source.non_asap() { - if !cols - .iter() - .enumerate() - .all(|(index, item)| matches!(item.expr, ScalarExpr::Column(column) if column == index)) - { - return None; - } - source = child; - } - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = source.asap() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - reduction: Reduction::Reduce(keys), - child: raw, - .. - }) = summary_input.asap() - else { - return None; - }; - if keys.is_without() - || !raw - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - { - return None; - } - grouped.guarantee.clone() -} - -/// The theoretical guarantee of a composed node, given its children's, -/// composed with the built-in analytical rules (`propagate`); -/// Stage 1 reads no data statistics or deployment evidence. Exact over exact -/// inputs; a summary estimate has its sized family's bound when -/// `estimate_has_bound`, composed with its input's when that is -/// approximate; an exact aggregation or accumulator over approximate values -/// uses its function's rule (`approximate_input_composition`); arithmetic composes as -/// `arithmetic_guarantee`. A rule that needs a statistic leaves it -/// symbolic (`BoundExpr::Unknown`), and plan selection (Stage 3) binds it. -/// `None` where no rule applies: a comparison, a value `HAVING` or any other -/// predicate over an approximate value, a rank error through any operator (a -/// rank interval counts as none in the MVP). A projection of columns only -/// keeps its child's guarantee. A summary state carries none itself. -/// -/// A guarantee holds per output point (per series or group), with its -/// failure probability per point ([`AccuracyScope::PerValue`]). Plan -/// selection states it over a whole result of `G` points as `G·δ`. -/// -/// [`AccuracyScope::PerValue`]: asap_types::ir::properties::AccuracyScope::PerValue -pub fn composed_guarantee(node: &OperatorNode) -> Option { - if let Some(guarantee) = selected_group_guarantee(node) { - return Some(guarantee); - } - if let Some(guarantee) = rolled_up_guarantee(node) { - return guarantee; - } - if let Some(guarantee) = merged_tumbling_windows_guarantee(node) { - return guarantee; - } - let exact = |node: &OperatorNode| { - node.guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - }; - // A summary state's input rows and any subquery in its filter. - let exact_inputs = |state: &OperatorNode| state.children().into_iter().all(|c| exact(c)); - // Over approximate input rows, a summary's filter must not decide which - // rows count by an estimate: it may read no input column (guarantees are - // per node, so any column may carry the estimate; column-level tracking - // is #689), and its subqueries must be exact. - let exact_filter = |state: &OperatorNode| { - let Some(ASAPOp::SummaryAgg { filter, .. }) = state.asap() else { - return false; - }; - filter - .as_ref() - .is_none_or(|filter| !reads_columns(&filter.0)) - && state.children().into_iter().skip(1).all(|c| exact(c)) - }; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(..), - .. - }) if exact_inputs(child) => Some(AccuracyGuarantee::exact( - "exact accumulator over exact input", - )), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(kind, _), - child: input, - reduction, - .. - }) if exact_filter(child) && !groups_by_estimate(reduction, input) => { - let stats = fold_stats(reduction, input)?; - let input = operand_guarantee(input)?; - let op = match kind { - // Counts rows, not values. - ExactKind::Count => return Some(AccuracyGuarantee::exact("count of rows")), - ExactKind::Sum => CompositionOperator::ExactSum, - ExactKind::Min | ExactKind::Max => CompositionOperator::ExactExtremum, - ExactKind::Rate => CompositionOperator::CounterRate, - ExactKind::IRate => CompositionOperator::InstantCounterRate, - ExactKind::Increase => CompositionOperator::CounterIncrease, - }; - propagate(&op, &[input], None, &stats).ok() - } - _ => None, - }, - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => match summary_input.asap() { - Some(ASAPOp::SummaryAgg { - summary_type: family, - input: update, - child: input, - reduction, - .. - }) if estimate_has_bound(family, update, query) - || estimate_bound_over_raw_data_samples(family, update, query).is_some() => - { - let local = match estimate_bound_over_raw_data_samples(family, update, query) { - Some(gated) => gated, - None => source_distinct_guarantee(family, query, input) - .or_else(|| { - todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters") - })?, - }; - let local = interpolation_guarantee(family, update, query, input, local)?; - if exact_inputs(summary_input) { - Some(local) - } else if exact_filter(summary_input) && !groups_by_estimate(reduction, input) { - compose( - &CompositionOperator::ApproximateAggregate, - vec![input.guarantee.clone()?], - Some(&local), - ) - } else { - None - } - } - _ => None, - }, - Operator::ASAP(_) => None, - // Renaming or reordering columns keeps the child's guarantee. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - child.guarantee.clone() - } - // Each step re-evaluates the child: every sample keeps its guarantee. - Operator::NonASAP(NonASAPOp::PromqlSubquery { child, .. }) => child.guarantee.clone(), - Operator::NonASAP(_) if node.children().into_iter().all(|child| exact(child)) => { - Some(AccuracyGuarantee::exact("exact over exact inputs")) - } - // Over a join, the child's sample value may be an exact input's - // column, so each column's own guarantee is read instead. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) if derives_from_join(child) => { - relational_project_guarantee(cols, child) - } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => project_guarantee(cols, child), - Operator::NonASAP(NonASAPOp::Join { .. }) => join_guarantee(node), - // A measure filter or `HAVING` over approximate values changes - // which rows or groups count: no rule. - Operator::NonASAP(NonASAPOp::Aggregate { - measures, - filters, - having: None, - reduction, - child, - .. - }) if filters.iter().all(Option::is_none) && !groups_by_estimate(reduction, child) => { - match measures.as_slice() { - [intent] => propagate( - &approximate_input_composition(intent)?, - &[operand_guarantee(child)?], - None, - &fold_stats(reduction, child)?, - ) - .ok(), - _ => None, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { - operator: - BinaryOperator { - kind: BinaryOpKind::Arithmetic(op), - .. - }, - lhs, - rhs, - .. - }) => arithmetic_guarantee( - op.clone(), - (operand_guarantee(lhs), None), - (operand_guarantee(rhs), None), - Rc::ptr_eq(lhs, rhs), - ), - Operator::NonASAP(_) => None, - } -} - -/// The guarantee of an estimate or finalized value read from a rollup (a -/// regrouping `SummaryMerge` of one input): the same read of the input -/// states, when the family's merged error for that read is exact or the -/// same as one build's over the union (the family-capability table). The -/// groups of one build partition its rows, so a rolled-up state is a merge -/// of disjoint parts, and a family whose bound holds for one build over -/// the union keeps it: KMV, HLL and DDSketch merge into the state a direct -/// coarse build would have, KLL into a state with the same rank bound, and -/// linear sketches into the direct build's counters, whose error is -/// relative to the coarse group's total. Each guarantee holds per output -/// point, so the number of groups does not change it. None when the merged -/// readout is approximate (heap top-k, entropy) or the family has no merge. -/// `None` for any node that does not read a rollup. -fn rolled_up_guarantee(node: &OperatorNode) -> Option> { - let state = match node.asap()? { - ASAPOp::SummaryEstimate { summary_input, .. } => summary_input, - ASAPOp::FinalizeExactAccumulator { child } => child, - _ => return None, - }; - let Some(ASAPOp::SummaryMerge { - children, - group_by: Some(_), - .. - }) = state.asap() - else { - return None; - }; - let [input] = children.as_slice() else { - return Some(None); - }; - // Kept whole, a selected state keeps the guarantee it was built with. - let input = - asap_types::ir::operator::asap::selected_from(input).unwrap_or_else(|| Rc::clone(input)); - let family = state.asap()?.produced_state()?; - let merged = match node.asap()? { - ASAPOp::SummaryEstimate { query, .. } => { - SummaryTypeCapability::of(family).merged_error(query) - } - // An exact accumulator merged across groups is exact. - _ => SummaryTypeCapability::merges_groups(family).then_some(MergedError::Exact), - }; - if !matches!( - merged, - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) { - return Some(None); - } - let consumer = OperatorNode::with_schema( - node.operator.map_children(|child| { - if Rc::ptr_eq(child, state) { - Rc::clone(&input) - } else { - Rc::clone(child) - } - }), - node.schema.clone(), - ); - Some(composed_guarantee(&consumer)) -} - -/// The guarantee of an estimate or finalized value read from merged -/// tumbling windows (a state whose [`OperatorNode::summary_build`] is a -/// chain): the guarantee of the same read of one build over the time range -/// the merged windows cover when the -/// summary type's merged error for that read is exact or the same as that build's -/// (the summary-type capability table); none when it is approximate (evidence -/// only, for plan selection to judge) or the family has no merge. `None` for -/// any node that does not read merged tumbling windows. -fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option> { - let (state, query) = match node.asap()? { - ASAPOp::SummaryEstimate { - summary_input, - query, - } => (summary_input, Some(query)), - ASAPOp::FinalizeExactAccumulator { child } => (child, None), - _ => return None, - }; - let ASAPOp::SummaryTumblingWindowChain { summary_type, .. } = state.summary_build()? else { - return None; - }; - let capability = SummaryTypeCapability::of(&summary_type); - let merged = match query { - Some(query) => capability.merged_error(query), - // An exact accumulator's merged state is exact. - None => capability.merge_inputs.map(|_| MergedError::Exact), - }; - if !matches!( - merged, - Some(MergedError::Exact | MergedError::AsIfBuiltOverUnion) - ) { - return Some(None); - } - todo!( - "#1207 PR 4: the guarantee of a read of merged tumbling windows, as of one build over their window, over memo groups" - ) -} - -/// The guarantee of a distinct count of one instance per group over a -/// source's own rows, whose failure probability is left symbolic in the -/// source's distinct bound, which Stage 1 does not read: -/// - classic HLL's ([`hll::classic_guarantee`]); -/// - UnivMon's, sized for a distinct count ([`univmon::distinct_guarantee`]). -/// -/// `None` for any other summary, or one over computed rows, whose distinct -/// items the source's bound does not bound. -pub(crate) fn source_distinct_guarantee( - family: &FieldDataType, - query: &SketchStatistic, - input: &Rc, -) -> Option { - let FieldDataType::Sketch(algorithm) = family else { - return None; - }; - if *query != SketchStatistic::Cardinality { - return None; - } - source_scan(input)?; - match algorithm { - SketchAlgorithm::Hll | SketchAlgorithm::UnivMon => { - todo!( - "#1207 PR 4: the distinct-count guarantee of θ from LogicalProperties::parameters" - ) - } - _ => None, - } -} - -/// The rows one group of an exact fold over approximate `input` counts. -/// Across PromQL series (`Reduce`) they are at most the input series, an -/// input-row-count statistic left symbolic for plan selection; across SQL -/// groups they are a group-row-count statistic nothing binds; over time -/// (`PerEntity`) they are one series' samples in the window: the steps of a -/// subquery with an explicit resolution, `⌈range / resolution⌉`. `None` -/// when that count is not in the query. -pub(crate) fn fold_stats( - reduction: &Reduction, - input: &Rc, -) -> Option { - let sign_statistic = - estimates_raw_data_samples_of(input).map(|metric| raw_data_samples_sign_statistic(&metric)); - let stats = match reduction { - // The series count of the sources read bounds a PromQL fold's rows - // (series), not the rows of a SQL group. - Reduction::Reduce(_) if reads_table(input) => Some(PropagationStats { - row_count_statistic: Some(GROUP_ROW_COUNT), - ..PropagationStats::default() - }), - Reduction::Reduce(_) => Some(PropagationStats::default()), - Reduction::PerEntity => match input.non_asap() { - Some(NonASAPOp::PromqlSubquery { - range, - resolution: Some(resolution), - .. - }) if !resolution.is_zero() => Some(PropagationStats { - input_row_count: Some(range.as_nanos().div_ceil(resolution.as_nanos()) as u64), - ..PropagationStats::default() - }), - _ => None, - }, - }?; - Some(PropagationStats { - sign_statistic, - ..stats - }) -} - -/// The metric whose raw data samples, or plain sums of them, a composed node's -/// values are or estimate, or `None`: [`raw_data_samples_of_metric`]'s strict rule -/// over a composed candidate. An exact sum accumulator, a projection of -/// columns and a -/// heap estimate whose update weights carry the -/// [`WeightDomain::RawDataSamplesOfMetric`] requirement keep it, and so does a -/// quantile estimate of those values, built over the raw window or merged -/// from its tumbling windows: it is one of them, or between two of them, so it -/// shares their sign. Anything else (rate, arithmetic, other -/// functions) ends it. Only the sign of the values is read from it. -pub(crate) fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option { - match &node.operator { - Operator::NonASAP(NonASAPOp::TimeRange { child, .. }) => { - estimates_raw_data_samples_of(child) - } - Operator::NonASAP(NonASAPOp::TimeShift { child, shift }) if shift.at.is_none() => { - estimates_raw_data_samples_of(child) - } - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) => match child.asap()? { - ASAPOp::SummaryAgg { - summary_type: FieldDataType::ExactAggregate(ExactKind::Sum, _), - child, - filter: None, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - }, - // Only a φ in [0, 1] picks one of the values; outside it the answer - // is -Inf or +Inf, whatever their sign. Merged tumbling windows of one - // definition are read through their `SummaryTumblingWindowChain`: - // a quantile of the merged state is still one of the tumbling windows' - // samples, or between two of them. - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: SketchStatistic::Quantile { q }, - }) if (0.0..=1.0).contains(q) => { - let build = match summary_input.summary_build()? { - chain @ ASAPOp::SummaryTumblingWindowChain { .. } => chain, - _ => summary_input.asap()?.clone(), - }; - match &build { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } - | ASAPOp::SummaryTumblingWindowChain { - input: - SummaryUpdate { - item: None, - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - .. - }, - filter: None, - child, - .. - } => estimates_raw_data_samples_of(child), - _ => None, - } - } - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - match summary_input.asap()? { - ASAPOp::SummaryAgg { - input: - SummaryUpdate { - weight_domain: WeightDomain::RawDataSamplesOfMetric { metric }, - .. - }, - filter: None, - .. - } => Some(metric.clone()), - _ => None, - } - } - // Renaming, reordering or dropping columns keeps every value. - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) - if cols - .iter() - .all(|item| matches!(item.expr, ScalarExpr::Column(_))) => - { - estimates_raw_data_samples_of(child) - } - _ => raw_data_samples_of_metric(node), - } -} - -/// Whether `node` reads a SQL table. -fn reads_table(node: &Rc) -> bool { - OperatorNode::reachable(node).iter().any(|node| { - matches!( - node.non_asap(), - Some(NonASAPOp::Scan { - source: Source::Table { .. }, - .. - }) - ) - }) -} - -/// Whether a grouping by `reduction` over an approximate `input` may decide -/// a row's group by an estimate: a grouping column other than a label (a -/// string) or the time index. Guarantees are per node, so any such column -/// may carry the estimate (column-level tracking is #689); a PromQL `by` -/// or `without` groups by labels only. `false` over exact input. -fn groups_by_estimate(reduction: &Reduction, input: &OperatorNode) -> bool { - let Some(by) = reduction.group_keys() else { - return false; - }; - if input - .guarantee - .as_ref() - .is_some_and(AccuracyGuarantee::is_exact) - { - return false; - } - let schema = &input.schema; - let columns: Vec = if by.is_without() { - let value = resolve_column_ref(&ColumnRef::SampleValue, schema).ok(); - (0..schema.fields.len()) - .filter(|i| !by.keys().contains(i) && Some(*i) != value) - .collect() - } else { - by.keys().to_vec() - }; - columns.into_iter().any(|i| { - Some(i) != schema.time_index - && !matches!( - schema.fields.get(i).map(|f| &f.dtype), - Some(FieldDataType::Plain(DataType::Utf8)) - ) - }) -} - -/// Whether `expr` reads any input column. -fn reads_columns(expr: &ScalarExpr) -> bool { - let mut reads = false; - expr.map_columns(&mut |column| { - reads = true; - column - }); - reads -} - -/// The built-in rule for `op`, with every statistic unknown (Stage 1 reads -/// none); `None` when no rule applies. -fn compose( - op: &CompositionOperator, - inputs: Vec, - local: Option<&AccuracyGuarantee>, -) -> Option { - propagate(op, &inputs, local, &PropagationStats::default()).ok() -} - -/// A projection that computes over its child's sample value: the guarantee -/// of the one item that reads it ([`project_column_guarantee`]). `None` when -/// no single item reads the value. -fn project_guarantee(cols: &[ProjectItem], child: &Rc) -> Option { - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; - let reads_value = |expr: &ScalarExpr| { - let mut reads = false; - expr.map_columns(&mut |column| { - reads |= column == value; - column - }); - reads - }; - let [item] = <[_; 1]>::try_from( - cols.iter() - .filter(|item| reads_value(&item.expr)) - .collect::>(), - ) - .ok()?; - scalar_guarantee(&item.expr, &|column| { - project_column_guarantee(child, column) - }) -} - -/// Whether `node` is a join, or reads one only through operators that keep -/// its columns (projections, filters, sorts, limits, deduplication). The -/// columns of such a node come from different inputs, so its sample value -/// says nothing about where the error lies. -fn derives_from_join(node: &OperatorNode) -> bool { - match node.non_asap() { - Some(NonASAPOp::Join { .. }) => true, - Some( - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. } - | NonASAPOp::Dedup { child, .. }, - ) => derives_from_join(child), - _ => false, - } -} - -/// The guarantee of each output column of `node`, where it is known: every -/// column of an exact node is exact; a join beside exact inputs -/// ([`join_guarantee`]) gives the exact inputs' columns exact and the -/// approximate input's columns its guarantee; a projection of columns maps -/// its child's. `None` anywhere else, a computed projection included. -/// Guarantees are per node, so every column of an approximate input counts -/// as approximate; column-level tracking is #689. -/// -/// This assumes every approximate node of Stage 1 has an exact row set: -/// its guarantee bounds each row's values, never which rows exist (a -/// top-k estimate, whose rows are approximate, has no bound; -/// `estimate_has_bound`). A guarantee on row membership would need these -/// rules revisited. -fn column_guarantees(node: &OperatorNode) -> Option> { - let width = node.schema.fields.len(); - if let Some(guarantee) = node.guarantee.as_ref().filter(|g| g.is_exact()) { - return Some(vec![guarantee.clone(); width]); - } - match &node.operator { - Operator::NonASAP(NonASAPOp::Join { - kind: - JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - left, - right, - .. - }) => { - let input = join_input_guarantees(left, right)?; - (input.len() == width).then_some(input) - } - Operator::NonASAP(NonASAPOp::Project { cols, child, .. }) => { - let child = column_guarantees(child)?; - cols.iter() - .map(|item| match item.expr { - ScalarExpr::Column(id) => child.get(id).cloned(), - _ => None, - }) - .collect() - } - _ => None, - } -} - -/// The guarantees of a join's input columns (`left ++ right`) when at most -/// one input is approximate and the others are exact: the exact input's -/// columns exact, the approximate input's columns its guarantee. -fn join_input_guarantees( - left: &OperatorNode, - right: &OperatorNode, -) -> Option> { - let exact = |node: &OperatorNode| node.guarantee.as_ref().is_some_and(|g| g.is_exact()); - if !exact(left) && !exact(right) { - return None; - } - let side = |node: &OperatorNode| { - let guarantee = node.guarantee.clone()?; - Some(vec![guarantee; node.schema.fields.len()]) - }; - let mut columns = side(left)?; - columns.extend(side(right)?); - Some(columns) -} - -/// A join whose inputs are exact but one keeps that input's guarantee when -/// its predicate reads only exact columns (a cross join, or a key equality -/// on the exact inputs' columns): which rows match, and how often, then -/// does not depend on an estimate, and each output row carries one row of -/// the approximate input unchanged. Outer joins pad on the same exact -/// decision. A semi- or anti-join, a key on an approximate input's column -/// (per-node guarantees count all its columns approximate; #689), or two -/// approximate inputs have none. An exact join is composed before this. -/// Like [`column_guarantees`], this assumes the approximate input's row set -/// is exact: only its values carry error. -fn join_guarantee(node: &OperatorNode) -> Option { - let Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner | JoinKind::Left | JoinKind::Right | JoinKind::Full | JoinKind::Cross, - pred, - left, - right, - }) = &node.operator - else { - return None; - }; - let columns = join_input_guarantees(left, right)?; - let pred_exact = - scalar_guarantee(&pred.0, &|id| columns.get(id).cloned()).is_some_and(|g| g.is_exact()); - if !pred_exact { - return None; - } - [left, right] - .into_iter() - .find_map(|input| input.guarantee.clone().filter(|g| !g.is_exact())) -} - -/// The guarantee of a projection over relational columns -/// ([`column_guarantees`]): each output column's as [`scalar_guarantee`] -/// (a `CASE` as [`case_guarantee`]), and the node's the weakest of them. -/// `None` when the child's column guarantees are unknown or any output -/// column has none. -fn relational_project_guarantee( - cols: &[ProjectItem], - child: &Rc, -) -> Option { - let columns = column_guarantees(child)?; - let column = |id: usize| columns.get(id).cloned(); - weakest( - cols.iter() - .map(|item| scalar_guarantee(&item.expr, &column)) - .collect::>>()?, - ) -} - -/// The weakest of `guarantees`: exact when all are, the approximate one -/// when there is one (or all approximate ones are equal), and otherwise -/// as an extremum of them ([`lipschitz_guarantee`]). -fn weakest(guarantees: Vec) -> Option { - let mut approximate: Vec = Vec::new(); - for guarantee in guarantees.into_iter().filter(|g| !g.is_exact()) { - if !approximate.contains(&guarantee) { - approximate.push(guarantee); - } - } - match approximate.len() { - 0 => Some(AccuracyGuarantee::exact("exact over exact operands")), - 1 => approximate.pop(), - _ => lipschitz_guarantee(approximate), - } -} - -/// A `CASE` whose operand and conditions are exact picks its branch by an -/// exact decision, so it keeps the weakest branch's guarantee (a literal or -/// NULL branch, or a missing `ELSE`, is exact). A condition over an -/// approximate value has none. -fn case_guarantee( - operand: Option<&ScalarExpr>, - branches: &[(ScalarExpr, ScalarExpr)], - else_expr: Option<&ScalarExpr>, - column: &dyn Fn(usize) -> Option, -) -> Option { - let exact = |expr: &ScalarExpr| scalar_guarantee(expr, column).is_some_and(|g| g.is_exact()); - if !operand.is_none_or(exact) || !branches.iter().all(|(when, _)| exact(when)) { - return None; - } - weakest( - branches - .iter() - .map(|(_, then)| then) - .chain(else_expr) - .map(|expr| scalar_guarantee(expr, column)) - .collect::>>()?, - ) -} - -/// The guarantee of `column` of a projection's `child`: over a join, the -/// column's own (`column_guarantees`); otherwise the child's sample -/// value has the child's as an operand ([`operand_guarantee`]) and other -/// columns (labels, timestamps) are exact. `None` when the child has no -/// sample value or the column's guarantee is unknown. -pub fn project_column_guarantee( - child: &Rc, - column: usize, -) -> Option { - // Over a join, each column's own guarantee ([`column_guarantees`]). - if derives_from_join(child) { - return column_guarantees(child)?.get(column).cloned(); - } - let value = resolve_column_ref(&ColumnRef::SampleValue, &child.schema).ok()?; - if column == value { - operand_guarantee(child) - } else { - Some(AccuracyGuarantee::exact( - "column other than the sample value", - )) - } -} - -/// The guarantee of a scalar expression, given each column's: `scalar(v)` -/// and a SQL scalar subquery have their relation's, a literal or the -/// evaluation time is exact, arithmetic composes as -/// [`arithmetic_guarantee`], and `clamp`/`clamp_min`/`clamp_max`/`abs` as -/// [`lipschitz_guarantee`]. Any other expression (a comparison, a rounding, -/// a non-linear function) is exact over exact operands and has none -/// otherwise. -pub(crate) fn scalar_guarantee( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - let literal = |expr: &ScalarExpr| match expr { - ScalarExpr::Literal(ScalarValue::Float64(value)) => Some(*value), - ScalarExpr::Literal(ScalarValue::Int64(value)) => Some(*value as f64), - _ => None, - }; - let operand = |expr: &ScalarExpr| (operand_of(expr, column), literal(expr)); - match expr { - ScalarExpr::Column(id) => column(*id), - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - node.guarantee.clone() - } - ScalarExpr::Literal(_) | ScalarExpr::CurrentTimestamp | ScalarExpr::EvalTimestamp => { - Some(AccuracyGuarantee::exact("constant")) - } - ScalarExpr::Negative { expr, .. } => { - lipschitz_guarantee(vec![scalar_guarantee(expr, column)?]) - } - ScalarExpr::Case { - operand, - branches, - else_expr, - } => case_guarantee(operand.as_deref(), branches, else_expr.as_deref(), column), - ScalarExpr::Arithmetic { - op, left, right, .. - } => arithmetic_guarantee( - op.clone(), - operand(left), - operand(right), - same_value(left, right), - ), - ScalarExpr::FunctionCall { name, args } if is_monotone_lipschitz(name) => { - lipschitz_guarantee( - args.iter() - .map(|arg| operand_of(arg, column)) - .collect::>()?, - ) - } - _ => { - let mut columns = Vec::new(); - expr.map_columns(&mut |id| { - columns.push(id); - id - }); - let exact = |g: Option| g.is_some_and(|g| g.is_exact()); - (columns.into_iter().all(|id| exact(column(id))) - && expr - .operator_refs() - .into_iter() - .all(|node| exact(node.guarantee.clone()))) - .then(|| AccuracyGuarantee::exact("exact over exact operands")) - } - } -} - -/// The guarantee of a scalar operand as an input to arithmetic or a -/// function: a planned operator's as [`operand_guarantee`], anything else -/// as `scalar_guarantee`. -pub fn operand_of( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - match expr { - ScalarExpr::PromqlScalarFromVector(node) | ScalarExpr::ScalarSubquery(node) => { - operand_guarantee(node) - } - _ => scalar_guarantee(expr, column), - } -} - -/// `node`'s guarantee as an input to a computation over its values: a -/// Count-Min or Count Sketch bound restated as an absolute error over the -/// stream norm its summary's update weight allows (`stream_norm`, -/// `additive_as_absolute`). -pub fn operand_guarantee(node: &Rc) -> Option { - let guarantee = node.guarantee.clone()?; - Some(additive_as_absolute(guarantee, stream_norm(node).as_ref())) -} - -/// The stream norm that bounds the one Count-Min or Count Sketch summary -/// below `node`, by its update weight: the source's own norm when the -/// weight is the source's raw value column (the summary's input is the -/// source's scan, at most windowed or filtered); `|c|` times the source's -/// row count for a constant weight `c`. `None` for any other weight (a -/// computed value such as `m * 1000` or `rate(…)`), several summaries or -/// sources, or none. -pub(crate) fn stream_norm(node: &Rc) -> Option { - let summaries: Vec<_> = OperatorNode::reachable(node) - .into_iter() - .filter(|node| { - matches!( - node.asap(), - Some(ASAPOp::SummaryAgg { - summary_type: FieldDataType::Sketch(algorithm), - .. - }) if matches!( - algorithm, - SketchAlgorithm::Cms - | SketchAlgorithm::CmsWithHeap - | SketchAlgorithm::CountSketch - | SketchAlgorithm::CountSketchWithHeap - ) - ) - }) - .collect(); - let [summary] = summaries.as_slice() else { - return None; - }; - let Some(ASAPOp::SummaryAgg { - input: update, - child, - .. - }) = summary.asap() - else { - return None; - }; - let source = source(child)?; - match &update.weight { - SummaryInputExpr::Constant(weight) => Some(StreamNorm::ConstantWeight { - weight: *weight, - source, - }), - SummaryInputExpr::Column(column) if raw_value_column(child, column) => { - Some(StreamNorm::Source(source)) - } - _ => None, - } -} - -/// Whether `column` of `node` is its source's raw data sample value: `node` is -/// the source's scan, at most behind filters and windows that keep its -/// schema, and `column` resolves to the scan's sample value. -fn raw_value_column(node: &Rc, column: &ColumnRef) -> bool { - let Some(schema) = source_scan(node) else { - return false; - }; - let resolved = resolve_column_ref(column, schema).ok(); - resolved.is_some() && resolved == resolve_column_ref(&ColumnRef::SampleValue, schema).ok() -} - -/// The scan's schema when `node` is a source's scan, at most behind filters -/// and windows that keep its schema: its rows are the source's own. -fn source_scan(node: &Rc) -> Option<&Schema> { - let mut current = node; - loop { - if current.schema != node.schema { - return None; - } - match current.non_asap() { - Some(NonASAPOp::Filter { child, .. } | NonASAPOp::TimeRange { child, .. }) => { - current = child - } - Some(NonASAPOp::Scan { schema, .. }) => return Some(schema), - _ => return None, - } - } -} - -/// The one source (PromQL metric or SQL table) `node` reads; `None` when it -/// reads several or none. -fn source(node: &Rc) -> Option { - let sources: HashSet = OperatorNode::reachable(node) - .iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => Some(match source { - Source::TimeSeries { metric } => metric.clone(), - Source::Table { table_ref } => table_ref.clone(), - }), - _ => None, - }) - .collect(); - let mut sources = sources.into_iter(); - match (sources.next(), sources.next()) { - (Some(source), None) => Some(source), - _ => None, - } -} - -/// Whether two scalar operands are one value: the same column, or the same -/// planned operator read twice. -fn same_value(left: &ScalarExpr, right: &ScalarExpr) -> bool { - match (left, right) { - (ScalarExpr::Column(a), ScalarExpr::Column(b)) => a == b, - ( - ScalarExpr::PromqlScalarFromVector(a) | ScalarExpr::ScalarSubquery(a), - ScalarExpr::PromqlScalarFromVector(b) | ScalarExpr::ScalarSubquery(b), - ) => Rc::ptr_eq(a, b), - _ => false, - } -} - -/// Functions that are 1-Lipschitz and, for the relative-error rule, -/// monotone in each argument with `|f(x)|` scaling with `|x|`: `clamp` -/// (`min(max(x, lo), hi)`), `clamp_min`, `clamp_max` and `abs`. -fn is_monotone_lipschitz(name: &str) -> bool { - matches!( - name, - "promql_clamp" | "promql_clamp_min" | "promql_clamp_max" | "promql_abs" | "abs" - ) -} - -/// The guarantee of a 1-Lipschitz function of `args` (negation, `abs`, -/// `clamp`): its value is within the largest of its arguments' absolute -/// errors, by the union bound over them; likewise for relative errors, as -/// for an exact `max`/`min` (`ExactExtremum`, one value per argument). -/// Count-Min and Count Sketch bounds count as absolute errors. `None` for a -/// rank error or a mix of absolute and relative ones. -fn lipschitz_guarantee(args: Vec) -> Option { - let approximate: Vec<_> = args - .into_iter() - .filter(|g| !g.is_exact()) - .map(|g| additive_as_absolute(g, None)) - .collect(); - if approximate.is_empty() { - return Some(AccuracyGuarantee::exact( - "Lipschitz function of exact operands", - )); - } - propagate( - &CompositionOperator::ExactExtremum, - &approximate, - None, - &PropagationStats { - input_row_count: Some(1), - ..PropagationStats::default() - }, - ) - .ok() -} - -/// An arithmetic operand: its guarantee, and its value when it is a literal. -type Operand = (Option, Option); - -/// The guarantee of `left op right` (#669, MVP). Count-Min and Count Sketch -/// bounds count as absolute errors (normalized bound times stream norm). -/// - `x ± y`: absolute errors add (union bound on the failure -/// probabilities); `x ± c` (`c` exact) keeps an absolute error. -/// - `c · x`, `x / c` (`c` a nonzero literal): an absolute error scales by -/// `|c|` (or `1/|c|`); a relative error is kept, as it is by `x · y` or -/// `x / y` with `y` exact. -/// - `x · y` of two relative errors: relative `a + b + a·b`. -/// - `x / y` with `y` a relative error and `x` a relative error or exact: -/// relative `(a + b)/(1 − b)` (`a = 0` for an exact `x`), none when -/// `b ≥ 1`. -/// - `x · y` of two other estimates, `x / y` with any other approximate -/// `y`: a bound that needs enforced operand domains, left symbolic -/// ([`operand_domain_statistic`]) for plan selection to bind. -/// - Everything else (a relative error shifted by a constant or added to -/// another, a rank error, `%`, `^`, `atan2`) has none. -/// -/// `same_operand`: both sides are one estimate (the same planned operator), -/// which fails or holds as one event, so its failure probability counts -/// once. Two estimates count once each, even over one shared summary: the -/// built-in guarantees hold per estimate, not simultaneously. -fn arithmetic_guarantee( - op: ArithmeticOpKind, - left: Operand, - right: Operand, - same_operand: bool, -) -> Option { - let (l, r) = ( - additive_as_absolute(left.0?, None), - additive_as_absolute(right.0?, None), - ); - let once = same_operand.then(|| l.spec.failure_probability.clone()); - let statistic = operand_domain_statistic(&op, &l, &r); - let composed = match (op, l.is_exact(), r.is_exact()) { - (_, true, true) => return Some(AccuracyGuarantee::exact("arithmetic over exact operands")), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, true, false) => shifted(r), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, false, true) => shifted(l), - (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, false, false) => compose( - &CompositionOperator::Lipschitz { constant: 1.0 }, - vec![l], - Some(&r), - ), - (ArithmeticOpKind::Mul, true, false) => scaled(r, left.1.map(f64::abs)), - (ArithmeticOpKind::Mul, false, true) => scaled(l, right.1.map(f64::abs)), - (ArithmeticOpKind::Mul, false, false) if statistic.is_none() => relative_product(l, r), - (ArithmeticOpKind::Div, false, true) => match right.1 { - Some(0.0) => None, - c => scaled(l, c.map(|c| 1.0 / c.abs())), - }, - (ArithmeticOpKind::Div, _, false) if statistic.is_none() => propagate( - &CompositionOperator::ExactDivision, - &[l, r], - None, - &PropagationStats::default(), - ) - .ok(), - (ArithmeticOpKind::Mul | ArithmeticOpKind::Div, _, _) => symbolic(l, r, statistic?), - _ => None, - }?; - let mut composed = composed; - if let Some(failure_probability) = once { - composed.spec.failure_probability = failure_probability; - } - Some(composed) -} - -/// The statistic a product or quotient of `l` and `r` leaves symbolic on -/// its operands' domains, when it is one whose bound needs them: a product -/// of two estimates, other than of two relative errors -/// ([`PRODUCT_OPERAND_DOMAINS`]); a quotient with an approximate -/// denominator, other than of relative or exact over relative -/// (`QUOTIENT_OPERAND_DOMAINS`). `None` for any other operation (a -/// product or quotient of relative errors has a closed form), or when an -/// operand's error has no value form (a rank error). Plan selection -/// (Stage 3) finds the operations to bind with this same rule. -pub fn operand_domain_statistic( - op: &ArithmeticOpKind, - l: &AccuracyGuarantee, - r: &AccuracyGuarantee, -) -> Option<&'static str> { - let valued = |g: &AccuracyGuarantee| { - g.is_exact() - || matches!( - g.spec.accuracy_metric, - AccuracyMetric::ABSOLUTE | AccuracyMetric::Relative - ) - }; - let relative = - |g: &AccuracyGuarantee| g.is_exact() || g.spec.accuracy_metric == AccuracyMetric::Relative; - if !(valued(l) && valued(r)) { - return None; - } - match op { - ArithmeticOpKind::Mul - if !l.is_exact() - && !r.is_exact() - && !(l.spec.accuracy_metric == AccuracyMetric::Relative - && r.spec.accuracy_metric == AccuracyMetric::Relative) => - { - Some(PRODUCT_OPERAND_DOMAINS) - } - ArithmeticOpKind::Div if !r.is_exact() && !(relative(l) && relative(r)) => { - Some(QUOTIENT_OPERAND_DOMAINS) - } - _ => None, - } -} - -/// A product of two relative-error estimates: within relative -/// `(1 + a)(1 + b) − 1 = a + b + a·b`, whatever the signs. `None` for any -/// other pair. -fn relative_product(l: AccuracyGuarantee, r: AccuracyGuarantee) -> Option { - if l.spec.accuracy_metric != AccuracyMetric::Relative - || r.spec.accuracy_metric != AccuracyMetric::Relative - { - return None; - } - let bound = BoundExpr::Sum { - terms: vec![ - l.spec.error.clone(), - r.spec.error.clone(), - BoundExpr::Product { - factors: vec![l.spec.error.clone(), r.spec.error.clone()], - }, - ], - }; - Some(binary_guarantee( - AccuracyMetric::Relative, - bound, - l, - r, - None, - CompositionOperator::ExactProduct, - "relative_product_union_bound", - )) -} - -/// `x` times an exact factor (`Some(|c|)` when it is a literal): a -/// relative error is kept; an absolute error scales by `|c|`. -fn scaled(input: AccuracyGuarantee, factor: Option) -> Option { - match (input.spec.accuracy_metric, factor) { - (AccuracyMetric::Relative, _) => Some(input), - (_, Some(constant)) => compose( - &CompositionOperator::Lipschitz { constant }, - vec![input], - None, - ), - _ => None, - } -} - -/// `x` plus an exact value: an absolute error is kept. -fn shifted(input: AccuracyGuarantee) -> Option { - compose( - &CompositionOperator::Lipschitz { constant: 1.0 }, - vec![input], - None, - ) -} - -/// A product or quotient whose bound needs its operands' domains, which -/// Stage 1 does not read: the bound is the symbolic `statistic` -/// ([`operand_domain_statistic`]), absolute, with the union bound on the -/// failure probability. -fn symbolic( - l: AccuracyGuarantee, - r: AccuracyGuarantee, - statistic: &'static str, -) -> Option { - let (operator, rule) = match statistic { - QUOTIENT_OPERAND_DOMAINS => ( - CompositionOperator::ExactDivision, - "absolute_division_needs_operand_domains", - ), - _ => ( - CompositionOperator::ExactProduct, - "absolute_product_needs_operand_domains", - ), - }; - Some(binary_guarantee( - AccuracyMetric::ABSOLUTE, - BoundExpr::Unknown { - statistic: statistic.into(), - }, - l, - r, - Some(statistic), - operator, - rule, - )) -} - -/// A guarantee over two operands with the union bound on their failure -/// probabilities, and its provenance. -fn binary_guarantee( - metric: AccuracyMetric, - bound: BoundExpr, - l: AccuracyGuarantee, - r: AccuracyGuarantee, - unavailable: Option<&str>, - operator: CompositionOperator, - rule: &str, -) -> AccuracyGuarantee { - let failure_probability = ProbabilityExpr::UnionBound { - terms: vec![ - l.spec.failure_probability.clone(), - r.spec.failure_probability.clone(), - ], - }; - let provenance = [l, r] - .into_iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g), - }) - .chain( - unavailable.map(|statistic| GuaranteeSource::UnavailableStatistic { - statistic: statistic.into(), - }), - ) - .chain([GuaranteeSource::CompositionStep { - operator, - rule: rule.into(), - }]) - .collect(); - AccuracyGuarantee::new(metric, bound, failure_probability, provenance) -} - -/// Whether the default accuracy model's bound for `family` answers `query` -/// over `update`. A heap sketch's bound is per item frequency, not Top-K -/// membership; a Count-Min bound assumes non-negative weights. -pub(crate) fn estimate_has_bound( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> bool { - let count_min = matches!( - family, - FieldDataType::Sketch(algorithm) - if matches!(algorithm, SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap) - ); - !matches!(query, SketchStatistic::TopK { .. }) - && !(count_min && !matches!(update.weight_domain, WeightDomain::NonNegative { .. })) -} - -/// DDS quantiles use linear interpolation of neighbouring representatives. -/// Relative error survives that convex combination only without cancellation. -/// Endpoints select an extremum; interior quantiles need a sign proof. Keep a -/// raw metric's requirement symbolic so Stage 3 can bind declared source facts. -fn interpolation_guarantee( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, - input: &OperatorNode, - mut guarantee: AccuracyGuarantee, -) -> Option { - let interpolates = matches!(family, FieldDataType::Sketch(SketchAlgorithm::DDSketch)) - && matches!(query, SketchStatistic::Quantile { q } if *q > 0.0 && *q < 1.0); - if !interpolates || matches!(update.weight_domain, WeightDomain::NonNegative { .. }) { - return Some(guarantee); - } - let metric = match &update.weight_domain { - WeightDomain::RawDataSamplesOfMetric { metric } => metric.clone(), - _ if matches!( - update.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ) => - { - estimates_raw_data_samples_of(input)? - } - _ => return None, - }; - guarantee.spec.error = asap_types::ir::properties::BoundExpr::Product { - factors: vec![ - guarantee.spec.error, - asap_types::ir::properties::BoundExpr::Unknown { - statistic: raw_data_samples_sign_statistic(&metric), - }, - ], - }; - Some(guarantee) -} - -/// A Count-Min estimate whose weights carry the open requirement that they -/// are raw data samples of one metric ([`WeightDomain::RawDataSamplesOfMetric`]): its local -/// guarantee with the bound gated on that metric's sign -/// ([`raw_data_samples_sign_statistic`]), which plan selection binds to 1 when the -/// metric is declared a counter. `None` for any other estimate. -pub(crate) fn estimate_bound_over_raw_data_samples( - family: &FieldDataType, - update: &SummaryUpdate, - query: &SketchStatistic, -) -> Option { - let WeightDomain::RawDataSamplesOfMetric { metric } = &update.weight_domain else { - return None; - }; - let proven = SummaryUpdate { - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples, - }, - ..update.clone() - }; - if !estimate_has_bound(family, &proven, query) { - return None; - } - let _ = metric; - todo!("#1207 PR 4: the guarantee of θ from the candidate's LogicalProperties::parameters, its bound gated on the metric's sign") -} - -pub fn split_and(expr: &ScalarExpr, out: &mut Vec) { - match expr { - ScalarExpr::BoolAnd(items) => items.iter().for_each(|item| split_and(item, out)), - other => out.push(other.clone()), - } -} - -/// The columns `pred` pins to one literal value each: its conjuncts -/// `c = v` and `c IN (v)`. -fn pinned_columns(pred: &ScalarExpr) -> Vec { - let mut conjuncts = Vec::new(); - split_and(pred, &mut conjuncts); - conjuncts.iter().filter_map(pinned_column).collect() -} - -/// The column `conjunct` pins to one literal value (`c = v`, `c IN (v)`). -pub fn pinned_column(conjunct: &ScalarExpr) -> Option { - match conjunct { - ScalarExpr::Compare { - left, - op: CompareOpKind::Eq, - right, - .. - } => match (&**left, &**right) { - (ScalarExpr::Column(column), ScalarExpr::Literal(_)) - | (ScalarExpr::Literal(_), ScalarExpr::Column(column)) => Some(*column), - _ => None, - }, - ScalarExpr::InList { - expr, - list, - negated: false, - } => match (&**expr, list.as_slice()) { - (ScalarExpr::Column(column), [ScalarExpr::Literal(_)]) => Some(*column), - _ => None, - }, - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::schema::{NonNegativeWeightProof, SketchAlgorithm, SummaryInputExpr}; - - /// A Count-Min estimate has a bound only over weights proven - /// non-negative: an open requirement (`RawDataSamplesOfMetric`) is not a proof. - #[test] - fn count_min_estimate_has_no_bound_over_an_open_requirement() { - let summary_type = FieldDataType::Sketch(SketchAlgorithm::Cms); - let query = SketchStatistic::PointCount { - key: ColumnRef::SampleValue, - value: None, - }; - let with = |weight_domain| SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::SampleValue)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain, - }; - assert!(!estimate_has_bound( - &summary_type, - &with(WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }), - &query - )); - assert!(!estimate_has_bound( - &summary_type, - &with(WeightDomain::UnknownOrSigned), - &query - )); - assert!(estimate_has_bound( - &summary_type, - &with(WeightDomain::NonNegative { - proof: NonNegativeWeightProof::CounterRawDataSamples - }), - &query - )); - } -} diff --git a/crates/accuracy/src/sizing.rs b/crates/accuracy/src/sizing.rs index bf8194ddd..ce3069f6e 100644 --- a/crates/accuracy/src/sizing.rs +++ b/crates/accuracy/src/sizing.rs @@ -1,6 +1,4 @@ -//! The statistic a summary is read for. (Today's sizing for an `(ε, δ)` is -//! test-only, `estimators::size_params`: the predefined lists are checked -//! against it.) +//! The statistic a summary is read for. use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::ColumnRef; @@ -22,36 +20,3 @@ pub fn sketch_statistic(intent: &AggIntent) -> Option { _ => return None, }) } - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; - - /// CMS width is `⌈e/ε⌉` and depth `⌈ln(1/δ)⌉`. - #[test] - fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count; - assert_eq!( - crate::estimators::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.001), - SketchParams::Cms { - width: 2719, - depth: 7 - } // ⌈e/0.001⌉, ⌈ln 1000⌉ - ); - assert_eq!( - crate::estimators::size_params(SketchAlgorithm::Cms, &intent, 0.001, 0.01), - SketchParams::Cms { - width: 2719, - depth: 5 - } - ); - assert_eq!( - crate::estimators::size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01), - SketchParams::Cms { - width: 272, - depth: 5 - } - ); - } -} diff --git a/crates/accuracy/src/test_support.rs b/crates/accuracy/src/test_support.rs deleted file mode 100644 index f70085555..000000000 --- a/crates/accuracy/src/test_support.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Helpers that only other crates' tests call, exposed under the -//! `test-support` feature so the normal public API stays as narrow as its -//! production users need. Each forwards to the crate-private function or -//! constant of the same name. -use std::rc::Rc; - -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::properties::guarantee::AccuracyGuarantee; -use asap_types::ir::schema::{FieldDataType, SketchStatistic}; -use asap_types::ir::{OperatorNode, ScalarExpr}; - -use crate::composition::StreamNorm; -pub use crate::evidence::PropagationStats; -use crate::propagation; - -/// [`propagation::composed_guarantee`]. -pub fn composed_guarantee(node: &OperatorNode) -> Option { - propagation::composed_guarantee(node) -} - -/// `propagation::scalar_guarantee`. -pub fn scalar_guarantee( - expr: &ScalarExpr, - column: &dyn Fn(usize) -> Option, -) -> Option { - propagation::scalar_guarantee(expr, column) -} - -/// `propagation::stream_norm`. -pub fn stream_norm(node: &Rc) -> Option { - propagation::stream_norm(node) -} - -/// `propagation::fold_stats`. -pub fn fold_stats(reduction: &Reduction, input: &Rc) -> Option { - propagation::fold_stats(reduction, input) -} - -/// `propagation::estimates_raw_data_samples_of`. -pub fn estimates_raw_data_samples_of(node: &OperatorNode) -> Option { - propagation::estimates_raw_data_samples_of(node) -} - -/// `propagation::source_distinct_guarantee`. -pub fn source_distinct_guarantee( - family: &FieldDataType, - query: &SketchStatistic, - input: &Rc, -) -> Option { - propagation::source_distinct_guarantee(family, query, input) -} - -/// `crate::estimators::cms::cms_depth`. -pub fn cms_depth(delta: f64) -> u32 { - crate::estimators::cms::cms_depth(delta) -} - -/// `crate::estimators::kll::kll_rank_error_99`. -pub fn kll_rank_error_99(k: u32) -> f64 { - crate::estimators::kll::kll_rank_error_99(k) -} - -/// `crate::composition::QUOTIENT_OPERAND_DOMAINS`. -pub const QUOTIENT_OPERAND_DOMAINS: &str = crate::composition::QUOTIENT_OPERAND_DOMAINS; - -/// `crate::composition::STREAM_ROW_COUNT`. -pub const STREAM_ROW_COUNT: &str = crate::composition::STREAM_ROW_COUNT; - -/// `crate::composition::source_norm_statistic`. -pub fn source_norm_statistic(norm: &str, source: &str) -> String { - crate::composition::source_norm_statistic(norm, source) -} - -/// `crate::composition::raw_data_samples_sign_statistic`. -pub fn raw_data_samples_sign_statistic(metric: &str) -> String { - crate::composition::raw_data_samples_sign_statistic(metric) -} - -/// `stats.sign_statistic` (see [`PropagationStats`]). -pub fn sign_statistic(stats: &PropagationStats) -> Option<&str> { - stats.sign_statistic.as_deref() -} diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 56c88886d..482f4de68 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -76,9 +76,8 @@ // - memo: `nodes`, one node table for the section (each distinct node // once, children as indices before their parents), and `steps`, the // memo after each step run (`stage0`: the frontends' DAG hash-consed, -// groups without candidates; `pass1`; `pass2_local_window_composition`: -// local window composition's tumbling-window candidates; `pass2`: Pass -// 2's output, the memo Stage 2 implements). Each +// groups without candidates; `pass1`; `pass2`: Pass 2's output, the memo +// Stage 2 implements). Each // step lists its groups (`id`, `label`, `children`, `parents`, // `queries`, `roots_of`, `node`) and their candidates (`index`, // `implements`, `children`, `origin`, `accuracy_budget_parts`, `added` @@ -380,13 +379,6 @@ fn memo_json(run: &PlanningStagesRun) -> Value { if let Some(pass1) = &run.run.pass1_memo { named.push(("pass1", "Stage 1, Pass 1", pass1)); } - if let Some(pass2) = &run.pass2_local_window_composition { - named.push(( - "pass2_local_window_composition", - "Stage 1, Pass 2: local window composition", - pass2, - )); - } if let Some(logical) = &run.run.logical_asap_candidates { named.push(("pass2", "Stage 1, Pass 2", &logical.memo)); } @@ -786,11 +778,9 @@ fn group_node(table: &mut MemoNodes, group: &MemoGroup) -> usize { fn origin_name(origin: &OriginOfCandidateForMemoGroup) -> String { match origin { OriginOfCandidateForMemoGroup::Pass1 => "pass1", - OriginOfCandidateForMemoGroup::SummaryInput => "summary_input", OriginOfCandidateForMemoGroup::TumblingWindows => "tumbling_windows", OriginOfCandidateForMemoGroup::Rollup => "rollup", OriginOfCandidateForMemoGroup::TopKReuse => "top_k_reuse", - OriginOfCandidateForMemoGroup::WidenedScans => "widened_scans", } .into() } diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index a93f9fd98..d737742ab 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -8,9 +8,7 @@ use asap_planner::{ }; use asap_stage0_memo::memo_of_queries; use asap_stage1_pass1_per_computation_candidates::pass1_memo; -use asap_stage1_pass2_reuse_candidates::{ - pass2_after_local_window_composition, pass2_local_window_composition, -}; +use asap_stage1_pass2_reuse_candidates::generate_logical_asap_memo_candidates; use asap_stage2_physical_optimization::generate_physical_asap_memo_candidates; use asap_stage3_plan_selection::select_plan; use asap_types::ir::schema::schema_support::with_promql_series_identity; @@ -71,9 +69,6 @@ pub struct PlanningStagesRun { /// Stage 0's memo groups (the roots hash-consed), without candidates; /// built by `run_stages_until` only (`asap_planner` prints them). pub stage0_memo_groups: Option, - /// Stage 1 Pass 2's memo after local window composition (rule 1), before - /// the other rules; `None` before Pass 2. - pub pass2_local_window_composition: Option, /// The stages' outputs: Pass 1, Pass 2, Stage 2 and the selection. pub run: PlanningRun, } @@ -202,7 +197,6 @@ pub fn run_search_over( Ok(PlanningStagesRun { run_until: RunUntil::Stage3, stage0_memo_groups: None, - pass2_local_window_composition: None, stage0_query_language_frontends: roots, run, }) @@ -233,7 +227,6 @@ pub fn run_stages_until( let mut stages = PlanningStagesRun { run_until: until, stage0_memo_groups: Some(stage0_dag.clone()), - pass2_local_window_composition: None, stage0_query_language_frontends: roots.clone(), run: PlanningRun { pass1_memo: None, @@ -265,15 +258,7 @@ pub fn run_stages_until( if until == RunUntil::Pass1 { return Ok(stages); } - let composed = match pass2_local_window_composition(pass1, entries) { - Ok(composed) => composed, - Err(error) => { - run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); - return Ok(stages); - } - }; - stages.pass2_local_window_composition = Some(composed.clone()); - let logical = match pass2_after_local_window_composition(composed, models.evidence) { + let logical = match generate_logical_asap_memo_candidates(pass1, entries, models.evidence) { Ok(logical) => logical, Err(error) => { run.selection = Err(SelectionError::LogicalASAPCandidates(error.to_string())); diff --git a/crates/devtools/src/planning_stages_text.rs b/crates/devtools/src/planning_stages_text.rs index 1f89f597a..d3e422865 100644 --- a/crates/devtools/src/planning_stages_text.rs +++ b/crates/devtools/src/planning_stages_text.rs @@ -68,23 +68,15 @@ pub fn planning_stages_text(stages: &PlanningStagesRun) -> String { return out; } - let (Some(pass2), Some(logical)) = ( - &stages.pass2_local_window_composition, - &run.logical_asap_candidates, - ) else { + let Some(logical) = &run.logical_asap_candidates else { failure(&mut out); return out; }; - section( - &mut out, - "Stage 1 Pass 2, after local window composition (tumbling-window candidates)", - ); - write_memo(&mut out, pass2, Some(pass1), true); section( &mut out, "Stage 1 Pass 2 output = Stage 2 input: LogicalASAPMemoCandidates", ); - write_memo(&mut out, &logical.memo, Some(pass2), true); + write_memo(&mut out, &logical.memo, Some(pass1), true); if until == RunUntil::Pass2 { return out; } diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 611b29c60..a10ccd330 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -1696,10 +1696,7 @@ fn memo_section_lists_each_step_and_the_selected_choice() { let memo = &document["memo"]; let steps = memo["steps"].as_array().unwrap(); let names: Vec<_> = steps.iter().map(|s| s["step"].as_str().unwrap()).collect(); - assert_eq!( - names, - ["stage0", "pass1", "pass2_local_window_composition", "pass2"] - ); + assert_eq!(names, ["stage0", "pass1", "pass2"]); let candidates = |step: &Value| -> u64 { step["groups"] .as_array() diff --git a/crates/devtools/tests/asap_planner_acceptance.rs b/crates/devtools/tests/asap_planner_acceptance.rs index 7567cbde3..f9c3068dc 100644 --- a/crates/devtools/tests/asap_planner_acceptance.rs +++ b/crates/devtools/tests/asap_planner_acceptance.rs @@ -180,10 +180,7 @@ fn json_sections_of_stages_not_run_are_null() { .iter() .map(|step| step["step"].as_str().unwrap()) .collect(); - assert_eq!( - steps, - ["stage0", "pass1", "pass2_local_window_composition", "pass2"] - ); + assert_eq!(steps, ["stage0", "pass1", "pass2"]); let stage3 = document("stage3"); assert!(stage3["stage2_physical_asap_optimization"].is_object()); diff --git a/crates/integration-tests/Cargo.toml b/crates/integration-tests/Cargo.toml index 8f5628b71..49d278e7b 100644 --- a/crates/integration-tests/Cargo.toml +++ b/crates/integration-tests/Cargo.toml @@ -22,7 +22,6 @@ asap-devtools = { path = "../devtools" } [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } asap-devtools = { path = "../devtools" } # The tests plan PromQL and SQL workloads through the facade. diff --git a/crates/planner/Cargo.toml b/crates/planner/Cargo.toml index 8a79dbcb9..bd9ded37b 100644 --- a/crates/planner/Cargo.toml +++ b/crates/planner/Cargo.toml @@ -37,7 +37,6 @@ thiserror = "2" [dev-dependencies] asap-stage3-plan-selection = { path = "../stage3-plan-selection" } asap-stage2-physical-optimization = { path = "../stage2-physical-optimization" } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } # Tests read a plan's roots as IR (`PhysicalASAPDAG::query_roots`). asap-types = { path = "../types", features = ["test-support"] } # The SQL frontend plans through DataFusion, which is async; `e2e_plan` is diff --git a/crates/stage1-pass1-per-computation-candidates/Cargo.toml b/crates/stage1-pass1-per-computation-candidates/Cargo.toml index 6df7c52df..0c8e36180 100644 --- a/crates/stage1-pass1-per-computation-candidates/Cargo.toml +++ b/crates/stage1-pass1-per-computation-candidates/Cargo.toml @@ -15,4 +15,3 @@ thiserror = "2" [dev-dependencies] asap-types = { path = "../types", features = ["test-support"] } -asap-accuracy = { path = "../accuracy", features = ["test-support"] } diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs index 98bbaef06..fcd9a9c3b 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs @@ -167,7 +167,7 @@ fn filter(memo: &mut Memo, child: MemoGroupId, pred: Predicate) -> /// reads p99 / p50 of it. KLL state group holds one build per θ of KLL's /// list, DDSketch's state group likewise; the p95, p99 and p50 groups each /// read either state through their own readout groups. The metric rule -/// (B's relative ratio cannot read KLL's rank error) drops edges in PR 4, +/// (B's relative ratio cannot read KLL's rank error) prunes in Pass 2, /// so Pass 1 offers both algorithms to every quantile group. #[test] fn running_example_gives_one_build_per_theta_and_separate_readouts() { diff --git a/crates/stage1-pass2-reuse-candidates/Cargo.toml b/crates/stage1-pass2-reuse-candidates/Cargo.toml index 430c48f0f..367da81d7 100644 --- a/crates/stage1-pass2-reuse-candidates/Cargo.toml +++ b/crates/stage1-pass2-reuse-candidates/Cargo.toml @@ -12,3 +12,6 @@ asap-types = { path = "../types" } # The deployment's measured accuracy (`AccuracyEvidence`) Pass 2 reads. asap-accuracy = { path = "../accuracy" } thiserror = "2" + +[dev-dependencies] +asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs new file mode 100644 index 000000000..1d7fd7331 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/dead_candidates.rs @@ -0,0 +1,192 @@ +//! Step 4, dead-candidate elimination (concepts doc, section 3): a +//! candidate that is not a query output is dead when it never had a +//! consumer, or when every edge to a consumer fails or reaches a dead +//! consumer; any candidate is dead when every edge on one of its inputs +//! fails or reads a dead producer. Elimination repeats until nothing +//! changes. Each dead candidate is listed with every root cause, one +//! shortest path per failing edge. + +use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; + +use crate::logical_memo_candidates::CandidateEdge; +use asap_types::stages::{ + root_groups, CandidateForMemoGroupId, Memo, MemoGroupId, PruneCause, PrunedLogicalCandidate, + RootPruneReason, +}; + +/// The dead candidates of `memo`, whose possible edges are `edges`, each +/// with its causes, in candidate order. +pub(crate) fn eliminate_dead_candidates( + memo: &Memo, + edges: &[CandidateEdge], +) -> Vec { + let graph = EdgeGraph::new(edges); + let outputs = root_groups(&memo.queries); + let all: Vec = memo + .groups() + .iter() + .enumerate() + .flat_map(|(group, memo_group)| { + (0..memo_group.candidates.len()).map(move |index_in_group| CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }) + }) + .collect(); + // Per dead candidate, why, judged against those dead before it: cut + // off from an input (`Some(input)`), or without a live consumer. + let mut dead: BTreeMap> = BTreeMap::new(); + loop { + let newly_dead: Vec<(CandidateForMemoGroupId, Option)> = all + .iter() + .copied() + .filter(|candidate| !dead.contains_key(candidate)) + .filter_map(|candidate| { + if let Some(input) = graph.cut_off_input(memo, candidate, &dead) { + Some((candidate, Some(input))) + } else if !outputs.contains(&candidate.group) + && !graph.has_live_consumer(candidate, &dead) + { + Some((candidate, None)) + } else { + None + } + }) + .collect(); + if newly_dead.is_empty() { + break; + } + dead.extend(newly_dead); + } + dead.keys() + .map(|&candidate| PrunedLogicalCandidate { + candidate, + causes: graph.causes(candidate, &dead), + }) + .collect() +} + +/// `edges` by producer and by (consumer, input). +struct EdgeGraph<'a> { + by_producer: HashMap>, + by_consumer_input: HashMap<(CandidateForMemoGroupId, usize), Vec<&'a CandidateEdge>>, +} + +impl<'a> EdgeGraph<'a> { + fn new(edges: &'a [CandidateEdge]) -> Self { + let mut graph = Self { + by_producer: HashMap::new(), + by_consumer_input: HashMap::new(), + }; + for edge in edges { + graph + .by_producer + .entry(edge.producer) + .or_default() + .push(edge); + graph + .by_consumer_input + .entry((edge.consumer, edge.input)) + .or_default() + .push(edge); + } + graph + } + + /// Whether `candidate` has a satisfied edge into a consumer not in `dead`. + fn has_live_consumer( + &self, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> bool { + self.by_producer.get(&candidate).is_some_and(|edges| { + edges + .iter() + .any(|edge| edge.satisfies.is_ok() && !dead.contains_key(&edge.consumer)) + }) + } + + /// The first input of `candidate` with no satisfied edge from a producer + /// not in `dead`: every edge on it is cut (the metric rule, a missed + /// requirement) or reads an eliminated producer. + fn cut_off_input( + &self, + memo: &Memo, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> Option { + (0..memo + .candidate(candidate) + .expression + .operator + .children() + .len()) + .find(|&input| { + !self + .by_consumer_input + .get(&(candidate, input)) + .is_some_and(|edges| { + edges.iter().any(|edge| { + edge.satisfies.is_ok() && !dead.contains_key(&edge.producer) + }) + }) + }) + } + + /// Every root cause of dead `candidate`, breadth first, so each failing + /// edge is reached by a shortest path: a candidate cut off from an + /// input (`dead`) is traced along that input's edges, through + /// eliminated producers; any other through its consumer edges and + /// eliminated consumers, ending in `NoConsumer` at one that never had a + /// consumer. + fn causes( + &self, + candidate: CandidateForMemoGroupId, + dead: &BTreeMap>, + ) -> Vec { + let mut causes = Vec::new(); + let mut visited = BTreeSet::from([candidate]); + let mut queue = VecDeque::from([(candidate, Vec::new())]); + while let Some((at, path)) = queue.pop_front() { + let (edges, next): ( + &[&CandidateEdge], + fn(&CandidateEdge) -> CandidateForMemoGroupId, + ) = match dead[&at] { + Some(input) => ( + self.by_consumer_input + .get(&(at, input)) + .map_or(&[], Vec::as_slice), + |edge| edge.producer, + ), + None => match self.by_producer.get(&at) { + Some(edges) => (edges.as_slice(), |edge| edge.consumer), + None => { + causes.push(PruneCause { + through_pruned_candidates: path, + root: RootPruneReason::NoConsumer, + }); + continue; + } + }, + }; + for edge in edges { + match &edge.satisfies { + Err(reason) => causes.push(PruneCause { + through_pruned_candidates: path.clone(), + root: reason.clone(), + }), + // A satisfied edge of a dead candidate leads to a dead + // consumer or producer. + Ok(()) => { + if visited.insert(next(edge)) { + let mut further = path.clone(); + further.push(next(edge)); + queue.push_back((next(edge), further)); + } + } + } + } + } + causes + } +} diff --git a/crates/stage1-pass2-reuse-candidates/src/lib.rs b/crates/stage1-pass2-reuse-candidates/src/lib.rs index 970b77f40..4f403da47 100644 --- a/crates/stage1-pass2-reuse-candidates/src/lib.rs +++ b/crates/stage1-pass2-reuse-candidates/src/lib.rs @@ -1,21 +1,25 @@ //! `asap-stage1-pass2-reuse-candidates` — #509 Stage 1, Pass 2: the reuse -//! rules over memo groups and the logical edges. Its entry, +//! rules over memo groups, the readouts' required accuracy, dead-candidate +//! elimination and the logical edges. Its entry, //! [`generate_logical_asap_memo_candidates`], extends Pass 1's memo into //! the [`LogicalASAPMemoCandidates`](asap_types::stages::LogicalASAPMemoCandidates) -//! Stage 2 expands, e.g. `Quantile(x, 0.99)` and `Quantile(x, 0.95)` reading -//! one built sketch. +//! Stage 2 expands, e.g. `sum by (job)` and `sum by (job, instance)` of one +//! count reading one state. //! -//! It depends only on `asap-types`, where its input and output are. It -//! depends on no other stage crate, and the planner facade calls it after -//! Pass 1. +//! It depends on `asap-types`, where its input and output are, and on +//! `asap-accuracy` for $\alpha^{\log}$. It depends on no other stage crate, +//! and the planner facade calls it after Pass 1. +/// Step 4: dead-candidate elimination and its root causes. +mod dead_candidates; /// Stage 1, Pass 2's entry, producing [`asap_types::stages::LogicalASAPMemoCandidates`]. mod logical_memo_candidates; -/// The rules over groups. -mod memo; +/// Step 3: reach, caps and readout requirements. +mod requirements; +/// Steps 1, 2 and 5: the rules that add groups and candidates. +mod rules; +#[cfg(test)] +mod tests; -pub use logical_memo_candidates::{ - generate_logical_asap_memo_candidates, pass2_after_local_window_composition, - pass2_local_window_composition, -}; -pub use memo::rules::Pass2RuleError; +pub use logical_memo_candidates::generate_logical_asap_memo_candidates; +pub use rules::Pass2RuleError; diff --git a/crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs index 4865c0a61..cd913ea39 100644 --- a/crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs +++ b/crates/stage1-pass2-reuse-candidates/src/logical_memo_candidates.rs @@ -1,60 +1,139 @@ //! Stage 1, Pass 2's entry and output: the Pass 1 memo extended by the -//! rules over groups, and the logical edges -//! ([`asap_types::stages::LogicalASAPMemoCandidates`]), which Stage 2 expands. +//! rules over groups, the readouts' required accuracy, the dead candidates +//! and the logical edges ([`asap_types::stages::LogicalASAPMemoCandidates`]), +//! which Stage 2 expands. -use crate::memo::rules::{ - finer_aggregate, rollup, scan_labels, summary_inputs, topk, Pass2RuleError, +use std::collections::{BTreeMap, BTreeSet}; + +use crate::dead_candidates::eliminate_dead_candidates; +use crate::requirements::set_readout_requirements; +use crate::rules::{chains, rollup, topk, Pass2RuleError}; +use asap_accuracy::{accepted_metrics, producer_satisfies_logical_requirements, AccuracyEvidence}; +use asap_types::stages::{ + CandidateForMemoGroupId, EvaluationGrid, LogicalASAPMemoCandidates, LogicalEdge, Memo, + MemoGroupId, PrunedLogicalCandidate, RootPruneReason, }; -use crate::memo::window_candidates::add_local_window_composition; -use asap_accuracy::AccuracyEvidence; -use asap_types::stages::{LogicalASAPMemoCandidates, Memo}; -use asap_types::workload::QueryWorkloadEntry; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; -/// Stage 1, Pass 2: rule 1 ([`pass2_local_window_composition`]), then rules -/// 2–7 and the logical edges ([`pass2_after_local_window_composition`]). -/// `entries[i]` is query `i`'s workload entry; `evidence` is the -/// deployment's measured accuracy, which each edge's $\alpha^{\log}$ reads -/// before the theoretical guarantee. +/// Stage 1, Pass 2 (concepts doc, section 3, "Stage 1, Pass 2"), in order: +/// 1. rollup from the finest common group; +/// 2. top-k reuse; +/// 3. reach, caps and readout requirements; +/// 4. dead-candidate elimination; +/// 5. chain groups for the surviving state groups, then dead-candidate +/// elimination again over the memo with them (a merge whose θ serves no +/// consumer); +/// 6. the logical edges between the surviving candidates, with +/// $\alpha^{\log}$ from `evidence` or the theoretical guarantee. +/// +/// `entries[i]` is query `i`'s workload entry (its evaluation grid). pub fn generate_logical_asap_memo_candidates( pass1: Memo, entries: &[QueryWorkloadEntry], evidence: &dyn AccuracyEvidence, ) -> Result { - pass2_after_local_window_composition(pass2_local_window_composition(pass1, entries)?, evidence) + let mut memo = pass1; + rollup::add_rollup_candidates(&mut memo)?; + topk::add_topk_candidates(&mut memo)?; + let unknown_values = set_readout_requirements(&mut memo)?; + let dead = |pruned: &[PrunedLogicalCandidate]| -> BTreeSet { + pruned.iter().map(|pruned| pruned.candidate).collect() + }; + let before_chains = + eliminate_dead_candidates(&memo, &candidate_edges(&memo, &unknown_values, evidence)); + chains::add_chain_groups(&mut memo, entries, &dead(&before_chains))?; + let candidate_edges = candidate_edges(&memo, &unknown_values, evidence); + let pruned = eliminate_dead_candidates(&memo, &candidate_edges); + let dead = dead(&pruned); + let edges = candidate_edges + .into_iter() + .filter(|edge| !dead.contains(&edge.producer) && !dead.contains(&edge.consumer)) + .map(|edge| LogicalEdge { + producer_candidate_id: edge.producer, + consumer_candidate_id: edge.consumer, + input_index_of_consumer: edge.input, + satisfies_logical_requirement: edge.satisfies.is_ok(), + }) + .collect(); + let evaluation_grid_of_group = evaluation_grids(&memo, entries); + Ok(LogicalASAPMemoCandidates { + memo, + edges, + pruned, + evaluation_grid_of_group, + }) } -/// Pass 2's rule 1 alone: Pass 1's memo with the tumbling-window layouts -/// of each summary state group (`asap_planner` prints this step). -pub fn pass2_local_window_composition( - pass1: Memo, - entries: &[QueryWorkloadEntry], -) -> Result { - let mut memo = pass1; - // 1. Local window composition. - add_local_window_composition(&mut memo, entries)?; - Ok(memo) +/// One possible $(u, c, i)$: `producer` is a candidate of the group input +/// `input` of `consumer` reads, and whether it satisfies that input's +/// logical requirement (`producer_satisfies_logical_requirements`). +pub(crate) struct CandidateEdge { + pub(crate) producer: CandidateForMemoGroupId, + pub(crate) consumer: CandidateForMemoGroupId, + pub(crate) input: usize, + pub(crate) satisfies: Result<(), RootPruneReason>, } -/// The rest of Pass 2 on [`pass2_local_window_composition`]'s memo: rules -/// 2–7, then the readouts' required accuracy and the logical edges, with -/// $\alpha^{\log}$ from `evidence` or the theoretical guarantee -/// (`asap_accuracy::producer_satisfies_logical_requirements`). -pub fn pass2_after_local_window_composition( - mut memo: Memo, +/// Every $(u, c, i)$ of `memo` (`Memo::producers`), grouped by consumer and +/// input, producers in group order. An edge into a readout of +/// `unknown_values` fails with `ValuesPerEvaluationUnknown`. +fn candidate_edges( + memo: &Memo, + unknown_values: &BTreeMap, evidence: &dyn AccuracyEvidence, -) -> Result { - // 2. Scan widening. - scan_labels::add_scan_label_candidates(&mut memo)?; - // 3. Summary inputs. - summary_inputs::factor_summary_inputs(&mut memo)?; - // 4. Top-k reuse. - topk::add_topk_candidates(&mut memo)?; - // 5. Finer aggregate. - finer_aggregate::add_finer_aggregate_candidates(&mut memo)?; - // 6. Rollup. - rollup::add_rollup_candidates(&mut memo)?; - // 7. Window rollup. - rollup::add_window_rollup_candidates(&mut memo)?; - let _ = (memo, evidence); - todo!("#1207 PR 4: readout requirements, dead-candidate elimination and the logical edges") +) -> Vec { + let accepted = accepted_metrics(memo); + let mut edges = Vec::new(); + for (group, memo_group) in memo.groups().iter().enumerate() { + for (index_in_group, candidate) in memo_group.candidates.iter().enumerate() { + let consumer = CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }; + for input in 0..candidate.expression.operator.children().len() { + for producer in memo.producers(consumer, input) { + edges.push(CandidateEdge { + producer, + consumer, + input, + satisfies: match unknown_values.get(&consumer) { + Some(&query) => { + Err(RootPruneReason::ValuesPerEvaluationUnknown { consumer, query }) + } + None => producer_satisfies_logical_requirements( + memo, &accepted, producer, consumer, input, evidence, + ), + }, + }); + } + } + } + } + edges +} + +/// Per group: the grid its result is evaluated on, the gcd of the +/// evaluation grids of the queries reaching it +/// (`QueryRecurrence::evaluation_grid_ms`) at phase 0; `None` when no query +/// reaches it or one has no known grid. +fn evaluation_grids(memo: &Memo, entries: &[QueryWorkloadEntry]) -> Vec> { + memo.groups() + .iter() + .map(|group| { + let mut period_ms = 0; + for &query in group.reached_by_queries() { + let entry = entries.get(query)?; + period_ms = greatest_common_divisor( + period_ms, + entry + .recurrence + .evaluation_grid_ms(entry.range_evaluation)?, + ); + } + (period_ms > 0).then_some(EvaluationGrid { + period_ms, + phase_ms: 0, + }) + }) + .collect() } diff --git a/crates/stage1-pass2-reuse-candidates/src/memo.rs b/crates/stage1-pass2-reuse-candidates/src/memo.rs deleted file mode 100644 index 2e507211e..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The logical memo's rules over the memo groups Stage 0 builds -//! (`asap_stage0_memo`). - -pub(crate) mod rules; -/// Rule 1: tumbling-window layouts of a summary state group. -pub(crate) mod window_candidates; diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/rules.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules.rs deleted file mode 100644 index c9b38ed85..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/rules.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Pass 2's rules over groups (concepts doc, section 3): each reads the -//! memo and adds groups (`Memo::find_or_add_group`) or candidates -//! (`Memo::add_candidate`); none copies a consumer. -pub(crate) mod finer_aggregate; -pub(crate) mod rollup; -pub(crate) mod scan_labels; -pub(crate) mod summary_inputs; -pub(crate) mod topk; - -/// Why a Pass 2 rule failed on a memo. -#[derive(Debug, thiserror::Error)] -pub enum Pass2RuleError { - #[error("memo contains an out-of-range group reference")] - InvalidGroupReference, - #[error("memo group capacity exceeds the group identifier range")] - TooManyGroups, - #[error("memo dependencies contain a cycle")] - CyclicDependencies, -} diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs deleted file mode 100644 index fee9f0bfd..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/rules/finer_aggregate.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Read a SQL aggregate from a finer aggregate's result -//! ([`match_finer`](crate::memo::rules::finer_aggregate::match_finer)): filter the finer -//! rows, then merge them into the query's groups with each measure's rollup. - -use crate::memo::rules::Pass2RuleError; -use asap_types::stages::Memo; - -/// Rule 5, finer aggregate: a candidate of a coarse group that filters and rolls up a finer group. -pub(crate) fn add_finer_aggregate_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: rule 5 over groups") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs deleted file mode 100644 index 2f5b8a77f..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/rules/rollup.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Producer-level exact and sketch rollups, using the existing capability rules. - -use crate::memo::rules::Pass2RuleError; -use asap_types::stages::Memo; - -/// Rule 6, rollup: a candidate of a coarse aggregate or summary group reading -/// a finer group whose state rolls up (exact or sketch merge). -pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: rule 6 over groups") -} - -/// Rule 7, window rollup: a candidate of a longer-range group merging a -/// shorter-range group's windows. -pub(crate) fn add_window_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: rule 7 over groups") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs deleted file mode 100644 index 345e7b5ec..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/rules/scan_labels.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Scan-label widening for explicit memo state sharing. Reuses the existing -//! Pass 2 widening rule; only its source-chain boundaries are interned here. - -use crate::memo::rules::Pass2RuleError; -use asap_types::stages::Memo; - -/// Rule 2, scan widening: a group for the widened scan, and in each original scan group a candidate `Project(widened)`. -pub(crate) fn add_scan_label_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: rule 2 over groups") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs deleted file mode 100644 index 2bea6eaf2..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/rules/summary_inputs.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Expose private value projections and filters below summary builds as canonical inputs. -//! This preserves their full expressions and schema, including opaque series -//! identity, rather than assuming that a projection is an identity operation. - -use crate::memo::rules::Pass2RuleError; -use asap_types::stages::Memo; - -/// Rule 3, summary inputs: a value projection or filter below a summary build as its own input group. -pub(crate) fn factor_summary_inputs(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: rule 3 over groups") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs b/crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs deleted file mode 100644 index 0a82547ba..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/rules/topk.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Exact ranking reuse with explicit exact-input groups. No global selection -//! flags are required: only exact implementations can supply these inputs. - -use crate::memo::rules::Pass2RuleError; -use asap_types::stages::Memo; - -/// Rule 4, top-k reuse: a candidate of each top-k group reading a larger-`k` group, with an exact-input requirement. -pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { - let _ = memo; - todo!("#1207 PR 4: rule 4 over groups") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs b/crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs deleted file mode 100644 index 804bcdd65..000000000 --- a/crates/stage1-pass2-reuse-candidates/src/memo/window_candidates.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! Rule 1, chain groups (concepts doc, section 3, "Chain groups"): a -//! `SummaryTumblingWindowChain[w](X)` group per width, and a `SummaryMerge` -//! over the `TimeRange` (and `TimeShift`s) over it as a candidate of the -//! summary state group. - -use crate::memo::rules::Pass2RuleError; -use asap_types::stages::Memo; -use asap_types::workload::QueryWorkloadEntry; - -/// Rule 1: for each summary state group that survived dead-candidate -/// elimination, a chain group and a merge candidate per tumbling width $w$ -/// from the gcd closure of the evaluation grids, ranges and offsets of the -/// queries that reach it and repeat on a known grid. `entries[i]` is query -/// `i`'s workload entry. -pub(crate) fn add_local_window_composition( - memo: &mut Memo, - entries: &[QueryWorkloadEntry], -) -> Result<(), Pass2RuleError> { - let _ = (memo, entries); - todo!("#1207 PR 4: rule 1, chain groups and the merges over them") -} diff --git a/crates/stage1-pass2-reuse-candidates/src/requirements.rs b/crates/stage1-pass2-reuse-candidates/src/requirements.rs new file mode 100644 index 000000000..3898f769c --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/requirements.rs @@ -0,0 +1,322 @@ +//! Step 3 (concepts doc, section 3, "Reach, caps and readout requirements"; +//! section 4, "Readout requirements" and "Validity caps and pruning caps"): +//! each readout's required accuracy, from the caps the queries that reach +//! it pass down through their consumers' error contracts. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rules::Pass2RuleError; +use asap_accuracy::readout_metric; +use asap_types::ir::operator::asap::summary_type_of_operator; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::{ASAPOp, Operator, QueryRoot}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalRequirements, Memo, MemoGroupId, RequiredAccuracy, +}; + +/// The error grid of the readouts of a composed query. +const MAX_ERROR_LEVELS: [f64; 7] = [0.1, 0.05, 0.02, 0.01, 0.005, 0.002, 0.001]; +/// The failure-probability grid of the readouts of a composed query: the +/// powers of ten from 1e-2 down to 1e-5, continued down to the first one at +/// or below a tenth of the query's per-value failure probability, so that up +/// to ten readouts each find a level and the levels sum within it (δ = 0.01 +/// over 10 000 series is 1e-6 per value: two quantiles of a ratio read at +/// 1e-7 each). +fn failure_probability_levels(per_value: f64) -> Vec { + let lowest = (per_value / 10.0).min(1e-5); + let deepest_exponent = (-lowest.log10() - 1e-9).ceil().min(300.0) as i32; + (2..=deepest_exponent) + .map(|exponent| 1.0 / 10f64.powi(exponent)) + .collect() +} + +/// Per group, by group id: per (query, index in `AccuracyMetric::ALL`), the +/// largest error in that metric the group's output may have for the query, +/// $\max_q \varepsilon_q / \lambda_{q,c,i}$ capped by the validity caps on +/// the way. +type Caps = Vec>; + +/// Gives every sketch readout (`SummaryEstimate` over a sketch state) its +/// required accuracy, one readout candidate per requirement level: Pass 1's +/// candidate takes the first level and a copy is added per further level, +/// each written with `Memo::set_input_requirements`. +/// - A query whose path reads one readout group gives one level: its target +/// divided by the path's factor, the readout group's cap, at the target's +/// failure probability per value. +/// - A composed query (several readout groups, such as a ratio) gives every +/// level of the grid [`MAX_ERROR_LEVELS`] × [`failure_probability_levels`] +/// not looser than the cap and the per-value failure probability (the cap +/// itself when no error level is), one δ level of 1 when the target sets +/// no failure probability. +/// - A readout group reached by several queries gets the union of their +/// levels; one no query with a target reaches in the readout's metric +/// gets none. +/// +/// Returns the readouts reached by a query whose whole-result failure +/// probability has no per-value one ($n_q$ unknown), each with that query: +/// Pass 2 prunes them (`RootPruneReason::ValuesPerEvaluationUnknown`). +pub(crate) fn set_readout_requirements( + memo: &mut Memo, +) -> Result, Pass2RuleError> { + let mut without_per_value_failure_probability = BTreeMap::new(); + let order = memo + .topological_order() + .ok_or(Pass2RuleError::CyclicDependencies)?; + let caps = caps(memo, &order); + let readout_groups: BTreeSet = (0..memo.groups().len()) + .map(|index| MemoGroupId(index as u32)) + .filter(|&group| !sketch_readouts(memo, group).is_empty()) + .collect(); + let composed = composed_queries(memo, &order, &readout_groups); + for &group in &readout_groups { + for (index_in_group, metric) in sketch_readouts(memo, group) { + let metric_index = AccuracyMetric::ALL + .iter() + .position(|known| *known == metric) + .expect("every metric is listed"); + let mut levels: Vec = Vec::new(); + let mut unknown_values_of = None; + for (&(query, index), &cap) in &caps[group.0 as usize] { + if index != metric_index { + continue; + } + let Some(target) = &memo.queries[query].accuracy_target else { + continue; + }; + let Some(query_levels) = query_levels( + target, + memo.queries[query].values_per_evaluation, + metric, + cap, + composed.contains(&query), + ) else { + unknown_values_of.get_or_insert(query); + continue; + }; + for level in query_levels { + if !levels.contains(&level) { + levels.push(level); + } + } + } + for readout in write_levels(memo, group, index_in_group, levels)? { + if let Some(query) = unknown_values_of { + without_per_value_failure_probability.insert(readout, query); + } + } + } + } + Ok(without_per_value_failure_probability) +} + +/// The requirement levels one query gives a readout in `metric` whose cap +/// for it is `cap`; `None` when its target's failure probability has no +/// per-value one (a whole-result δ with $n_q$ unknown). +fn query_levels( + target: &AccuracyTarget, + values_per_evaluation: Option, + metric: AccuracyMetric, + cap: f64, + composed: bool, +) -> Option> { + // The target's failure probability per value: a whole-result δ split + // over the values one evaluation returns; 1 without a δ. + let per_value = match target.failure_probability { + None => 1.0, + Some(delta) => { + delta + / target + .accuracy_scope + .max_values_covered(values_per_evaluation)? + .max(1) as f64 + } + }; + let level = |max_error, failure_probability| RequiredAccuracy { + metric, + max_error, + failure_probability, + }; + if !composed { + return Some(vec![level(cap, per_value)]); + } + let within = |grid: &[f64], limit: f64| -> Vec { + let kept: Vec = grid.iter().copied().filter(|&x| x <= limit).collect(); + if kept.is_empty() { + vec![limit] + } else { + kept + } + }; + let errors = within(&MAX_ERROR_LEVELS, cap); + let failure_probabilities = if target.failure_probability.is_none() { + vec![1.0] + } else { + within(&failure_probability_levels(per_value), per_value) + }; + Some( + errors + .iter() + .flat_map(|&max_error| { + failure_probabilities + .iter() + .map(move |&failure_probability| level(max_error, failure_probability)) + }) + .collect(), + ) +} + +/// Writes `levels` on readout `index_in_group` of `group`: the first on it, +/// each further one on a copy added to the group. No level leaves it +/// without a requirement. Returns the readout and its copies. +fn write_levels( + memo: &mut Memo, + group: MemoGroupId, + index_in_group: usize, + levels: Vec, +) -> Result, Pass2RuleError> { + let readout = CandidateForMemoGroupId { + group, + index_in_group, + }; + let unset = memo.candidate(readout).clone(); + let mut written = vec![readout]; + for (n, level) in levels.into_iter().enumerate() { + let id = if n == 0 { + readout + } else { + memo.add_candidate(group, unset.clone()) + .map_err(Pass2RuleError::Candidate)? + }; + if n > 0 { + written.push(id); + } + memo.set_input_requirements( + id, + vec![LogicalRequirements { + required_accuracy: Some(level), + }], + ); + } + Ok(written) +} + +/// The sketch readouts of `group` without a requirement yet, each with the +/// metric of its error ([`readout_metric`]). +fn sketch_readouts(memo: &Memo, group: MemoGroupId) -> Vec<(usize, AccuracyMetric)> { + memo.groups()[group.0 as usize] + .candidates + .iter() + .enumerate() + .filter_map(|(index_in_group, candidate)| { + let Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input, + query, + }) = &candidate.expression.operator + else { + return None; + }; + if candidate.input_requirements[0].required_accuracy.is_some() { + return None; + } + let summary_type = + summary_type_of_operator(&memo.groups()[summary_input.0 as usize].expression)?; + Some((index_in_group, readout_metric(summary_type, query)?)) + }) + .collect() +} + +/// Each group's caps ([`Caps`]): each query root group's target, in the +/// target's metric (every metric when it names none), then, consumers +/// before producers, every candidate's caps passed to its inputs through +/// its error contract, $\min(b_{c,i},\ \text{cap}/\beta_{c,i})$, keeping +/// the largest per query and metric (the bottleneck over paths). A scalar +/// root's operands get none. +fn caps(memo: &Memo, order: &[MemoGroupId]) -> Caps { + let mut caps: Caps = vec![BTreeMap::new(); memo.groups().len()]; + for (query, memo_query) in memo.queries.iter().enumerate() { + let (Some(target), QueryRoot::Operator(group)) = + (&memo_query.accuracy_target, &memo_query.root) + else { + continue; + }; + for (metric_index, metric) in AccuracyMetric::ALL.iter().enumerate() { + if target.accuracy_metric.is_none_or(|named| named == *metric) { + raise( + &mut caps[group.0 as usize], + (query, metric_index), + target.max_error, + ); + } + } + } + for &group in order.iter().rev() { + let group_caps = caps[group.0 as usize].clone(); + if group_caps.is_empty() { + continue; + } + for consumer in &memo.groups()[group.0 as usize].candidates { + let operator = &consumer.expression.operator; + for (&(query, metric_index), &cap) in &group_caps { + let Some(propagation) = LogicalRequirements::error_propagation( + memo, + operator, + AccuracyMetric::ALL[metric_index], + ) else { + continue; + }; + for (input, &child) in operator.children().into_iter().enumerate() { + let factor = propagation.factor_per_input[input]; + let validity = + propagation.validity_cap_per_input[input].unwrap_or(f64::INFINITY); + let passed = if factor > 0.0 { + validity.min(cap / factor) + } else { + validity + }; + raise(&mut caps[child.0 as usize], (query, metric_index), passed); + } + } + } + } + caps +} + +/// The queries that combine several readouts: a candidate they reach has +/// two or more inputs below which a readout group lies (a ratio of two +/// quantiles). Readouts of one value by different summary types are +/// alternatives in one group's candidates, not a combination. +fn composed_queries( + memo: &Memo, + order: &[MemoGroupId], + readout_groups: &BTreeSet, +) -> BTreeSet { + let mut reads_a_readout = vec![false; memo.groups().len()]; + let mut composed = BTreeSet::new(); + for &group in order { + let candidates = &memo.groups()[group.0 as usize].candidates; + let inputs_reading = |operator: &Operator| { + operator + .children() + .into_iter() + .filter(|child| reads_a_readout[child.0 as usize]) + .count() + }; + let most_inputs_reading = candidates + .iter() + .map(|candidate| inputs_reading(&candidate.expression.operator)) + .max() + .unwrap_or(0); + if most_inputs_reading > 1 { + composed.extend(memo.groups()[group.0 as usize].queries.iter().copied()); + } + reads_a_readout[group.0 as usize] = + readout_groups.contains(&group) || most_inputs_reading > 0; + } + composed +} + +/// Keeps the larger of `cap` and the known cap of `key`. +fn raise(caps: &mut BTreeMap<(usize, usize), f64>, key: (usize, usize), cap: f64) { + let known = caps.entry(key).or_insert(cap); + *known = known.max(cap); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules.rs b/crates/stage1-pass2-reuse-candidates/src/rules.rs new file mode 100644 index 000000000..5e186c1a3 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules.rs @@ -0,0 +1,105 @@ +//! Pass 2's rules over groups (concepts doc, section 3, "Stage 1, Pass 2"): +//! each reads the memo and adds groups (`Memo::find_or_add_group`) or +//! candidates (`Memo::add_candidate`); none copies a consumer, so every +//! consumer of a group sees what a rule adds to it. +pub(crate) mod chains; +pub(crate) mod rollup; +pub(crate) mod topk; + +use asap_types::ir::schema::SketchParams; +use asap_types::ir::{Operator, SchemaDerivationError}; +use asap_types::stages::{ + CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, + LogicalProperties, LogicalPropertiesError, LogicalRequirements, Memo, MemoGroupId, + OriginOfCandidateForMemoGroup, +}; + +/// Why Pass 2 failed on a memo: a rule built an expression the node rules +/// reject (a bug in the rule), or the memo's reads form a cycle. +#[derive(Debug, thiserror::Error)] +pub enum Pass2RuleError { + /// A group, or one of its candidates, reads itself through other groups. + #[error("memo dependencies contain a cycle")] + CyclicDependencies, + /// A rule built a group expression the node rules reject. + #[error("a rule built a group that breaks the node rules: {0}")] + Group(#[from] SchemaDerivationError), + /// A rule built a candidate expression the node rules reject. + #[error("a rule built a candidate that breaks the node rules: {0}")] + Candidate(CandidateError), + /// A rule gave a candidate parameters its expression cannot have. + #[error("a rule built invalid logical properties: {0}")] + LogicalProperties(#[from] LogicalPropertiesError), +} + +/// Adds the candidate `operator` with $\boldsymbol\theta$ `parameters` to +/// `group`, its inputs without a requirement (step 3 sets the readouts'). +/// `None` when its output is not the group's +/// (`CandidateError::OutputDiffersFromGroup`): the rules match shapes and +/// let the memo decide whether a match computes the group's result. Any +/// other rejection is a bug in the rule and an error. +pub(crate) fn add_candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, + origin: OriginOfCandidateForMemoGroup, +) -> Result, Pass2RuleError> { + let expression = CandidateExpression { operator }; + let logical_properties = LogicalProperties::new(&expression, parameters)?; + let input_requirements = + vec![LogicalRequirements::default(); expression.operator.children().len()]; + match memo.add_candidate( + group, + CandidateForMemoGroup { + expression, + logical_properties, + input_requirements, + fused_groups: Vec::new(), + origin, + }, + ) { + Ok(id) => Ok(Some(id)), + Err(CandidateError::OutputDiffersFromGroup { .. }) => Ok(None), + Err(error) => Err(Pass2RuleError::Candidate(error)), + } +} + +/// Adds `operator`, which passes on the states or rows of its one input +/// group (a `SummaryMerge`, or a `Filter`, `TimeRange` or `TimeShift`), to +/// `group`: one candidate per θ among the input group's candidates (concepts +/// doc, section 2: such a candidate carries the θ of the states it reads), +/// one without a θ over rows or exact states. +pub(crate) fn add_candidates_per_input_parameters( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + origin: OriginOfCandidateForMemoGroup, +) -> Result<(), Pass2RuleError> { + let input = *operator.children()[0]; + let mut parameters: Vec> = Vec::new(); + for candidate in &memo.groups()[input.0 as usize].candidates { + let theta = candidate.logical_properties.parameters().cloned(); + if !parameters.contains(&theta) { + parameters.push(theta); + } + } + for theta in parameters { + add_candidate(memo, group, operator.clone(), theta, origin.clone())?; + } + Ok(()) +} + +/// The group of `operator`, with its own expression as its candidates +/// ([`add_candidates_per_input_parameters`]): a group a rule adds between +/// others, a compensating `Filter`, a `TimeRange` or `TimeShift` over a +/// chain. +pub(crate) fn add_group_computing_itself( + memo: &mut Memo, + operator: Operator, + origin: OriginOfCandidateForMemoGroup, +) -> Result { + let group = memo.find_or_add_group(operator.clone())?; + add_candidates_per_input_parameters(memo, group, operator, origin)?; + Ok(group) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs new file mode 100644 index 000000000..6f961c959 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/chains.rs @@ -0,0 +1,251 @@ +//! Step 5, chain groups (concepts doc, section 3, "Chain groups"): a +//! `SummaryTumblingWindowChain[w](X)` group per width, and a `SummaryMerge` +//! over the `TimeRange` (and `TimeShift`s) over it as a candidate of the +//! summary state group. + +use std::collections::BTreeSet; + +use crate::rules::{ + add_candidate, add_candidates_per_input_parameters, add_group_computing_itself, Pass2RuleError, +}; +use asap_types::ir::operator::operator_properties::{AtModifier, TimeShift}; +use asap_types::ir::schema::{SketchParams, SummaryTypeCapability}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, TimeRangeKind}; +use asap_types::stages::{ + CandidateForMemoGroupId, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; +use asap_types::workload::{greatest_common_divisor, QueryWorkloadEntry}; + +/// For each summary state group S = `SummaryAgg(TimeRange(W, +/// TimeShift*(X)))` of a summary type that merges, with a build that +/// survived dead-candidate elimination (`pruned`), and each width $w$: the +/// chain group $G_w$ = `SummaryTumblingWindowChain[w](X)` (the shifts +/// stripped, so ranges and offsets over X share it) with one build per θ of +/// S's surviving builds, the shifts and the `TimeRange` re-wrapped around +/// it, and the candidates `SummaryMerge` over the outermost of them in S, +/// each of these one per θ of the group it reads (a chain shared by several +/// state groups has the builds of all of them). +/// +/// The widths are the gcd closure of $\gcd(g_q, W, o_1, \ldots)$ over the +/// queries reaching S that repeat on a known evaluation grid $g_q$ +/// (`QueryRecurrence::evaluation_grid_ms`), with the offsets $o_j$ on S's +/// path; no query with a grid, no chain. Under `@ T` S adds no width: it +/// reads only chains other states created whose width divides W, the +/// offsets and T. Under `@ end()` or `@ start()` (a `TimeShift` the front +/// end puts above the subtree it anchors) no chain is read. +/// `entries[q]` is query `q`'s workload entry. +pub(crate) fn add_chain_groups( + memo: &mut Memo, + entries: &[QueryWorkloadEntry], + pruned: &BTreeSet, +) -> Result<(), Pass2RuleError> { + // First every chain group with its builds, then the groups reading + // them, so a chain shared by state groups offers each every θ it has. + let mut chains_of_states: Vec<(MemoGroupId, ChainPath, Vec)> = Vec::new(); + for index in 0..memo.groups().len() { + let state = MemoGroupId(index as u32); + let Some(path) = chain_path(memo, state) else { + continue; + }; + let parameters: Vec> = { + let mut parameters = Vec::new(); + for (index_in_group, candidate) in memo.groups()[index].candidates.iter().enumerate() { + let survived = !pruned.contains(&CandidateForMemoGroupId { + group: state, + index_in_group, + }); + let own = candidate.expression.operator == memo.groups()[index].expression; + let theta = candidate.logical_properties.parameters().cloned(); + if survived && own && !parameters.contains(&theta) { + parameters.push(theta); + } + } + parameters + }; + if parameters.is_empty() { + continue; + } + let divisor = path.shifts.iter().fold(path.window_ms, |gcd, shift| { + greatest_common_divisor(gcd, shift.offset_ms.unsigned_abs()) + }); + let widths: BTreeSet = match path.anchored_at_ms { + Some(at_ms) => existing_chain_widths(memo, &path) + .into_iter() + .filter(|&width| divisor % width == 0 && at_ms.unsigned_abs() % width == 0) + .collect(), + None => gcd_closure(memo.groups()[index].queries.iter().filter_map(|&query| { + let entry = entries.get(query)?; + let grid = entry + .recurrence + .evaluation_grid_ms(entry.range_evaluation)?; + Some(greatest_common_divisor(grid, divisor)) + })), + }; + let mut chain_groups = Vec::new(); + for width in widths { + let chain = path.chain(width); + let chain_group = memo.find_or_add_group(chain.clone())?; + for theta in ¶meters { + add_candidate( + memo, + chain_group, + chain.clone(), + theta.clone(), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + chain_groups.push(chain_group); + } + chains_of_states.push((state, path, chain_groups)); + } + for (state, path, chain_groups) in chains_of_states { + for chain_group in chain_groups { + let mut top = chain_group; + for shift in path.shifts.iter().rev() { + top = add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: *shift, + child: top, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + let windows = add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: std::time::Duration::from_millis(path.window_ms), + kind: TimeRangeKind::Range, + child: top, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + add_candidates_per_input_parameters( + memo, + state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + OriginOfCandidateForMemoGroup::TumblingWindows, + )?; + } + } + Ok(()) +} + +/// A state group's walk down to the rows its chain reads: +/// `SummaryAgg` → `TimeRange(W)` → `TimeShift`s → X. +struct ChainPath { + /// The `SummaryAgg`, its child to be replaced by X. + build: ASAPOp, + window_ms: u64, + /// Top down. + shifts: Vec, + /// T of an `@ T` among the shifts. + anchored_at_ms: Option, + rows: MemoGroupId, +} + +impl ChainPath { + /// `SummaryTumblingWindowChain[width](X)` with the build's arguments. + fn chain(&self, width: u64) -> Operator { + let ASAPOp::SummaryAgg { + summary_type, + input, + reduction, + filter, + .. + } = &self.build + else { + unreachable!("a chain path starts at a SummaryAgg"); + }; + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: self.rows, + summary_type: summary_type.clone(), + input: input.clone(), + reduction: reduction.clone(), + filter: filter.clone(), + tumbling_window_width_ms: width, + }) + } +} + +/// S's [`ChainPath`]: a `SummaryAgg` of a summary type whose states merge +/// (its capability row), over a range `TimeRange(W)` with `W > 0`, over +/// `TimeShift`s with at most one `@ T` and no `@ end()` or `@ start()`. +fn chain_path(memo: &Memo, state: MemoGroupId) -> Option { + let expression = &memo.groups()[state.0 as usize].expression; + let Operator::ASAP( + build @ ASAPOp::SummaryAgg { + child, + summary_type, + .. + }, + ) = expression + else { + return None; + }; + SummaryTypeCapability::of(summary_type).merge_inputs?; + let Operator::NonASAP(NonASAPOp::TimeRange { + range, + kind: TimeRangeKind::Range, + child: below, + }) = &memo.groups()[child.0 as usize].expression + else { + return None; + }; + let window_ms = u64::try_from(range.as_millis()).ok().filter(|&ms| ms > 0)?; + let (mut shifts, mut anchored_at_ms, mut rows) = (Vec::new(), None, *below); + while let Operator::NonASAP(NonASAPOp::TimeShift { shift, child }) = + &memo.groups()[rows.0 as usize].expression + { + match shift.at { + None => {} + Some(AtModifier::Timestamp(at_ms)) if anchored_at_ms.is_none() => { + anchored_at_ms = Some(at_ms) + } + Some(_) => return None, + } + shifts.push(*shift); + rows = *child; + } + Some(ChainPath { + build: build.clone(), + window_ms, + shifts, + anchored_at_ms, + rows, + }) +} + +/// The widths of the chain groups over `path`'s rows with its build's +/// arguments that the memo has already. +fn existing_chain_widths(memo: &Memo, path: &ChainPath) -> Vec { + memo.groups() + .iter() + .filter_map(|group| match &group.expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) if group.expression == path.chain(*tumbling_window_width_ms) => { + Some(*tumbling_window_width_ms) + } + _ => None, + }) + .collect() +} + +/// The gcd closure of `widths`: each, and the gcd of every two in it. +fn gcd_closure(widths: impl IntoIterator) -> BTreeSet { + let mut closure: BTreeSet = BTreeSet::new(); + for width in widths.into_iter().filter(|&width| width > 0) { + let gcds: Vec = closure + .iter() + .map(|&known| greatest_common_divisor(known, width)) + .collect(); + closure.insert(width); + closure.extend(gcds); + } + closure +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs b/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs new file mode 100644 index 000000000..957c8bfe2 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/rollup.rs @@ -0,0 +1,446 @@ +//! Step 1, rollup from the finest common group (concepts doc, section 3): +//! a coarse aggregate group $G_c$ computed from a finer group $G_f$ of the +//! same definition, and per cluster of equal definitions one finest common +//! group $G^*$ every member can roll up from. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rules::{ + add_candidate, add_candidates_per_input_parameters, add_group_computing_itself, Pass2RuleError, +}; +use asap_types::ir::operator::operator_properties::SourceColumn; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::properties::summary_coverage::{Constraint, SummaryStateDefinition}; +use asap_types::ir::scalar::{CompareOpKind, ExprSemantics, Predicate, ScalarExpr, ScalarValue}; +use asap_types::ir::schema::{FieldDataType, SummaryTypeCapability}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorResultKind}; +use asap_types::stages::{ + parameter_list, ColumnLineage, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; + +/// Adds, to each aggregate group $G_c$, a candidate computing it from each +/// finer group $G_f$ of its cluster ([`rollup_from`]), after synthesizing +/// each cluster's finest common group ([`finest_common_group`]). A cluster +/// is the summary state groups (`SummaryAgg` of a summary type whose +/// states merge across groups) or exact `Aggregate` groups (of sums, +/// counts, minima and maxima) with equal definitions apart from their keys +/// ([`Member::definition`]) and equal interval constraints (the time +/// window). +pub(crate) fn add_rollup_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let mut clusters: Vec> = Vec::new(); + for index in 0..memo.groups().len() { + let Some(member) = Member::of(memo, MemoGroupId(index as u32)) else { + continue; + }; + match clusters + .iter_mut() + .find(|cluster| cluster[0].same_cluster(&member)) + { + Some(cluster) => cluster.push(member), + None => clusters.push(vec![member]), + } + } + for mut cluster in clusters.into_iter().filter(|cluster| cluster.len() > 1) { + if let Some(finest) = finest_common_group(memo, &cluster)? { + if cluster.iter().all(|member| member.group != finest) { + cluster.extend(Member::of(memo, finest)); + } + } + for coarse in &cluster { + for fine in &cluster { + if coarse.group != fine.group { + rollup_from(memo, coarse, fine)?; + } + } + } + } + Ok(()) +} + +/// One aggregate group as a rollup rule reads it. +#[derive(Debug, Clone)] +struct Member { + group: MemoGroupId, + /// The group's expression. + expression: Operator, + /// The rows it aggregates: the `SummaryAgg`'s or `Aggregate`'s input. + rows: MemoGroupId, + /// Its group-by keys, in output order, by the source column they copy. + keys: Vec, + /// Its coverage's selection, one box. + selection: BTreeMap, + /// Its coverage's definition without its keys: the top operator's + /// reduction (and an `Aggregate`'s output names) blanked, and an input + /// group that is a `Scan` written as that `Scan` (as when one of its + /// predicates is lifted), so equal computations over one source compare + /// equal whatever their keys and label selections. + definition: SummaryStateDefinition, +} + +impl Member { + fn of(memo: &Memo, group: MemoGroupId) -> Option { + let memo_group = &memo.groups()[group.0 as usize]; + let (rows, by) = match &memo_group.expression { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + reduction: Reduction::Reduce(by), + .. + }) if SummaryTypeCapability::merges_groups(summary_type) => (*child, by), + Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) if filters.iter().all(Option::is_none) + && measures + .iter() + .all(|measure| rolled_up(measure, 0).is_some()) => + { + (*child, by) + } + _ => return None, + }; + if by.is_without() { + return None; + } + let coverage = memo_group.group_properties.aggregate_coverage.as_ref()?; + let [selection] = coverage.selection() else { + return None; + }; + let rows_lineage = &memo.groups()[rows.0 as usize] + .group_properties + .column_lineage; + let keys = by + .keys() + .iter() + .map(|&key| match rows_lineage.get(key)? { + ColumnLineage::SourceColumn(column) => Some(column.clone()), + ColumnLineage::Computed => None, + }) + .collect::>>()?; + let mut definition = coverage.definition().clone(); + match definition.unlifted_operators.first_mut()? { + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) => { + *reduction = Reduction::PerEntity + } + Operator::NonASAP(NonASAPOp::Aggregate { + reduction, + output_names, + .. + }) => { + *reduction = Reduction::PerEntity; + output_names.clear(); + } + _ => return None, + } + if let Some(input) = definition.input_group { + if let scan @ Operator::NonASAP(NonASAPOp::Scan { .. }) = + &memo.groups()[input.0 as usize].expression + { + definition.input_group = None; + definition + .unlifted_operators + .push(scan.map_children(|&child| Some(child))); + } + } + Some(Self { + group, + expression: memo_group.expression.clone(), + rows, + keys, + selection: selection.columns().clone(), + definition, + }) + } + + /// Whether `other` computes the same thing over the same time window: + /// equal definitions without keys and equal interval constraints. + fn same_cluster(&self, other: &Self) -> bool { + self.definition == other.definition && self.intervals() == other.intervals() + } + + /// Its interval constraints (the time window, value ranges): equal + /// across a cluster, never compensated. + fn intervals(&self) -> BTreeMap<&SourceColumn, &Constraint> { + self.selection + .iter() + .filter(|(_, constraint)| matches!(constraint, Constraint::Interval { .. })) + .collect() + } +} + +/// The cluster's finest common group $G^*$: its selection is, per column, +/// the union of the members' value-set constraints (none where a member +/// has none); its keys are every member's keys plus each column on which a +/// member selects fewer rows than $G^*$. It is a member when one has that +/// selection and those keys; otherwise it is built over the rows of a +/// member whose selection is the union, with one build per listed θ. +/// `None` when no member's rows have the union selection (a new scan of +/// the union is not built) or a key is not a column of those rows. +fn finest_common_group( + memo: &mut Memo, + cluster: &[Member], +) -> Result, Pass2RuleError> { + let columns: BTreeSet<&SourceColumn> = cluster + .iter() + .flat_map(|member| member.selection.keys()) + .collect(); + let mut hull = BTreeMap::new(); + let mut keys: BTreeSet = cluster + .iter() + .flat_map(|member| member.keys.iter().cloned()) + .collect(); + for column in columns { + let constraints: Option> = cluster + .iter() + .map(|member| member.selection.get(column)) + .collect(); + let union = constraints.and_then(|constraints| { + if constraints + .iter() + .all(|constraint| *constraint == constraints[0]) + { + return Some(constraints[0].clone()); + } + let mut values: Vec = Vec::new(); + for constraint in constraints { + let Constraint::In(more) = constraint else { + return None; + }; + values.extend( + more.iter() + .filter(|value| !values.contains(value)) + .cloned() + .collect::>(), + ); + } + Some(Constraint::In(values)) + }); + if let Some(union) = union { + hull.insert(column.clone(), union); + } + if cluster + .iter() + .any(|member| member.selection.get(column) != hull.get(column)) + { + keys.insert(column.clone()); + } + } + if let Some(member) = cluster.iter().find(|member| { + member.selection == hull && member.keys.iter().cloned().collect::>() == keys + }) { + return Ok(Some(member.group)); + } + let Some(base) = cluster.iter().find(|member| member.selection == hull) else { + return Ok(None); + }; + let rows_lineage = &memo.groups()[base.rows.0 as usize] + .group_properties + .column_lineage; + let mut key_columns = Vec::new(); + for key in &keys { + let Some(column) = column_of(rows_lineage, key) else { + return Ok(None); + }; + key_columns.push(column); + } + key_columns.sort_unstable(); + let mut expression = base.expression.clone(); + match &mut expression { + Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) + | Operator::NonASAP(NonASAPOp::Aggregate { reduction, .. }) => { + *reduction = Reduction::by(key_columns) + } + _ => unreachable!("a member is a SummaryAgg or an Aggregate"), + } + let finest = memo.find_or_add_group(expression.clone())?; + let parameters: Vec<_> = match &expression { + Operator::ASAP(ASAPOp::SummaryAgg { + summary_type: FieldDataType::Sketch(algorithm), + .. + }) => parameter_list(*algorithm) + .iter() + .cloned() + .map(Some) + .collect(), + _ => vec![None], + }; + for theta in parameters { + add_candidate( + memo, + finest, + expression.clone(), + theta, + OriginOfCandidateForMemoGroup::Rollup, + )?; + } + Ok(Some(finest)) +} + +/// Adds to `coarse` the candidate computing it from `fine` when (concepts +/// doc, section 3, conditions 1–5) its keys are among `fine`'s, its +/// selection lies inside `fine`'s, narrower only on `fine`'s keys, and the +/// cluster's equal definition and time window hold: +/// `Rollup(by coarse's keys)(Filter(compensation, fine))`, without the +/// `Filter` when the selections are equal. States roll up with a +/// regrouping `SummaryMerge`; sums and counts with a sum, minima and maxima +/// with themselves. +fn rollup_from(memo: &mut Memo, coarse: &Member, fine: &Member) -> Result<(), Pass2RuleError> { + if !coarse.keys.iter().all(|key| fine.keys.contains(key)) { + return Ok(()); + } + let mut compensation = Vec::new(); + for column in coarse.selection.keys().chain(fine.selection.keys()) { + match (coarse.selection.get(column), fine.selection.get(column)) { + (Some(narrow), wide) if Some(narrow) != wide => { + let within = match (narrow, wide) { + (_, None) => true, + (Constraint::In(values), Some(Constraint::In(wider))) => { + values.iter().all(|value| wider.contains(value)) + } + _ => false, + }; + if !within || !fine.keys.contains(column) { + return Ok(()); + } + if !compensation.iter().any(|(known, _)| known == column) { + compensation.push((column.clone(), narrow.clone())); + } + } + (None, Some(_)) => return Ok(()), + _ => {} + } + } + if compensation.is_empty() && coarse.keys.len() == fine.keys.len() { + return Ok(()); + } + let fine_properties = &memo.groups()[fine.group.0 as usize].group_properties; + let lineage = fine_properties.column_lineage.clone(); + let semantics = if fine_properties.result_kind == OperatorResultKind::Relation { + ExprSemantics::Sql + } else { + ExprSemantics::Promql + }; + let mut conjuncts = Vec::new(); + for (column, constraint) in &compensation { + let Some(at) = column_of(&lineage, column) else { + return Ok(()); + }; + let Some(conjunct) = selecting(at, constraint, semantics) else { + return Ok(()); + }; + conjuncts.push(conjunct); + } + let input = match conjuncts.len() { + 0 => fine.group, + _ => add_group_computing_itself( + memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(if conjuncts.len() == 1 { + conjuncts.remove(0) + } else { + ScalarExpr::BoolAnd(conjuncts) + }), + child: fine.group, + }), + OriginOfCandidateForMemoGroup::Rollup, + )?, + }; + let Some(group_by) = coarse + .keys + .iter() + .map(|key| column_of(&lineage, key)) + .collect::>>() + else { + return Ok(()); + }; + let rollup = match &coarse.expression { + Operator::ASAP(ASAPOp::SummaryAgg { .. }) => Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![input], + group_by: Some(group_by), + }), + Operator::NonASAP(NonASAPOp::Aggregate { + measures, + output_names, + filters, + .. + }) => { + let Some(measures) = measures + .iter() + .enumerate() + .map(|(index, measure)| rolled_up(measure, fine.keys.len() + index)) + .collect::>>() + else { + return Ok(()); + }; + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(group_by), + measures, + output_names: output_names.clone(), + filters: vec![None; filters.len()], + having: None, + child: input, + }) + } + _ => unreachable!("a member is a SummaryAgg or an Aggregate"), + }; + add_candidates_per_input_parameters( + memo, + coarse.group, + rollup, + OriginOfCandidateForMemoGroup::Rollup, + ) +} + +/// The measure rolling `measure`'s finalized values up from column +/// `column` of the finer result: a sum or count by a sum, a minimum or +/// maximum by itself; `None` for any other measure (it rolls up only from +/// states). +fn rolled_up(measure: &AggIntent, column: usize) -> Option { + let col = Some(column); + Some(match measure { + AggIntent::Sum { .. } | AggIntent::Count => AggIntent::Sum { col }, + AggIntent::Min { .. } => AggIntent::Min { col }, + AggIntent::Max { .. } => AggIntent::Max { col }, + _ => return None, + }) +} + +/// The column of `lineage` that copies `column`, when exactly one does. +fn column_of(lineage: &[ColumnLineage], column: &SourceColumn) -> Option { + let mut at = lineage + .iter() + .enumerate() + .filter(|(_, known)| matches!(known, ColumnLineage::SourceColumn(c) if c == column)) + .map(|(index, _)| index); + let first = at.next()?; + at.next().is_none().then_some(first) +} + +/// The predicate selecting the rows whose column `at` meets `constraint`: +/// `at = v` or `at IN (…)`, `at NOT IN (…)`; `None` for an interval (never +/// a compensation: the time window is equal across a cluster). +fn selecting( + at: usize, + constraint: &Constraint, + semantics: ExprSemantics, +) -> Option> { + let literal = |value: &ScalarValue| ScalarExpr::Literal(value.clone()); + Some(match constraint { + Constraint::In(values) if values.len() == 1 => ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(at)), + op: CompareOpKind::Eq, + right: Box::new(literal(&values[0])), + semantics, + }, + Constraint::In(values) | Constraint::NotIn(values) => ScalarExpr::InList { + expr: Box::new(ScalarExpr::Column(at)), + list: values.iter().map(literal).collect(), + negated: matches!(constraint, Constraint::NotIn(_)), + }, + Constraint::Interval { .. } => return None, + }) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs b/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs new file mode 100644 index 000000000..912ec8eb4 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/rules/topk.rs @@ -0,0 +1,87 @@ +//! Step 2, top-k reuse: an exact top-k reads a larger top-k of the same +//! input and partition, whose ranked rows hold its own. + +use crate::rules::{add_candidate, Pass2RuleError}; +use asap_types::ir::operator::{AggIntent, Reduction}; +use asap_types::ir::{NonASAPOp, Operator}; +use asap_types::stages::{Memo, MemoGroupId, OriginOfCandidateForMemoGroup}; + +/// To each top-k group (`Aggregate` of one `TopK` measure, partitioned `by` +/// keys, without filters or `HAVING`) another top-k group over the same +/// input group and partition with the largest `k` serves, the candidate +/// `TopK(k)` over that group's ranked rows, partitioned by its leading key +/// columns: the `k` highest of the `k_max` highest rows are the `k` +/// highest. Only exact ranked rows give the reuse its answer: a ranking has +/// no error contract (`LogicalRequirements::error_propagation`), so the +/// metric rule gives an approximate producer (a heap's readout) no edge +/// when a query with an accuracy target reaches it. +pub(crate) fn add_topk_candidates(memo: &mut Memo) -> Result<(), Pass2RuleError> { + let topks: Vec<(MemoGroupId, MemoGroupId, Reduction, usize)> = (0..memo.groups().len()) + .filter_map(|index| { + let group = MemoGroupId(index as u32); + let (child, reduction, k) = topk(&memo.groups()[index].expression)?; + Some((group, child, reduction, k)) + }) + .collect(); + for (group, child, reduction, k) in &topks { + let Some((largest, k_max)) = topks + .iter() + .filter(|(_, other_child, other_reduction, _)| { + other_child == child && other_reduction == reduction + }) + .map(|(other, _, _, other_k)| (*other, *other_k)) + .max_by_key(|&(other, other_k)| (other_k, std::cmp::Reverse(other))) + else { + continue; + }; + if k_max <= *k { + continue; + } + let Operator::NonASAP(NonASAPOp::Aggregate { + measures, + output_names, + filters, + .. + }) = memo.groups()[group.0 as usize].expression.clone() + else { + continue; + }; + let keys = reduction.group_keys().map_or(0, |by| by.keys().len()); + add_candidate( + memo, + *group, + Operator::NonASAP(NonASAPOp::Aggregate { + child: largest, + reduction: Reduction::by((0..keys).collect()), + measures, + output_names, + filters, + having: None, + }), + None, + OriginOfCandidateForMemoGroup::TopKReuse, + )?; + } + Ok(()) +} + +/// The input group, partition and `k` of a top-k group: one `TopK` measure, +/// `by` keys (not `without`), no measure filter, no `HAVING`. +fn topk(operator: &Operator) -> Option<(MemoGroupId, Reduction, usize)> { + let Operator::NonASAP(NonASAPOp::Aggregate { + child, + reduction: reduction @ Reduction::Reduce(by), + measures, + filters, + having: None, + .. + }) = operator + else { + return None; + }; + let [AggIntent::TopK { k, .. }] = measures.as_slice() else { + return None; + }; + (!by.is_without() && filters.iter().all(Option::is_none)) + .then(|| (*child, reduction.clone(), *k)) +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests.rs b/crates/stage1-pass2-reuse-candidates/src/tests.rs new file mode 100644 index 000000000..9e86b2e20 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests.rs @@ -0,0 +1,1077 @@ +//! Pass 2's tests. Each builds its input memo through +//! `Memo::find_or_add_group`, `Memo::add_candidate` and `Memo::add_query`, +//! with the candidates Pass 1 would add, and reads what the rules, the +//! readout requirements, dead-candidate elimination and the edges give; no +//! other stage crate is called. +use std::collections::BTreeSet; +use std::time::Duration; + +use asap_accuracy::NoMeasuredGuarantees; +use asap_types::ir::operator::operator_properties::{ + AtModifier, BinaryOpKind, Reduction, Source, TimeShift, +}; +use asap_types::ir::operator::AggIntent; +use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; +use asap_types::ir::schema::schema_support::with_promql_series_identity; +use asap_types::ir::schema::{ + DataType, ExactKind, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, + SketchStatistic, SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, + ProjectItem, QueryRoot, ScalarExpr, TimeRangeKind, +}; +use asap_types::stages::{ + parameter_list, CandidateError, CandidateExpression, CandidateForMemoGroup, + CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalProperties, LogicalRequirements, + Memo, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, PruneCause, + PrunedLogicalCandidate, RequiredAccuracy, RootPruneReason, +}; +use asap_types::workload::{ + Predictability, Query, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RepeatedDemand, + RepetitionInterval, TimestampMs, +}; + +use crate::generate_logical_asap_memo_candidates; + +/// The examples decided on 2026-11-07. +mod decided_examples; + +// ── Fixtures ────────────────────────────────────────────────────────────── + +/// Adds `operator` with `parameters` to `group`, as Pass 1 does. +fn try_candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> Result { + let expression = CandidateExpression { operator }; + let logical_properties = LogicalProperties::new(&expression, parameters).unwrap(); + let input_requirements = + vec![LogicalRequirements::default(); expression.operator.children().len()]; + memo.add_candidate( + group, + CandidateForMemoGroup { + expression, + logical_properties, + input_requirements, + fused_groups: Vec::new(), + origin: OriginOfCandidateForMemoGroup::Pass1, + }, + ) +} + +fn candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> CandidateForMemoGroupId { + try_candidate(memo, group, operator, parameters).unwrap() +} + +/// The group of `operator`, with `operator` as its exact candidate. +fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + candidate(memo, group, operator, None); + group +} + +/// `[ts, value, labels...]`, the series identity realized as the planner +/// realizes it before Stage 0. +fn series_schema(labels: &[&str]) -> Schema { + let mut fields = vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]; + fields.extend( + labels + .iter() + .map(|n| Field::plain(*n, DataType::Utf8, true)), + ); + let open = OperatorNode::new_shared(scan_operator( + "m", + Schema::with_time_index(fields, 0, vec![]), + vec![], + )) + .unwrap(); + with_promql_series_identity(&open).unwrap().schema.clone() +} + +fn scan_operator(metric: &str, schema: Schema, predicates: Vec>) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates, + schema, + }) +} + +/// A time-series scan of `metric` listing `labels`. +fn scan(memo: &mut Memo, metric: &str, labels: &[&str]) -> MemoGroupId { + group(memo, scan_operator(metric, series_schema(labels), vec![])) +} + +/// `column = value`, PromQL semantics. +fn equals(column: usize, value: &str) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op: CompareOpKind::Eq, + right: Box::new(ScalarExpr::Literal(ScalarValue::Utf8(value.into()))), + semantics: ExprSemantics::Promql, + } +} + +/// A scan of `metric` listing `labels` whose label matchers are `matchers`. +fn scan_where( + memo: &mut Memo, + metric: &str, + labels: &[&str], + matchers: Vec>, +) -> MemoGroupId { + group(memo, scan_operator(metric, series_schema(labels), matchers)) +} + +/// `child[seconds]`. +fn range(memo: &mut Memo, child: MemoGroupId, seconds: u64) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(seconds), + kind: TimeRangeKind::Range, + child, + }), + ) +} + +/// `child offset `, or `child @ at`. +fn shift(memo: &mut Memo, child: MemoGroupId, seconds: i64, at: Option) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: seconds * 1000, + at, + }, + child, + }), + ) +} + +/// A summary build of `summary_type` over the sample values of `child`, +/// grouped by `reduction`. +fn build( + child: MemoGroupId, + summary_type: FieldDataType, + reduction: Reduction, +) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction, + filter: None, + }) +} + +/// The state group of a sketch build, one candidate per listed θ. +fn sketch_state( + memo: &mut Memo, + child: MemoGroupId, + algorithm: SketchAlgorithm, + reduction: Reduction, +) -> MemoGroupId { + let operator = build(child, FieldDataType::Sketch(algorithm), reduction); + let state = memo.find_or_add_group(operator.clone()).unwrap(); + for parameters in parameter_list(algorithm) { + candidate(memo, state, operator.clone(), Some(parameters.clone())); + } + state +} + +/// An exact accumulator state of `kind` over `window` grouped by +/// `reduction`, read by a `FinalizeExactAccumulator` group (the query +/// root): what Pass 1 offers for `sum_over_time` (per series) or for a +/// `sum by (keys)` of `count_over_time` (the count by keys). +fn exact_state( + memo: &mut Memo, + window: MemoGroupId, + kind: ExactKind, + reduction: Reduction, +) -> (MemoGroupId, MemoGroupId) { + let state = group(memo, build(window, FieldDataType::exact(kind), reduction)); + let finalized = group( + memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), + ); + (state, finalized) +} + +/// A readout group of `statistic` over `state`. +fn estimate(memo: &mut Memo, state: MemoGroupId, statistic: SketchStatistic) -> MemoGroupId { + group( + memo, + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: statistic, + }), + ) +} + +fn arithmetic( + memo: &mut Memo, + kind: ArithmeticOpKind, + lhs: MemoGroupId, + rhs: MemoGroupId, +) -> MemoGroupId { + group( + memo, + Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(kind), + vector_match: None, + }, + return_bool: false, + lhs, + rhs, + }), + ) +} + +/// Query `root` with `target` (`None`: accuracy not enforced). +fn query(memo: &mut Memo, root: MemoGroupId, target: Option) -> usize { + memo.add_query(MemoQuery { + root: QueryRoot::Operator(root), + accuracy_target: target, + values_per_evaluation: Some(1), + }) +} + +/// A query repeating every `interval_ms` at phase 0 (`FixedIntervalAt`). +fn entry(interval_ms: u32) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("q".into()), + requirements: QueryRequirements::default(), + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval_ms), + evaluation_phase: TimestampMs(0), + }), + range_evaluation: None, + } +} + +fn pass2(memo: Memo, entries: &[QueryWorkloadEntry]) -> LogicalASAPMemoCandidates { + generate_logical_asap_memo_candidates(memo, entries, &NoMeasuredGuarantees).unwrap() +} + +fn candidates_of(memo: &Memo, group: MemoGroupId) -> &[CandidateForMemoGroup] { + &memo.groups()[group.0 as usize].candidates +} + +/// The edges into input `input` of `consumer`: each producer and α^log. +fn edges_into( + logical: &LogicalASAPMemoCandidates, + consumer: CandidateForMemoGroupId, + input: usize, +) -> Vec<(CandidateForMemoGroupId, bool)> { + logical + .edges + .iter() + .filter(|edge| { + edge.consumer_candidate_id == consumer && edge.input_index_of_consumer == input + }) + .map(|edge| { + ( + edge.producer_candidate_id, + edge.satisfies_logical_requirement, + ) + }) + .collect() +} + +fn own_candidate(group: MemoGroupId) -> CandidateForMemoGroupId { + CandidateForMemoGroupId { + group, + index_in_group: 0, + } +} + +/// The pruned entry of `candidate`, if Pass 2 eliminated it. +fn pruned_entry( + logical: &LogicalASAPMemoCandidates, + candidate: CandidateForMemoGroupId, +) -> Option<&PrunedLogicalCandidate> { + logical + .pruned + .iter() + .find(|pruned| pruned.candidate == candidate) +} + +/// The required accuracy of each readout candidate of `group` (its only +/// input's). +fn requirements_of(memo: &Memo, group: MemoGroupId) -> Vec> { + candidates_of(memo, group) + .iter() + .filter(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryEstimate { .. }) + ) + }) + .map(|c| c.input_requirements[0].required_accuracy) + .collect() +} + +/// `quantile_over_time(q, m[seconds])` per series through a sketch state +/// of `algorithm`: the range group, the state group and the readout group. +fn quantile_over_time( + memo: &mut Memo, + algorithm: SketchAlgorithm, + q: f64, + seconds: u64, +) -> (MemoGroupId, MemoGroupId, MemoGroupId) { + let input = scan(memo, "m", &["job"]); + let window = range(memo, input, seconds); + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let readout = estimate(memo, state, SketchStatistic::Quantile { q }); + (window, state, readout) +} + +/// The tumbling-window widths of the chains whose merges state group +/// `state` has as candidates. +fn chain_widths_serving(memo: &Memo, state: MemoGroupId) -> BTreeSet { + candidates_of(memo, state) + .iter() + .filter_map(|c| match &c.expression.operator { + Operator::ASAP(ASAPOp::SummaryMerge { + children, + group_by: None, + }) => chain_width_below(memo, children[0]), + _ => None, + }) + .collect() +} + +/// The width of the tumbling-window chain `group` reads through +/// `TimeRange` and `TimeShift` groups, if any. +fn chain_width_below(memo: &Memo, mut group: MemoGroupId) -> Option { + loop { + match &memo.groups()[group.0 as usize].expression { + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) => return Some(*tumbling_window_width_ms), + Operator::NonASAP( + NonASAPOp::TimeRange { child, .. } | NonASAPOp::TimeShift { child, .. }, + ) => group = *child, + _ => return None, + } + } +} + +/// The groups whose expression matches `pattern`. +fn groups_where(memo: &Memo, pattern: impl Fn(&Operator) -> bool) -> Vec { + memo.groups() + .iter() + .enumerate() + .filter(|(_, group)| pattern(&group.expression)) + .map(|(index, _)| MemoGroupId(index as u32)) + .collect() +} + +/// The group whose expression is `operator`, if the memo has one. +fn group_of(memo: &Memo, operator: &Operator) -> Option { + groups_where(memo, |expression| expression == operator) + .first() + .copied() +} + +// ── Step 1: rollup from the finest common group ─────────────────────────── + +/// Concepts doc Example 1: F = `quantile by (job, host)(latency[5m])`, C = +/// `quantile by (host)(latency{job="api"}[5m])`. Equal definitions, the +/// narrowing `job ∈ {api}` on a key of F, {host} ⊆ {job, host}: C's state +/// gets `SummaryMerge(by host)(Filter(job = "api", F))`, and F is the +/// finest common group (nothing is synthesized). +#[test] +fn a_filtered_coarse_sketch_state_merges_the_finer_states_through_a_compensating_filter() { + let mut memo = Memo::default(); + let labels = ["job", "host"]; + let all = scan(&mut memo, "latency", &labels); + let api = scan_where( + &mut memo, + "latency", + &labels, + vec![Predicate(equals(2, "api"))], + ); + let fine_window = range(&mut memo, all, 300); + let coarse_window = range(&mut memo, api, 300); + let fine = sketch_state( + &mut memo, + fine_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2, 3]), + ); + let coarse = sketch_state( + &mut memo, + coarse_window, + SketchAlgorithm::Kll, + Reduction::by(vec![3]), + ); + let groups_before = memo.groups().len(); + for state in [fine, coarse] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let filter = group_of( + memo, + &Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(equals(0, "api")), + child: fine, + }), + ) + .expect("Filter(job = \"api\", F)"); + assert!(candidates_of(memo, coarse).iter().any(|c| c.origin + == OriginOfCandidateForMemoGroup::Rollup + && c.expression.operator + == Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![filter], + group_by: Some(vec![1]), + }))); + // F reads nothing from C, and no finest common group was added: the + // memo grew by the readouts, the filter, and the chains only. + assert!(candidates_of(memo, fine) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::Rollup)); + assert!(groups_where(memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryAgg { .. }) + )) + .iter() + .all(|&g| g == fine || g == coarse || g.0 as usize >= groups_before)); +} + +/// Concepts doc Example 3: Q1 = `quantile by (host)(latency[5m])`, Q2 = +/// `quantile by (job)(latency{host="xxx"}[5m])`. Neither is finer, so the +/// finest common group G* = `SummaryAgg[KLL, by job, host]` over Q1's rows +/// is synthesized with one build per listed θ, Q1's state merges it by +/// host and Q2's merges `Filter(host = "xxx", G*)` by job. +#[test] +fn a_finest_common_group_no_query_asks_for_serves_both() { + let mut memo = Memo::default(); + let labels = ["job", "host"]; + let all = scan(&mut memo, "latency", &labels); + let xxx = scan_where( + &mut memo, + "latency", + &labels, + vec![Predicate(equals(3, "xxx"))], + ); + let all_window = range(&mut memo, all, 300); + let xxx_window = range(&mut memo, xxx, 300); + let by_host = sketch_state( + &mut memo, + all_window, + SketchAlgorithm::Kll, + Reduction::by(vec![3]), + ); + let by_job = sketch_state( + &mut memo, + xxx_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2]), + ); + for state in [by_host, by_job] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let finest = group_of( + memo, + &build( + all_window, + FieldDataType::Sketch(SketchAlgorithm::Kll), + Reduction::by(vec![2, 3]), + ), + ) + .expect("G*: KLL by job, host over Q1's rows"); + assert_eq!( + candidates_of(memo, finest) + .iter() + .filter(|c| c.expression.operator == memo.groups()[finest.0 as usize].expression) + .count(), + parameter_list(SketchAlgorithm::Kll).len() + ); + let merges = |state: MemoGroupId| -> Vec> { + candidates_of(memo, state) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect() + }; + let listed = parameter_list(SketchAlgorithm::Kll).len(); + // One merge per θ of G*'s builds (A1). + assert_eq!( + merges(by_host), + vec![ + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![finest], + group_by: Some(vec![1]), + }); + listed + ] + ); + let filter = group_of( + memo, + &Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(equals(1, "xxx")), + child: finest, + }), + ) + .expect("Filter(host = \"xxx\", G*)"); + assert_eq!(candidates_of(memo, filter).len(), listed); + assert_eq!( + merges(by_job), + vec![ + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![filter], + group_by: Some(vec![0]), + }); + listed + ] + ); +} + +/// A `quantile by (job)` over `status = "500"` rows does not roll up from +/// `quantile by (job, host)` over every row: `status` is no key of the +/// finer state, whose states mix both statuses. +#[test] +fn a_narrowing_on_a_column_that_is_no_key_gives_no_rollup() { + let mut memo = Memo::default(); + let labels = ["job", "host", "status"]; + let all = scan(&mut memo, "latency", &labels); + let errors = scan_where( + &mut memo, + "latency", + &labels, + vec![Predicate(equals(4, "500"))], + ); + let all_window = range(&mut memo, all, 300); + let errors_window = range(&mut memo, errors, 300); + let fine = sketch_state( + &mut memo, + all_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2, 3]), + ); + let coarse = sketch_state( + &mut memo, + errors_window, + SketchAlgorithm::Kll, + Reduction::by(vec![2]), + ); + for state in [fine, coarse] { + let readout = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.99 }); + query(&mut memo, readout, None); + } + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + assert!(candidates_of(&logical.memo, coarse).iter().all(|c| !c + .expression + .operator + .children() + .contains(&&fine))); +} + +/// Concepts doc Example 2, exact values: `sum by (job)` rolls up the +/// finalized `sum by (job, instance)` of the same per-series rows with a +/// sum over its sum column. +#[test] +fn a_coarse_exact_sum_rolls_up_the_finer_sum() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "x", &["job", "instance"]); + let window = range(&mut memo, input, 60); + let rate = group( + &mut memo, + build( + window, + FieldDataType::exact(ExactKind::Rate), + Reduction::PerEntity, + ), + ); + let rows = group( + &mut memo, + Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: rate }), + ); + let sum_by = |memo: &mut Memo, keys: Vec, name: &str| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: rows, + reduction: Reduction::by(keys), + measures: vec![AggIntent::Sum { col: None }], + output_names: vec![name.into()], + filters: vec![None], + having: None, + }), + ) + }; + let fine = sum_by(&mut memo, vec![2, 3], "value"); + let coarse = sum_by(&mut memo, vec![2], "value"); + query(&mut memo, fine, Some(AccuracyTarget::exact())); + query(&mut memo, coarse, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let rollups: Vec<_> = candidates_of(&logical.memo, coarse) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::Rollup) + .map(|c| c.expression.operator.clone()) + .collect(); + assert_eq!( + rollups, + vec![Operator::NonASAP(NonASAPOp::Aggregate { + child: fine, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Sum { col: Some(2) }], + output_names: vec!["value".into()], + filters: vec![None], + having: None, + })] + ); +} + +// ── Step 2: top-k reuse ─────────────────────────────────────────────────── + +/// `topk(5, x)` reads `topk(10, x)`'s ranked rows: the 5 highest of the 10 +/// highest are the 5 highest. +#[test] +fn a_smaller_topk_reads_the_largest_topk_of_the_same_input() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "x", &["job"]); + let topk = |memo: &mut Memo, k: usize| { + group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: input, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::TopK { k }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ) + }; + let top5 = topk(&mut memo, 5); + let top10 = topk(&mut memo, 10); + query(&mut memo, top5, Some(AccuracyTarget::exact())); + query(&mut memo, top10, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let reuse: Vec<_> = candidates_of(&logical.memo, top5) + .iter() + .filter(|c| c.origin == OriginOfCandidateForMemoGroup::TopKReuse) + .collect(); + assert_eq!(reuse.len(), 1); + assert_eq!(reuse[0].expression.operator.children(), vec![&top10]); + assert!(candidates_of(&logical.memo, top10) + .iter() + .all(|c| c.origin != OriginOfCandidateForMemoGroup::TopKReuse)); +} + +// ── Step 3: readout requirements ────────────────────────────────────────── + +/// A single quantile query with rank error 0.01: its one readout gets that +/// requirement (λ = 1), and only the KLL builds whose rank error at δ = 1 +/// is within 0.01 satisfy it. +#[test] +fn a_single_readout_requires_the_query_target() { + let mut memo = Memo::default(); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + query(&mut memo, readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + requirements_of(&logical.memo, readout), + vec![Some(RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error: 0.01, + failure_probability: 1.0, + })] + ); + let admitted: Vec = edges_into(&logical, own_candidate(readout), 0) + .into_iter() + .filter(|&(_, satisfies)| satisfies) + .filter_map(|(producer, _)| { + match logical + .memo + .candidate(producer) + .logical_properties + .parameters() + { + Some(SketchParams::Kll { k }) => Some(*k), + _ => None, + } + }) + .collect(); + assert!(!admitted.is_empty()); + assert!(admitted.iter().all(|&k| k >= 269), "{admitted:?}"); + assert!(logical + .pruned + .iter() + .filter(|p| p.candidate.group == state) + .all(|p| p + .causes + .iter() + .all(|cause| matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. })))); +} + +/// A whole-result failure probability is split over the values one +/// evaluation returns: δ = 0.01 over 100 series is 1e-4 per value. +#[test] +fn a_whole_result_failure_probability_is_split_per_value() { + let mut memo = Memo::default(); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + values_per_evaluation: Some(100), + }); + let logical = pass2(memo, &[entry(60_000)]); + let [Some(required)] = requirements_of(&logical.memo, readout)[..] else { + panic!("one requirement"); + }; + assert!((required.failure_probability - 1e-4).abs() < 1e-12); +} + +/// Decided 2026-11-09 (N10): a whole-result failure probability with the +/// number of values per evaluation unknown gives no per-value one, so the +/// readouts the query reaches are pruned with `ValuesPerEvaluationUnknown` +/// instead of requiring δ = 0. +#[test] +fn a_whole_result_failure_probability_without_values_per_evaluation_prunes_the_readout() { + let mut memo = Memo::default(); + let (_, _, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let query = memo.add_query(MemoQuery { + root: QueryRoot::Operator(readout), + accuracy_target: Some(AccuracyTarget::max_error(0.01).failure_probability(0.01)), + values_per_evaluation: None, + }); + let logical = pass2(memo, &[entry(60_000)]); + let pruned = pruned_entry(&logical, own_candidate(readout)).expect("the readout is pruned"); + assert!(!pruned.causes.is_empty()); + for cause in &pruned.causes { + assert!( + matches!( + cause.root, + RootPruneReason::ValuesPerEvaluationUnknown { consumer, query: q } + if consumer == own_candidate(readout) && q == query + ), + "{cause:?}" + ); + } +} + +/// A ratio reads no rank error: with a relative target, the KLL readout's +/// edges into the division and from its states fail the metric rule; it +/// gets no requirement. +#[test] +fn a_ratio_reads_no_rank_error() { + let mut memo = Memo::default(); + let (_, _, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let p50 = estimate( + &mut memo, + MemoGroupId(2), + SketchStatistic::Quantile { q: 0.5 }, + ); + let ratio = arithmetic(&mut memo, ArithmeticOpKind::Div, p99, p50); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, ratio, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!(requirements_of(&logical.memo, p99), vec![None]); + // Its metric, rank, is accepted neither by the ratio nor at its own + // output: every edge into it is cut. + let pruned = pruned_entry(&logical, own_candidate(p99)).expect("pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + cause, + PruneCause { + root: RootPruneReason::MetricRule { + producer_metric: AccuracyMetric::Rank, + .. + }, + .. + } + ))); +} + +/// Decided 2026-11-09 (D1), the snapshot workload +/// `ratio-of-quantiles-1h-mvp`: `quantile_over_time(0.9, data[1h]) / +/// quantile_over_time(0.5, data[1h])` with relative error 0.05 and δ = 0.01 +/// over the whole result of 10 000 series is δ = 1e-6 per value, below the +/// grid's 1e-5. The δ levels continue in powers of ten below it, so both +/// readouts find levels whose sum stays within 1e-6 and (K) can hold. +#[test] +fn a_ratio_whose_per_value_failure_probability_is_below_the_grid_splits_it_over_its_readouts() { + let mut memo = Memo::default(); + let (_, state, p90) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 3600); + let p50 = estimate(&mut memo, state, SketchStatistic::Quantile { q: 0.5 }); + let ratio = arithmetic(&mut memo, ArithmeticOpKind::Div, p90, p50); + memo.add_query(MemoQuery { + root: QueryRoot::Operator(ratio), + accuracy_target: Some( + AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_metric(AccuracyMetric::Relative), + ), + values_per_evaluation: Some(10_000), + }); + let logical = pass2(memo, &[entry(60_000)]); + let smallest_failure_probability = |readout| { + requirements_of(&logical.memo, readout) + .into_iter() + .map(|required| { + required + .expect("every readout has a level") + .failure_probability + }) + .fold(f64::INFINITY, f64::min) + }; + let (p90_delta, p50_delta) = ( + smallest_failure_probability(p90), + smallest_failure_probability(p50), + ); + assert!( + p90_delta + p50_delta <= 1e-6, + "{p90_delta} + {p50_delta} > 1e-6" + ); +} + +/// Without accuracy targets (accuracy not enforced) readouts get no +/// requirement and no metric rule applies: every edge is admissible. +#[test] +fn without_targets_every_edge_is_admissible() { + let mut memo = Memo::default(); + let (_, _, p99) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.99, 300); + let p50 = estimate( + &mut memo, + MemoGroupId(2), + SketchStatistic::Quantile { q: 0.5 }, + ); + let ratio = arithmetic(&mut memo, ArithmeticOpKind::Div, p99, p50); + query(&mut memo, ratio, None); + let logical = pass2(memo, &[entry(60_000)]); + assert!(logical.pruned.is_empty()); + assert!(logical + .edges + .iter() + .all(|edge| edge.satisfies_logical_requirement)); +} + +/// A top-k heap smaller than k never serves the readout, whatever accuracy +/// is required: `topk(50)` reads only heaps of at least 50. +#[test] +fn a_heap_smaller_than_k_is_pruned() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "x", &["job"]); + let window = range(&mut memo, input, 300); + let state = sketch_state( + &mut memo, + window, + SketchAlgorithm::CmsWithHeap, + Reduction::by(vec![]), + ); + let readout = estimate(&mut memo, state, SketchStatistic::TopK { k: 50 }); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(60_000)]); + let mut small = 0; + for pruned in logical.pruned.iter().filter(|p| p.candidate.group == state) { + let Some(SketchParams::CmsWithHeap { heap_size, .. }) = logical + .memo + .candidate(pruned.candidate) + .logical_properties + .parameters() + else { + panic!("a heap build"); + }; + assert!(*heap_size < 50); + assert!(matches!( + pruned.causes[..], + [PruneCause { + root: RootPruneReason::HeapSmallerThanTopK { k: 50, .. }, + .. + }] + )); + small += 1; + } + // Heaps of 10 and 20, with each width and depth. + assert_eq!(small, 2 * 28); +} + +// ── Step 5: chain groups ────────────────────────────────────────────────── + +/// A mergeable state over a range gets one chain group per width with a +/// build per surviving θ; its readout reads the builds and the merges, +/// one per θ. +#[test] +fn a_mergeable_state_reads_a_chain_with_one_build_per_surviving_parameters() { + let mut memo = Memo::default(); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(120_000)]); + let memo = &logical.memo; + assert_eq!(chain_widths_serving(memo, state), BTreeSet::from([60_000])); + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) + }); + let listed = parameter_list(SketchAlgorithm::Kll).len(); + assert_eq!(candidates_of(memo, chains[0]).len(), listed); + // The builds and one merge per θ. + assert_eq!( + edges_into(&logical, own_candidate(readout), 0).len(), + 2 * listed + ); +} + +/// No chain for a summary type whose states do not merge (an instant +/// rate), under `@ end()`, or for a query without a known grid. +#[test] +fn no_chain_without_a_merge_an_anchor_or_a_grid() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let window = range(&mut memo, input, 300); + let (irate, irate_root) = + exact_state(&mut memo, window, ExactKind::IRate, Reduction::PerEntity); + let anchored = shift(&mut memo, input, 0, Some(AtModifier::End)); + let anchored_window = range(&mut memo, anchored, 300); + let (pinned, pinned_root) = exact_state( + &mut memo, + anchored_window, + ExactKind::Sum, + Reduction::PerEntity, + ); + let other = scan(&mut memo, "n", &["job"]); + let other_window = range(&mut memo, other, 300); + let (no_grid, no_grid_root) = exact_state( + &mut memo, + other_window, + ExactKind::Sum, + Reduction::PerEntity, + ); + for root in [irate_root, pinned_root, no_grid_root] { + query(&mut memo, root, None); + } + let without_phase = QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + ..entry(60_000) + }; + let logical = pass2(memo, &[entry(60_000), entry(60_000), without_phase]); + for state in [irate, pinned, no_grid] { + assert!( + chain_widths_serving(&logical.memo, state).is_empty(), + "G{}", + state.0 + ); + } + assert!(groups_where(&logical.memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + )) + .is_empty()); +} + +/// A state under `@ T` adds no width; it reads a chain another query's +/// width created when that width divides T. +#[test] +fn an_absolute_anchor_reuses_only_existing_chains() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let window = range(&mut memo, input, 300); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + let mut pinned = Vec::new(); + // T = …040 s lies on the 60 s grid; T = …010 s does not. + for at_ms in [1_700_000_040_000, 1_700_000_010_000] { + let at = shift(&mut memo, input, 0, Some(AtModifier::Timestamp(at_ms))); + let at_window = range(&mut memo, at, 300); + let (state, root) = exact_state(&mut memo, at_window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, None); + pinned.push(state); + } + let logical = pass2(memo, &vec![entry(60_000); 3]); + let widths = |state| chain_widths_serving(&logical.memo, state); + assert_eq!(widths(state), BTreeSet::from([60_000])); + assert_eq!(widths(pinned[0]), BTreeSet::from([60_000])); + assert!(widths(pinned[1]).is_empty()); +} + +// ── Step 6: logical edges and evaluation grids ──────────────────────────── + +/// Every candidate of an input group is an edge of each consumer input, +/// except that a candidate carrying a θ without building (a merge, a +/// `TimeRange` over a chain) reads only producers with its θ (A1). +#[test] +fn every_candidate_of_an_input_group_is_an_edge_of_matching_parameters() { + let mut memo = Memo::default(); + let (_, state, readout) = quantile_over_time(&mut memo, SketchAlgorithm::DDSketch, 0.9, 300); + query(&mut memo, readout, None); + let logical = pass2(memo, &[entry(300_000)]); + let listed = parameter_list(SketchAlgorithm::DDSketch).len(); + // The builds and one merge per θ over the 300 s chain. + assert_eq!( + edges_into(&logical, own_candidate(readout), 0).len(), + 2 * listed + ); + let memo = &logical.memo; + for (index_in_group, merge) in candidates_of(memo, state).iter().enumerate() { + if !matches!( + merge.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + ) { + continue; + } + let consumer = CandidateForMemoGroupId { + group: state, + index_in_group, + }; + let producers = edges_into(&logical, consumer, 0); + assert_eq!(producers.len(), 1); + assert_eq!( + memo.candidate(producers[0].0) + .logical_properties + .parameters(), + merge.logical_properties.parameters() + ); + } +} + +/// A group's evaluation grid is the gcd of its reaching queries' grids; +/// `None` when one has no known grid. +#[test] +fn a_groups_evaluation_grid_is_the_gcd_of_its_queries_grids() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let window = range(&mut memo, input, 300); + let (_, a) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + let other = scan(&mut memo, "n", &["job"]); + query(&mut memo, a, None); + query(&mut memo, a, None); + query(&mut memo, other, None); + let without_phase = QueryWorkloadEntry { + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + ..entry(60_000) + }; + let logical = pass2(memo, &[entry(60_000), entry(120_000), without_phase]); + let grid = |group: MemoGroupId| logical.evaluation_grid_of_group[group.0 as usize]; + assert_eq!(grid(window).map(|g| g.period_ms), Some(60_000)); + assert_eq!(grid(other), None); +} diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs b/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs new file mode 100644 index 000000000..f05798f19 --- /dev/null +++ b/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs @@ -0,0 +1,515 @@ +//! The examples decided on 2026-11-07 (rule order, offsets in the gcd, +//! dead-candidate elimination with its causes, one range-free chain group, +//! constant error contracts, the mean of quantile readouts), each asserting +//! only the decided structure: the memo's groups and candidates, the +//! logical edges and Pass 2's pruned list. +use super::*; + +/// The readouts Pass 1 offers one quantile group, per summary type. +struct QuantileReadouts { + /// The `Aggregate` group `quantile_over_time(q, window)` per series. + result: MemoGroupId, + /// Per algorithm: its state group and the readout candidate of `result`. + by_algorithm: Vec<(SketchAlgorithm, MemoGroupId, CandidateForMemoGroupId)>, +} + +/// `quantile_over_time(q, window)` per series as Pass 1 leaves it: the +/// aggregate group with its exact candidate and, per algorithm, a state +/// group (one build per listed θ) and a candidate of the aggregate group +/// reading it (through a renaming `Project` over a readout group when the +/// column names differ, as Pass 1's `add_output_candidate` does). +fn quantile_over_time_readouts( + memo: &mut Memo, + window: MemoGroupId, + q: f64, + algorithms: &[SketchAlgorithm], +) -> QuantileReadouts { + let result = group( + memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: window, + reduction: Reduction::PerEntity, + measures: vec![AggIntent::Quantile { col: None, q }], + output_names: vec![String::new()], + filters: vec![None], + having: None, + }), + ); + let mut by_algorithm = Vec::new(); + for &algorithm in algorithms { + let state = sketch_state(memo, window, algorithm, Reduction::PerEntity); + let direct = Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q }, + }); + let id = match try_candidate(memo, result, direct, None) { + Ok(id) => id, + Err(CandidateError::OutputDiffersFromGroup { .. }) => { + let readout = estimate(memo, state, SketchStatistic::Quantile { q }); + let to = &memo.groups()[result.0 as usize].group_properties.schema; + let renamed = Operator::NonASAP(NonASAPOp::Project { + cols: to + .fields + .iter() + .enumerate() + .map(|(column, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(column), + }) + .collect(), + child: readout, + }); + candidate(memo, result, renamed, None) + } + Err(error) => panic!("{error}"), + }; + by_algorithm.push((algorithm, state, id)); + } + QuantileReadouts { + result, + by_algorithm, + } +} + +/// The readout of `state` a candidate `id` of a quantile group is or reads. +fn readout_of(memo: &Memo, id: CandidateForMemoGroupId) -> CandidateForMemoGroupId { + match &memo.candidate(id).expression.operator { + Operator::NonASAP(NonASAPOp::Project { child, .. }) => own_candidate(*child), + _ => id, + } +} + +// ── (1) Rule order and reach (E3) ───────────────────────────────────────── + +/// Decided 2026-11-07 (E3, rule order): chain groups come after the rollup +/// rule and the reach along candidate edges, so a state's widths come from +/// every query that reaches it through a candidate. +/// +/// A `sum by (job, instance) (count_over_time(m[5m]))` every 60 s; B +/// `sum by (job) (count_over_time(m[5m]))` every 90 s (both at phase 0). +/// B's state rolls up A's, so both reach A's state, whose widths are the +/// gcd closure of {60 s, 90 s} with 300 s: 30 s and 60 s. +#[test] +fn e3_chain_groups_run_after_reach_so_a_rolled_up_query_adds_its_grid() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job", "instance"]); + let window = range(&mut memo, input, 300); + let (by_job_instance, a) = exact_state( + &mut memo, + window, + ExactKind::Count, + Reduction::by(vec![2, 3]), + ); + let (by_job, b) = exact_state(&mut memo, window, ExactKind::Count, Reduction::by(vec![2])); + let query_a = query(&mut memo, a, Some(AccuracyTarget::exact())); + let query_b = query(&mut memo, b, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000), entry(90_000)]); + let memo = &logical.memo; + assert!( + candidates_of(memo, by_job).iter().any(|c| { + c.origin == OriginOfCandidateForMemoGroup::Rollup + && c.expression.operator.children() == vec![&by_job_instance] + }), + "B's state reads A's through a rollup" + ); + let reached_by = memo.groups()[by_job_instance.0 as usize].reached_by_queries(); + assert!( + reached_by.contains(&query_a) && reached_by.contains(&query_b), + "A's state is reached by A and B: {reached_by:?}" + ); + assert_eq!( + chain_widths_serving(memo, by_job_instance), + BTreeSet::from([30_000, 60_000]) + ); + assert_eq!(chain_widths_serving(memo, by_job), BTreeSet::from([30_000])); +} + +// ── (2) Offset in the gcd (E3) ──────────────────────────────────────────── + +/// Decided 2026-11-07 (E3, offsets): a window read at an `offset` takes the +/// offset into the gcd, since its windows' edges move by it. +/// `sum_over_time(m[5m] offset 45s)` every 60 s (phase 0): widths divide +/// 60 s, 300 s and 45 s, so 15 s only. +#[test] +fn e3_an_offset_enters_the_gcd_of_the_widths() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let shifted = shift(&mut memo, input, 45, None); + let window = range(&mut memo, shifted, 300); + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, Some(AccuracyTarget::exact())); + let logical = pass2(memo, &[entry(60_000)]); + assert_eq!( + chain_widths_serving(&logical.memo, state), + BTreeSet::from([15_000]) + ); +} + +// ── (3) Dead-candidate elimination and Pass 2's pruned list ────────────── + +/// Decided 2026-11-07, concepts doc example "ratio of quantiles": +/// `quantile_over_time(0.99, m[5m]) / quantile_over_time(0.5, m[5m])`, +/// relative error 0.05, every 60 s; Pass 1 offers KLL and DDSketch to both +/// quantiles. The ratio carries only relative error, so each KLL readout is +/// pruned by the metric rule and every KLL build through it. The readouts +/// of DDSketch get the levels 0.02 … 0.001 (cap 0.045), so the builds with +/// alpha 0.1 and 0.05 miss every level they could serve. Chain groups are +/// added only for the surviving builds. +#[test] +fn dead_candidates_are_pruned_and_listed_before_chain_groups() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let window = range(&mut memo, input, 300); + let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &algorithms); + let p50 = quantile_over_time_readouts(&mut memo, window, 0.5, &algorithms); + let ratio = arithmetic(&mut memo, ArithmeticOpKind::Div, p99.result, p50.result); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, ratio, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + let memo = &logical.memo; + let (kll_state, ddsketch_state) = (p99.by_algorithm[0].1, p99.by_algorithm[1].1); + // Each KLL readout: the metric rule at the ratio, directly. + for quantile in [&p99, &p50] { + let kll_readout = quantile.by_algorithm[0].2; + let pruned = pruned_entry(&logical, kll_readout).expect("the KLL readout is pruned"); + assert!(pruned.causes.iter().all(|cause| matches!( + &cause.root, + RootPruneReason::MetricRule { producer_metric: AccuracyMetric::Rank, accepted_metrics, .. } + if accepted_metrics == &vec![AccuracyMetric::Relative] + ))); + } + // Every KLL build: the metric rule on its edge into a KLL readout, + // whose output accepts only relative error (A2). + for index_in_group in 0..parameter_list(SketchAlgorithm::Kll).len() { + let build = CandidateForMemoGroupId { + group: kll_state, + index_in_group, + }; + let pruned = pruned_entry(&logical, build).expect("every KLL build is pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + &cause.root, + RootPruneReason::MetricRule { producer_metric: AccuracyMetric::Rank, accepted_metrics, .. } + if accepted_metrics == &vec![AccuracyMetric::Relative] + ))); + } + // DDSketch: alpha 0.1 and 0.05 pruned below every required accuracy; + // the levels of the readouts are 0.02 … 0.001. + let pruned_alphas: Vec = logical + .pruned + .iter() + .filter(|pruned| pruned.candidate.group == ddsketch_state) + .map(|pruned| { + assert!(pruned + .causes + .iter() + .all(|cause| cause.through_pruned_candidates.is_empty() + && matches!(cause.root, RootPruneReason::BelowRequiredAccuracy { .. }))); + match memo + .candidate(pruned.candidate) + .logical_properties + .parameters() + { + Some(SketchParams::DDSketch { alpha }) => *alpha, + other => panic!("{other:?}"), + } + }) + .collect(); + assert_eq!(pruned_alphas, vec![0.1, 0.05]); + let readout_group = readout_of(memo, p99.by_algorithm[1].2).group; + let mut levels: Vec = requirements_of(memo, readout_group) + .into_iter() + .map(|required| required.expect("a DDSketch readout's level").max_error) + .collect(); + levels.sort_by(f64::total_cmp); + assert_eq!(levels, vec![0.001, 0.002, 0.005, 0.01, 0.02]); + // Chain builds exist only for the five surviving alphas. + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. }) + if *summary_type == FieldDataType::Sketch(SketchAlgorithm::DDSketch)) + }); + assert_eq!(chains.len(), 1); + assert_eq!(candidates_of(memo, chains[0]).len(), 5); + assert!(groups_where(memo, |e| matches!( + e, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. }) + if *summary_type == FieldDataType::Sketch(SketchAlgorithm::Kll) + )) + .is_empty()); + // No logical edge names a pruned candidate. + let pruned: BTreeSet<_> = logical.pruned.iter().map(|p| p.candidate).collect(); + assert!(logical + .edges + .iter() + .all(|edge| !pruned.contains(&edge.producer_candidate_id) + && !pruned.contains(&edge.consumer_candidate_id))); + let _ = kll_state; +} + +// ── (4) One range-free chain group across ranges and offsets (E2) ───────── + +/// Decided 2026-11-07 (E2, design (c)): a tumbling-window chain is a group +/// of its own, `SummaryTumblingWindowChain` over the rows with a width and +/// no range, offset or window count, so every range and offset of one rows +/// and width reads the same chain group through `TimeRange` (and +/// `TimeShift`) groups and a `SummaryMerge`. +/// +/// A `sum_over_time(m[5m])`, B `sum_over_time(m[5m] offset 1h)`, C +/// `sum_over_time(m[1h])`, each every 60 s (phase 0): one chain group Gw of +/// width 60 s over the scan of m; the groups `TimeShift(1h, Gw)`, +/// `TimeRange(5m, Gw)`, `TimeRange(5m, TimeShift(1h, Gw))` and +/// `TimeRange(1h, Gw)`; each state group has a `SummaryMerge` candidate +/// over its `TimeRange` group. +#[test] +fn e2_one_range_free_chain_group_serves_every_range_and_offset() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let five_minutes = range(&mut memo, input, 300); + let shifted_input = shift(&mut memo, input, 3600, None); + let five_minutes_an_hour_ago = range(&mut memo, shifted_input, 300); + let one_hour = range(&mut memo, input, 3600); + let mut states = Vec::new(); + for window in [five_minutes, five_minutes_an_hour_ago, one_hour] { + let (state, root) = exact_state(&mut memo, window, ExactKind::Sum, Reduction::PerEntity); + query(&mut memo, root, Some(AccuracyTarget::exact())); + states.push(state); + } + let logical = pass2(memo, &vec![entry(60_000); 3]); + let memo = &logical.memo; + let chains = groups_where(memo, |e| { + matches!(e, Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. })) + }); + let [chain] = chains[..] else { + panic!("one chain group, found {chains:?}"); + }; + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + tumbling_window_width_ms, + .. + }) = &memo.groups()[chain.0 as usize].expression + else { + unreachable!(); + }; + assert_eq!(*tumbling_window_width_ms, 60_000); + assert_eq!( + *child, input, + "the chain reads the scan: no range or shift below it" + ); + let range_over = |range_s: u64, child: MemoGroupId| { + group_of( + memo, + &Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(range_s), + kind: TimeRangeKind::Range, + child, + }), + ) + }; + let shifted_chain = group_of( + memo, + &Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 3_600_000, + at: None, + }, + child: chain, + }), + ) + .expect("TimeShift(1h, Gw)"); + let ranges = [ + range_over(300, chain).expect("TimeRange(5m, Gw)"), + range_over(300, shifted_chain).expect("TimeRange(5m, TimeShift(1h, Gw))"), + range_over(3600, chain).expect("TimeRange(1h, Gw)"), + ]; + for (state, range) in states.iter().zip(ranges) { + assert!( + candidates_of(memo, *state).iter().any(|c| matches!( + &c.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { children, group_by: None }) + if *children == vec![range] + )), + "G{} reads G{} through a SummaryMerge", + state.0, + range.0 + ); + } +} + +// ── (5) Constant error contracts (E1) ───────────────────────────────────── + +/// Decided 2026-11-07 (E1): a `Filter` whose predicate reads only label +/// columns keeps or drops whole rows, so a value's error passes through it +/// unchanged. `quantile_over_time(0.99, latency[5m])` behind `job = "api"`, +/// relative error 0.05: the DDSketch readout keeps an admissible edge into +/// the filter. (The contract alone is tested in `asap_types::stages`.) +#[test] +fn e1_a_key_only_filter_keeps_the_ddsketch_readout_admissible() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "latency", &["job"]); + let window = range(&mut memo, input, 300); + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &[SketchAlgorithm::DDSketch]); + let job = memo.groups()[p99.result.0 as usize] + .group_properties + .schema + .fields + .iter() + .position(|field| field.name == "job") + .expect("the result keeps the job label"); + let filtered = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Filter { + pred: Predicate(equals(job, "api")), + child: p99.result, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, filtered, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + let ddsketch_readout = p99.by_algorithm[0].2; + let edges = edges_into(&logical, own_candidate(filtered), 0); + assert!( + edges.contains(&(ddsketch_readout, true)), + "edges into the filter: {edges:?}" + ); +} + +// ── (6) H2: avg over relative errors stays exact-only ───────────────────── + +/// Decided 2026-11-07 (H2): `avg` has an absolute contract only here; a +/// relative bound on a mean needs the sign facts of #1207 PR 8. So +/// `avg(quantile_over_time(0.99, m[5m]))` with a relative target has no +/// admissible edge from a sketch readout into the `avg`: it reads the +/// exact quantile only. +#[test] +fn h2_avg_of_readouts_under_a_relative_target_reads_exact_only() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let window = range(&mut memo, input, 300); + let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; + let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &algorithms); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: p99.result, + reduction: Reduction::by(vec![]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::Relative); + query(&mut memo, avg, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + let edges = edges_into(&logical, own_candidate(avg), 0); + for (_, _, readout) in &p99.by_algorithm { + assert!( + !edges.contains(&(*readout, true)), + "a sketch readout is admissible into avg: {edges:?}" + ); + } + assert!( + edges.contains(&(own_candidate(p99.result), true)), + "the exact quantile stays admissible: {edges:?}" + ); +} + +// ── Decided 2026-11-09 ──────────────────────────────────────────────────── + +/// Decided 2026-11-09 (A1): a merge carries the θ of the states it reads. +/// P = `quantile_over_time(0.9, m[5m])` at rank 0.005 keeps KLL builds +/// from k = 547; Q = `quantile_over_time(0.9, m[10m])` at rank 0.01 from +/// k = 269. Both read one 60 s chain, which has the builds of both, so P's +/// state gets `SummaryMerge[k=269]` and `SummaryMerge[k=547]`: the second +/// serves P's readout, the first is pruned with `BelowRequiredAccuracy`. +#[test] +fn a1_a_merge_is_judged_by_the_parameters_it_carries() { + let mut memo = Memo::default(); + let (_, p_state, p_readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 300); + let (_, _, q_readout) = quantile_over_time(&mut memo, SketchAlgorithm::Kll, 0.9, 600); + query(&mut memo, p_readout, Some(AccuracyTarget::max_error(0.005))); + query(&mut memo, q_readout, Some(AccuracyTarget::max_error(0.01))); + let logical = pass2(memo, &[entry(60_000), entry(60_000)]); + let memo = &logical.memo; + let merge_with_k = |k: u32| -> CandidateForMemoGroupId { + let index_in_group = candidates_of(memo, p_state) + .iter() + .position(|c| { + matches!( + c.expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + ) && c.logical_properties.parameters() == Some(&SketchParams::Kll { k }) + }) + .unwrap_or_else(|| panic!("SummaryMerge[k={k}] in P's state group")); + CandidateForMemoGroupId { + group: p_state, + index_in_group, + } + }; + let into_p = edges_into(&logical, own_candidate(p_readout), 0); + assert!(into_p.contains(&(merge_with_k(547), true)), "{into_p:?}"); + let pruned = pruned_entry(&logical, merge_with_k(269)).expect("SummaryMerge[k=269] is pruned"); + assert!(!pruned.causes.is_empty()); + assert!(pruned.causes.iter().all(|cause| matches!( + cause.root, + RootPruneReason::BelowRequiredAccuracy { + required: RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error, + .. + }, + .. + } if max_error == 0.005 + ))); +} + +/// Decided 2026-11-09 (A2): the metric rule cuts edges, not candidates. +/// `avg by (job)(q0.5 / q0.5 + q0.9)` at absolute error 0.05, each +/// quantile per series with KLL, DDSketch and exact candidates: the `+` +/// carries absolute error only, so the division's inputs accept no metric +/// and the edges from the DDSketch and KLL readouts into it are cut; the +/// exact division stays, read from the exact quantile, and its edge into +/// the `+` holds. +#[test] +fn a2_the_metric_rule_cuts_the_readout_edges_and_keeps_the_exact_division() { + let mut memo = Memo::default(); + let input = scan(&mut memo, "m", &["job"]); + let window = range(&mut memo, input, 300); + let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; + let p50 = quantile_over_time_readouts(&mut memo, window, 0.5, &algorithms); + let p90 = quantile_over_time_readouts(&mut memo, window, 0.9, &algorithms); + let ratio = arithmetic(&mut memo, ArithmeticOpKind::Div, p50.result, p50.result); + let sum = arithmetic(&mut memo, ArithmeticOpKind::Add, ratio, p90.result); + let avg = group( + &mut memo, + Operator::NonASAP(NonASAPOp::Aggregate { + child: sum, + reduction: Reduction::by(vec![0]), + measures: vec![AggIntent::Avg { col: None }], + output_names: vec!["avg".into()], + filters: vec![None], + having: None, + }), + ); + let target = AccuracyTarget::max_error(0.05).accuracy_metric(AccuracyMetric::ABSOLUTE); + query(&mut memo, avg, Some(target)); + let logical = pass2(memo, &[entry(60_000)]); + let division = own_candidate(ratio); + assert!( + pruned_entry(&logical, division).is_none(), + "{:?}", + logical.pruned + ); + assert!(edges_into(&logical, own_candidate(sum), 0).contains(&(division, true))); + for input in 0..2 { + let into_division = edges_into(&logical, division, input); + assert!(into_division.contains(&(own_candidate(p50.result), true))); + for (_, _, readout) in &p50.by_algorithm { + assert!( + !into_division.contains(&(*readout, true)), + "{into_division:?}" + ); + } + } +} diff --git a/crates/types/src/ir/properties/accuracy.rs b/crates/types/src/ir/properties/accuracy.rs index 4ae1db96b..d862f1b12 100644 --- a/crates/types/src/ir/properties/accuracy.rs +++ b/crates/types/src/ir/properties/accuracy.rs @@ -59,6 +59,14 @@ impl AccuracyMetric { pub const ABSOLUTE_STREAM_L2: Self = Self::Absolute { scale: Some(NormScale::StreamL2), }; + /// Every metric, for a contract or a target read in each of them. + pub const ALL: [Self; 5] = [ + Self::ABSOLUTE, + Self::ABSOLUTE_STREAM_L1, + Self::ABSOLUTE_STREAM_L2, + Self::Relative, + Self::Rank, + ]; } /// The wire name of an [`AccuracyMetric`]. diff --git a/crates/types/src/ir/properties/summary_coverage.rs b/crates/types/src/ir/properties/summary_coverage.rs index 24b681ddc..2fafbd998 100644 --- a/crates/types/src/ir/properties/summary_coverage.rs +++ b/crates/types/src/ir/properties/summary_coverage.rs @@ -64,7 +64,6 @@ pub struct SummaryCoverage { pub(crate) selection: Vec, } -#[cfg(any(test, feature = "test-support"))] impl SummaryCoverage { pub fn definition(&self) -> &SummaryStateDefinition { &self.definition @@ -1223,8 +1222,8 @@ impl Constraint { } } -#[cfg(any(test, feature = "test-support"))] impl SelectionBox { + #[cfg(any(test, feature = "test-support"))] pub fn new(columns: BTreeMap) -> Self { Self { columns } } diff --git a/crates/types/src/stages/candidates.rs b/crates/types/src/stages/candidates.rs index 8d4768da8..a5d87e200 100644 --- a/crates/types/src/stages/candidates.rs +++ b/crates/types/src/stages/candidates.rs @@ -8,17 +8,21 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use super::column_properties::{columns_of_group, output_column_properties}; use super::hash_consing::{expression_hash, same_expression, same_value}; -use super::{GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId, ValueSign}; +use super::{ColumnLineage, GroupProperties, MemoGroup, MemoGroupAsInput, MemoGroupId, ValueSign}; use crate::ir::operator::asap::summary_type_of_operator; +use crate::ir::operator::operator_properties::BinaryOpKind; use crate::ir::operator::operator_properties::{AtModifier, SourceColumn, TimeShift}; +use crate::ir::operator::AggIntent; use crate::ir::properties::summary_coverage::SummaryCoverage; use crate::ir::properties::{AccuracyMetric, AccuracyTarget}; -use crate::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; +use crate::ir::scalar::expr_ir::ArithmeticOpKind; +use crate::ir::schema::{ExactKind, FieldDataType, SketchAlgorithm, SketchParams}; use crate::ir::ASAPOp; use crate::ir::Operator; +use crate::ir::OperatorResultKind; use crate::ir::QueryRoot; use crate::ir::SchemaDerivationError; -use crate::ir::{NonASAPOp, ScalarExpr}; +use crate::ir::{BinaryOperator, NonASAPOp, ScalarExpr}; /// One candidate: position `index_in_group` in group `group`'s candidates /// ([`MemoGroup::candidates`]). @@ -140,15 +144,18 @@ impl Memo { /// Adds `candidate` to `group`'s candidates and returns its id, unless /// the group already has a candidate with the same expression - /// (`same_expression`) and logical properties: then that candidate's - /// id, and `candidate` is dropped. A new candidate makes `group` a parent + /// (`same_expression`), logical properties and input requirements: then + /// that candidate's id, and `candidate` is dropped. A new candidate makes `group` a parent /// of each of its input groups, and `group`'s queries reach each input /// group and every group below it. /// /// Fails, leaving the memo unchanged, when the candidate's expression /// breaks the node rules a group's does ([`derive_group_properties`]), - /// or when its output schema or kind differs from the group's: then it - /// does not compute the group's result. + /// when its output schema or kind differs from the group's (then it + /// does not compute the group's result), or when its θ does not fit the + /// group's output: a candidate of a sketch state group (a build, or a + /// merge, `Filter`, `TimeRange` or `TimeShift` passing states on) has a + /// θ from that algorithm's list, any other candidate none. pub fn add_candidate( &mut self, group: MemoGroupId, @@ -162,10 +169,15 @@ impl Memo { { return Err(CandidateError::OutputDiffersFromGroup { group }); } + parameters_fit( + sketch_algorithm_of_state(group_properties), + candidate.logical_properties.parameters(), + ) + .map_err(CandidateError::LogicalProperties)?; let candidates = &mut self.groups[group.0 as usize].candidates; if let Some(index_in_group) = candidates .iter() - .position(|known| same_expression_and_logical_properties(known, &candidate)) + .position(|known| same_candidate(known, &candidate)) { return Ok(CandidateForMemoGroupId { group, @@ -248,6 +260,34 @@ impl Memo { candidate.input_requirements = input_requirements; } + /// The candidates `consumer` may read on input `input`: those of the + /// input group, except that a consumer with a θ (a build, or a candidate + /// passing states on) reads only producers with the same θ or none. + pub fn producers( + &self, + consumer: CandidateForMemoGroupId, + input: usize, + ) -> Vec { + let candidate = self.candidate(consumer); + let group = *candidate.expression.operator.children()[input]; + let theta = candidate.logical_properties.parameters(); + self.groups[group.0 as usize] + .candidates + .iter() + .enumerate() + .filter( + |(_, producer)| match (theta, producer.logical_properties.parameters()) { + (Some(theta), Some(produced)) => theta == produced, + _ => true, + }, + ) + .map(|(index_in_group, _)| CandidateForMemoGroupId { + group, + index_in_group, + }) + .collect() + } + /// The groups, each after every group it or one of its candidates /// reads, the lower index first among those ready: a dependency order for /// candidate traversal, whatever order rules appended groups in. `None` @@ -382,6 +422,26 @@ pub enum CandidateError { /// not compute the group's result (a bug in the rule that built it). #[error("the candidate's output differs from group {group:?}'s")] OutputDiffersFromGroup { group: MemoGroupId }, + /// Its θ does not fit the group's output (a bug in the rule that built + /// it). + #[error("the candidate's parameters do not fit its group's output: {0}")] + LogicalProperties(LogicalPropertiesError), +} + +/// The sketch algorithm of the state a group outputs; `None` for rows, +/// values and exact states. +fn sketch_algorithm_of_state(group: &GroupProperties) -> Option { + if group.result_kind != OperatorResultKind::State { + return None; + } + group + .schema + .fields + .iter() + .find_map(|field| match field.dtype { + FieldDataType::Sketch(algorithm) => Some(algorithm), + _ => None, + }) } /// Lists `parent` among `group`'s parents, once. @@ -392,17 +452,16 @@ fn add_parent(group: &mut MemoGroup, parent: MemoGroupId) { } /// Whether `left` and `right` are one candidate of a group: the same -/// expression ([`same_expression`]) and logical properties (parameters by -/// [`same_value`]). -fn same_expression_and_logical_properties( - left: &CandidateForMemoGroup, - right: &CandidateForMemoGroup, -) -> bool { +/// expression ([`same_expression`]), logical properties (parameters by +/// [`same_value`]) and input requirements, so two readouts of one state +/// with different required accuracy are two candidates. +fn same_candidate(left: &CandidateForMemoGroup, right: &CandidateForMemoGroup) -> bool { same_expression(&left.expression.operator, &right.expression.operator) && same_value( &left.logical_properties.parameters, &right.logical_properties.parameters, ) + && left.input_requirements == right.input_requirements } /// $v = (e_v, p^{\log}_v)$: one candidate of a memo group. @@ -433,17 +492,13 @@ pub struct CandidateForMemoGroup { pub enum OriginOfCandidateForMemoGroup { /// Pass 1: exact, exact accumulator, or a summary type. Pass1, - /// Canonical value preprocessing feeding a summary build. - SummaryInput, /// A tumbling-window chain, or a merge reading one, of a summary state group. TumblingWindows, - /// A coarse group regrouping a finer group. + /// A coarse group rolled up from a finer one, the compensating `Filter` + /// it reads, or a build of a finest common group. Rollup, /// A top-k read from another group's ranking. TopKReuse, - /// Its metric's scans widened to a shared label set, read through a - /// `Project` (scan widening). - WidenedScans, } /// $e_v = \operatorname{op}_v(G_{v,1}, \dots, G_{v,n})$: one operator, with @@ -456,9 +511,13 @@ pub struct CandidateExpression { /// $p^{\log}_v = \boldsymbol\theta_v$. The summary type is the operator's /// argument, not a property, and a tumbling-window chain's width is the -/// chain operator's argument. [`Self::new`] rejects parameters of another -/// algorithm, not in the algorithm's list, missing on a sketch build (a chain -/// included) or present on an exact operator. +/// chain operator's argument. A candidate passing states on without building +/// them (`SummaryMerge`, and a `Filter`, `TimeRange` or `TimeShift` over +/// states) carries the θ of the states it reads. [`Self::new`] rejects +/// parameters of another algorithm, not in the algorithm's list or missing +/// on a sketch build (a chain included), and parameters on an operator that +/// neither builds nor passes on states; [`Memo::add_candidate`] checks a +/// pass-through's θ against its group's state. #[derive(Debug, Clone)] pub struct LogicalProperties { /// $\boldsymbol\theta_v$, from the algorithm's predefined list; `None` for @@ -471,19 +530,21 @@ impl LogicalProperties { expression: &CandidateExpression, parameters: Option, ) -> Result { - let algorithm = match summary_type_of_operator(&expression.operator) { - Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), - _ => None, - }; - match (algorithm, ¶meters) { - (Some(_), None) => return Err(LogicalPropertiesError::SketchBuildWithoutParameters), - (None, Some(_)) => return Err(LogicalPropertiesError::ParametersOnExactOperator), - (Some(algorithm), Some(parameters)) => { - if !parameter_list(algorithm).contains(parameters) { - return Err(LogicalPropertiesError::ParametersNotInList); - } - } - (None, None) => {} + let passes_states_on = matches!( + expression.operator, + Operator::ASAP(ASAPOp::SummaryMerge { .. }) + | Operator::NonASAP( + NonASAPOp::Filter { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + ) + ); + if !passes_states_on { + let algorithm = match summary_type_of_operator(&expression.operator) { + Some(FieldDataType::Sketch(algorithm)) => Some(*algorithm), + _ => None, + }; + parameters_fit(algorithm, parameters.as_ref())?; } Ok(Self { parameters }) } @@ -494,13 +555,29 @@ impl LogicalProperties { } } +/// Whether θ `parameters` fit an output of sketch `algorithm` (`None`: no +/// sketch state): from its list, and present exactly for a sketch. +fn parameters_fit( + algorithm: Option, + parameters: Option<&SketchParams>, +) -> Result<(), LogicalPropertiesError> { + match (algorithm, parameters) { + (Some(_), None) => Err(LogicalPropertiesError::SketchBuildWithoutParameters), + (None, Some(_)) => Err(LogicalPropertiesError::ParametersOnExactOperator), + (Some(algorithm), Some(parameters)) if !parameter_list(algorithm).contains(parameters) => { + Err(LogicalPropertiesError::ParametersNotInList) + } + _ => Ok(()), + } +} + /// Logical properties that do not fit their operator: bad input to /// [`LogicalProperties::new`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum LogicalPropertiesError { #[error("an exact operator has no summary parameters")] ParametersOnExactOperator, - #[error("a sketch build needs parameters from its algorithm's list")] + #[error("a sketch build, or a candidate passing sketch states on, needs parameters from its algorithm's list")] SketchBuildWithoutParameters, #[error("the parameters are not in their algorithm's predefined list")] ParametersNotInList, @@ -510,10 +587,10 @@ pub enum LogicalPropertiesError { /// error of each, for a statistic at a failure probability, is its guarantee /// (`asap_accuracy::theoretical_guarantee`). /// -/// The widths, `k`s, precisions, `alpha`s and heap sizes are today's sizing -/// (`asap_accuracy`'s test-only `size_params`) over the errors ε = 0.1, -/// 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, without -/// repeats; an `asap_accuracy` test keeps them equal. Distinct counts (KMV, +/// The widths, `k`s, precisions, `alpha`s and heap sizes span the errors +/// ε = 0.1, 0.05, 0.02, 0.01, 0.005, 0.002 and 0.001, smallest first, +/// without repeats; an `asap_accuracy` test checks every entry has a +/// guarantee at δ = 0.01. Distinct counts (KMV, /// and UnivMon's distinct-count shapes) stop at ε = 0.01. Where δ is set by /// a structural parameter, the list covers the δ levels with it: Count-Min /// lists each width with each depth of [`CMS_DEPTHS`], Count Sketch with @@ -771,18 +848,172 @@ pub struct RequiredAccuracy { impl LogicalRequirements { /// $g_c$'s linear bound for errors in `metric`, each input's and the /// output's: $\beta_{c,i}$ and $b_{c,i}$ per input, read from the - /// operator and its arguments. `None` when the consumer has no error - /// contract for `metric`: an input whose error is in `metric` then gives - /// no edge (the metric rule). No factor is guessed. + /// operator, its arguments and, for a `Filter`, its input group's column + /// lineage in `memo`. `None` when the consumer has no error contract for + /// `metric`: an input whose error is in `metric` then gives no edge (the + /// metric rule). No factor is guessed; every contract keeps the metric + /// (concepts doc, section 4, "Local propagation"): + /// - value-preserving (a `Project` of columns and literals, `Sort`, + /// `TimeRange`, `TimeShift`, `Concat`, a readout, a finalize, a + /// `SummaryMerge`): $\beta = 1$ in any metric; + /// - a `Filter` whose predicate reads only columns copied from a source + /// (lineage `SourceColumn`, labels and keys): $\beta = 1$ in any metric; + /// - a `Project` whose computed items are `abs`, `clamp`, `clamp_min`, + /// `clamp_max` or a negation of a column: absolute, $\beta = 1$; + /// - an `Aggregate` (or exact accumulator build) of counts ($\beta = 0$), + /// min and max (absolute or relative, $\beta = 1$) and avg (absolute, + /// $\beta = 1$), without measure filters or `HAVING`; + /// - `+` and `-`: absolute, $\beta = 1$; `/`: relative, + /// $\beta = 1/(1-b)$ on both, the denominator capped at $b$; `*`: + /// relative, $(1+b, 1)$, the right operand capped at $b$ + /// ($(1+x)(1+y)-1 \le (1+b)\,x + y$ for $y \le b$), with $b$ = + /// [`RELATIVE_OPERAND_VALIDITY_CAP`]. + /// + /// Every other operator (a filter comparing a computed value, a ranking, + /// a `sum`, a comparison, another function) has none. pub fn error_propagation( + memo: &Memo, consumer: &Operator, metric: AccuracyMetric, ) -> Option { - let _ = (consumer, metric); - todo!("#1207 PR 4: the error contract of each operator, per metric") + let inputs = consumer.children().len(); + let uniform = |factor: f64| ErrorPropagation { + factor_per_input: vec![factor; inputs], + validity_cap_per_input: vec![None; inputs], + }; + let absolute = matches!(metric, AccuracyMetric::Absolute { .. }); + let extremum_metric = absolute || metric == AccuracyMetric::Relative; + match consumer { + Operator::NonASAP(op) => match op { + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::Concat { .. } => Some(uniform(1.0)), + NonASAPOp::Project { cols, .. } => { + let preserves = |expr: &ScalarExpr| { + matches!(expr, ScalarExpr::Column(_) | ScalarExpr::Literal(_)) + }; + // A computed item that changes a value by at most its + // error keeps an absolute bound only. + cols.iter() + .all(|item| { + preserves(&item.expr) || (absolute && one_lipschitz(&item.expr)) + }) + .then(|| uniform(1.0)) + } + NonASAPOp::Filter { pred, child } => { + let lineage = &memo.groups()[child.0 as usize] + .group_properties + .column_lineage; + (pred.0.operator_refs().is_empty() + && pred.0.columns_referenced().iter().all(|&column| { + matches!(lineage.get(column), Some(ColumnLineage::SourceColumn(_))) + })) + .then(|| uniform(1.0)) + } + NonASAPOp::Aggregate { + measures, + filters, + having: None, + .. + } if filters.iter().all(Option::is_none) && !measures.is_empty() => { + let mut factor: f64 = 0.0; + for measure in measures { + match measure { + AggIntent::Count => {} + AggIntent::Min { .. } | AggIntent::Max { .. } if extremum_metric => { + factor = factor.max(1.0) + } + AggIntent::Avg { .. } if absolute => factor = factor.max(1.0), + _ => return None, + } + } + Some(uniform(factor)) + } + NonASAPOp::BinaryOp { + operator: + BinaryOperator { + kind: BinaryOpKind::Arithmetic(kind), + .. + }, + return_bool: false, + .. + } => { + let b = RELATIVE_OPERAND_VALIDITY_CAP; + match (kind, metric) { + (ArithmeticOpKind::Add | ArithmeticOpKind::Sub, _) if absolute => { + Some(uniform(1.0)) + } + (ArithmeticOpKind::Div, AccuracyMetric::Relative) => { + Some(ErrorPropagation { + factor_per_input: vec![1.0 / (1.0 - b); 2], + validity_cap_per_input: vec![None, Some(b)], + }) + } + (ArithmeticOpKind::Mul, AccuracyMetric::Relative) => { + Some(ErrorPropagation { + factor_per_input: vec![1.0 + b, 1.0], + validity_cap_per_input: vec![None, Some(b)], + }) + } + _ => None, + } + } + _ => None, + }, + Operator::ASAP(op) => match op { + ASAPOp::SummaryEstimate { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::SummaryMerge { .. } => Some(uniform(1.0)), + ASAPOp::SummaryAgg { + summary_type: FieldDataType::ExactAggregate(kind, _), + filter: None, + .. + } + | ASAPOp::SummaryTumblingWindowChain { + summary_type: FieldDataType::ExactAggregate(kind, _), + filter: None, + .. + } => match kind { + ExactKind::Count => Some(uniform(0.0)), + ExactKind::Min | ExactKind::Max if extremum_metric => Some(uniform(1.0)), + _ => None, + }, + _ => None, + }, + } } } +/// Whether `expr` changes a column's value by at most its error: `abs`, +/// `clamp`, `clamp_min` or `clamp_max` of a column with literal bounds, or +/// a column's negation (1-Lipschitz). +fn one_lipschitz(expr: &ScalarExpr) -> bool { + match expr { + ScalarExpr::Negative { expr, .. } => matches!(**expr, ScalarExpr::Column(_)), + ScalarExpr::FunctionCall { name, args } => { + matches!( + name.to_ascii_lowercase().as_str(), + "abs" | "clamp" | "clamp_min" | "clamp_max" + ) && matches!(args.first(), Some(ScalarExpr::Column(_))) + && args[1..] + .iter() + .all(|arg| matches!(arg, ScalarExpr::Literal(_))) + } + _ => false, + } +} + +/// $b$ of a ratio's denominator and a product's right operand: the +/// relative error under which [`LogicalRequirements::error_propagation`]'s +/// linear bound holds, $\varepsilon_{X/Y} \le (\varepsilon_X + +/// \varepsilon_Y) / (1 - b)$ (concepts doc, section 4). 0.1 is the weakest +/// error of every algorithm's list, so it never excludes a listed θ. +const RELATIVE_OPERAND_VALIDITY_CAP: f64 = 0.1; + /// The local error propagation of one consumer: its output error is at /// most $\sum_i \beta_{c,i}\varepsilon_{c,i}$ while each input's error stays /// within its validity cap (concepts doc, section 4; approved 2026-11-05). @@ -830,8 +1061,8 @@ mod tests { /// `LogicalProperties::new` over every operator kind and parameters: a /// sketch build (a chain included) needs parameters of its algorithm from - /// its list; parameters are rejected on an operator that builds no - /// sketch. + /// its list; a merge may carry any (its group decides); parameters are + /// rejected on an operator that neither builds nor passes on states. #[test] fn logical_properties_new_over_every_operator_and_parameters() { use LogicalPropertiesError as Invalid; @@ -852,6 +1083,12 @@ mod tests { query: crate::ir::schema::SketchStatistic::Quantile { q: 0.5 }, }), }; + let merge = CandidateExpression { + operator: Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![MemoGroupId(0)], + group_by: None, + }), + }; // k = 270 is KLL's structure but not in its list. let parameters = [ None, @@ -895,6 +1132,8 @@ mod tests { (summary_agg(FieldDataType::exact(ExactKind::Sum)), exact), (kll_chain, kll_build), (readout, exact), + // A merge's θ is checked against its group by `add_candidate`. + (merge, [Ok(()); 5]), ]; for (expression, by_parameters) in table { for (parameters, expected) in parameters.iter().zip(by_parameters) { @@ -1006,6 +1245,7 @@ mod memo_tests { use super::*; use crate::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; use crate::ir::scalar::expr_ir::ColumnRef; + use crate::ir::scalar::{CompareOpKind, ScalarValue}; use crate::ir::schema::{ DataType, Field, FieldDataType, Schema, SketchStatistic, SummaryUpdate, }; @@ -1077,9 +1317,13 @@ mod memo_tests { ) -> CandidateForMemoGroup { let input_requirements = vec![LogicalRequirements::default(); operator.children().len()]; CandidateForMemoGroup { - // Built directly: the doc example's k = 200 and k = 345 are not - // in today's KLL list, which `LogicalProperties::new` checks. - logical_properties: LogicalProperties { parameters }, + logical_properties: LogicalProperties::new( + &CandidateExpression { + operator: operator.clone(), + }, + parameters, + ) + .unwrap(), expression: CandidateExpression { operator }, input_requirements, fused_groups: vec![], @@ -1152,26 +1396,26 @@ mod memo_tests { assert!(memo.groups()[g2.0 as usize].parent_groups().is_empty()); } - /// KLL builds with k = 200 and k = 345 are two candidates of one group + /// KLL builds with k = 269 and k = 547 are two candidates of one group /// G2: θ is a logical property, not part of the expression. #[test] - fn kll_builds_with_k_200_and_k_345_share_one_group() { + fn kll_builds_with_k_269_and_k_547_share_one_group() { let mut memo = Memo::default(); let g1 = memo.find_or_add_group(scan("latency")).unwrap(); - let k200_group = memo.find_or_add_group(kll_state(g1)).unwrap(); - let k200 = memo - .add_candidate(k200_group, candidate(kll_state(g1), kll_k(200))) + let k269_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k269 = memo + .add_candidate(k269_group, candidate(kll_state(g1), kll_k(269))) .unwrap(); - let k345_group = memo.find_or_add_group(kll_state(g1)).unwrap(); - let k345 = memo - .add_candidate(k345_group, candidate(kll_state(g1), kll_k(345))) + let k547_group = memo.find_or_add_group(kll_state(g1)).unwrap(); + let k547 = memo + .add_candidate(k547_group, candidate(kll_state(g1), kll_k(547))) .unwrap(); - assert_eq!(k200_group, k345_group); - assert_ne!(k200, k345); - assert_eq!(memo.groups()[k200_group.0 as usize].candidates.len(), 2); + assert_eq!(k269_group, k547_group); + assert_ne!(k269, k547); + assert_eq!(memo.groups()[k269_group.0 as usize].candidates.len(), 2); assert_eq!( - memo.candidate(k345).logical_properties.parameters(), - kll_k(345).as_ref() + memo.candidate(k547).logical_properties.parameters(), + kll_k(547).as_ref() ); } @@ -1183,10 +1427,10 @@ mod memo_tests { let g1 = memo.find_or_add_group(scan("latency")).unwrap(); let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); let first = memo - .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .add_candidate(g2, candidate(kll_state(g1), kll_k(269))) .unwrap(); let again = memo - .add_candidate(g2, candidate(kll_state(g1), kll_k(200))) + .add_candidate(g2, candidate(kll_state(g1), kll_k(269))) .unwrap(); assert_eq!(first, again); assert_eq!( @@ -1196,7 +1440,7 @@ mod memo_tests { index_in_group: 0 } ); - let other_size = candidate(kll_state(g1), kll_k(345)); + let other_size = candidate(kll_state(g1), kll_k(547)); assert_ne!(memo.add_candidate(g2, other_size).unwrap(), first); assert_eq!(memo.groups()[g2.0 as usize].candidates.len(), 2); } @@ -1302,4 +1546,290 @@ mod memo_tests { assert_eq!(memo.groups().len(), 1); assert!(memo.groups()[g1.0 as usize].parent_groups().is_empty()); } + + /// `Scan latency`: `[ts, value, job]`, `job` a label column. + fn scan_with_job(metric: &str) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + Field::plain("job", DataType::Utf8, true), + ], + 0, + vec![], + ), + }) + } + + /// `column literal`, PromQL semantics. + fn compare(column: usize, op: CompareOpKind, literal: ScalarValue) -> ScalarExpr { + ScalarExpr::Compare { + left: Box::new(ScalarExpr::Column(column)), + op, + right: Box::new(ScalarExpr::Literal(literal)), + semantics: crate::ir::ExprSemantics::Promql, + } + } + + /// Decided 2026-11-07 (E1): constant error contracts. A `Filter` whose + /// predicate reads only columns copied from a source keeps or drops + /// whole rows: β = 1 in every metric. Over a readout of a KLL state of + /// `latency` by `job`, `job = "api"` reads such a column; `value > 0.5` + /// compares the computed estimate and has no contract. A `Project` of + /// `abs`, `clamp` or a negation of the value changes it by at most its + /// error: β = 1 for absolute errors only. + #[test] + fn e1_key_filters_and_value_preserving_projects_have_unit_contracts() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan_with_job("latency")).unwrap(); + let state = memo + .find_or_add_group(Operator::ASAP(ASAPOp::SummaryAgg { + child: g1, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![2]), + filter: None, + })) + .unwrap(); + let readout = memo + .find_or_add_group(quantile_readout(state, 0.9)) + .unwrap(); + let lineage = &memo.groups()[readout.0 as usize] + .group_properties + .column_lineage; + let column_where = |copied: bool| { + lineage + .iter() + .position(|l| matches!(l, ColumnLineage::SourceColumn(_)) == copied) + .expect("the readout has a label and an estimate") + }; + let (job, estimate) = (column_where(true), column_where(false)); + let filter = |pred| { + Operator::NonASAP(NonASAPOp::Filter { + pred: crate::ir::Predicate(pred), + child: readout, + }) + }; + let by_job = filter(compare( + job, + CompareOpKind::Eq, + ScalarValue::Utf8("api".into()), + )); + let by_value = filter(compare( + estimate, + CompareOpKind::Gt, + ScalarValue::Float64(0.5), + )); + for metric in AccuracyMetric::ALL { + assert_eq!( + LogicalRequirements::error_propagation(&memo, &by_job, metric) + .map(|propagation| propagation.factor_per_input), + Some(vec![1.0]), + "key-only filter, {metric:?}" + ); + assert_eq!( + LogicalRequirements::error_propagation(&memo, &by_value, metric), + None, + "a filter comparing the estimate, {metric:?}" + ); + } + let value = || Box::new(ScalarExpr::Column(1)); + let literal = |x: f64| ScalarExpr::Literal(ScalarValue::Float64(x)); + let computed = [ + ScalarExpr::FunctionCall { + name: "abs".into(), + args: vec![*value()], + }, + ScalarExpr::FunctionCall { + name: "clamp".into(), + args: vec![*value(), literal(0.0), literal(100.0)], + }, + ScalarExpr::Negative { + expr: value(), + semantics: crate::ir::ExprSemantics::Promql, + }, + ]; + for expr in computed { + let project = Operator::NonASAP(NonASAPOp::Project { + cols: vec![ + crate::ir::ProjectItem { + alias: None, + expr: ScalarExpr::Column(0), + }, + crate::ir::ProjectItem { + alias: Some("value".into()), + expr: expr.clone(), + }, + crate::ir::ProjectItem { + alias: None, + expr: ScalarExpr::Column(2), + }, + ], + child: g1, + }); + assert_eq!( + LogicalRequirements::error_propagation(&memo, &project, AccuracyMetric::ABSOLUTE) + .map(|propagation| propagation.factor_per_input), + Some(vec![1.0]), + "{expr:?}" + ); + assert_eq!( + LogicalRequirements::error_propagation(&memo, &project, AccuracyMetric::Relative), + None, + "{expr:?}" + ); + } + } + + /// A ratio carries relative error with factor 1/(1 − 0.1) on both + /// inputs and caps its denominator at 0.1; a count reads no value + /// (factor 0); a sum has no contract here (#1207 PR 8 adds its sign + /// condition). + #[test] + fn error_contracts_of_a_ratio_a_count_and_a_sum() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan_with_job("latency")).unwrap(); + let divide = Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + }, + return_bool: false, + lhs: g1, + rhs: g1, + }); + assert_eq!( + LogicalRequirements::error_propagation(&memo, ÷, AccuracyMetric::Relative), + Some(ErrorPropagation { + factor_per_input: vec![1.0 / 0.9; 2], + validity_cap_per_input: vec![None, Some(0.1)], + }) + ); + for metric in [AccuracyMetric::ABSOLUTE, AccuracyMetric::Rank] { + assert_eq!( + LogicalRequirements::error_propagation(&memo, ÷, metric), + None + ); + } + let aggregate = |measure| { + Operator::NonASAP(NonASAPOp::Aggregate { + reduction: Reduction::by(vec![2]), + measures: vec![measure], + output_names: vec![], + filters: vec![None], + having: None, + child: g1, + }) + }; + assert_eq!( + LogicalRequirements::error_propagation( + &memo, + &aggregate(AggIntent::Count), + AccuracyMetric::Rank + ) + .map(|propagation| propagation.factor_per_input), + Some(vec![0.0]) + ); + assert_eq!( + LogicalRequirements::error_propagation( + &memo, + &aggregate(AggIntent::Sum { col: Some(1) }), + AccuracyMetric::ABSOLUTE + ), + None + ); + } + + /// One readout's required accuracy. + fn required(max_error: f64) -> Vec { + vec![LogicalRequirements { + required_accuracy: Some(RequiredAccuracy { + metric: AccuracyMetric::Rank, + max_error, + failure_probability: 1.0, + }), + }] + } + + /// Decided 2026-11-07 (D3): `Memo::set_input_requirements` replaces + /// $r^{\log}$ of the one candidate it names; two readouts of one state + /// with different required accuracy are two candidates. + #[test] + fn set_input_requirements_replaces_only_the_named_candidate() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let g3 = memo.find_or_add_group(quantile_readout(g2, 0.9)).unwrap(); + let first = memo + .add_candidate(g3, candidate(quantile_readout(g2, 0.9), None)) + .unwrap(); + memo.set_input_requirements(first, required(0.01)); + let second = memo + .add_candidate(g3, candidate(quantile_readout(g2, 0.9), None)) + .unwrap(); + assert_ne!(first, second, "an unset copy is a new candidate"); + memo.set_input_requirements(second, required(0.02)); + let requirements: Vec<_> = memo.groups()[g3.0 as usize] + .candidates + .iter() + .map(|c| c.input_requirements.clone()) + .collect(); + assert_eq!(requirements, vec![required(0.01), required(0.02)]); + } + + /// Decided 2026-11-09 (A1): a merge of KLL states carries the θ of the + /// states it reads. Without one it is rejected; `merge[k=547]` reads only + /// the k = 547 build of its input group. + #[test] + fn a_merge_carries_the_parameters_of_the_states_it_reads() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + for k in [269, 547] { + memo.add_candidate(g2, candidate(kll_state(g1), kll_k(k))) + .unwrap(); + } + let merge = Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![g2], + group_by: None, + }); + let g3 = memo.find_or_add_group(merge.clone()).unwrap(); + assert!(matches!( + memo.add_candidate(g3, candidate(merge.clone(), None)), + Err(CandidateError::LogicalProperties( + LogicalPropertiesError::SketchBuildWithoutParameters + )) + )); + let merged = memo + .add_candidate(g3, candidate(merge, kll_k(547))) + .unwrap(); + assert_eq!( + memo.producers(merged, 0), + vec![CandidateForMemoGroupId { + group: g2, + index_in_group: 1 + }] + ); + } + + /// Decided 2026-11-07 (D3): one requirement per input group, or a + /// panic. + #[test] + #[should_panic(expected = "one input requirement per input group")] + fn set_input_requirements_panics_on_a_length_mismatch() { + let mut memo = Memo::default(); + let g1 = memo.find_or_add_group(scan("latency")).unwrap(); + let g2 = memo.find_or_add_group(kll_state(g1)).unwrap(); + let build = memo + .add_candidate(g2, candidate(kll_state(g1), kll_k(269))) + .unwrap(); + memo.set_input_requirements(build, Vec::new()); + } } diff --git a/crates/types/src/stages/logical_memo_candidates.rs b/crates/types/src/stages/logical_memo_candidates.rs index c2d567f81..7240fb172 100644 --- a/crates/types/src/stages/logical_memo_candidates.rs +++ b/crates/types/src/stages/logical_memo_candidates.rs @@ -48,12 +48,13 @@ pub struct PrunedLogicalCandidate { pub causes: Vec, } -/// One path from the candidate, through consumers that are themselves +/// One path from the candidate, through candidates that are themselves /// eliminated, to the edge that fails. One shortest path per failing edge. #[derive(Debug, Clone)] pub struct PruneCause { - /// The eliminated consumers on the path, nearest first. - pub through_pruned_consumers: Vec, + /// The eliminated candidates on the path, nearest first: consumers + /// above, or, for a candidate cut off from an input, producers below. + pub through_pruned_candidates: Vec, pub root: RootPruneReason, } @@ -68,12 +69,32 @@ pub enum RootPruneReason { input: usize, required: RequiredAccuracy, }, - /// The consumer's error contract carries none of the producer's metrics. + /// The metric rule: the producer is a readout whose metric the + /// consumer does not accept on that input; or, on a readout's own input, + /// the readout's metric is not one its output accepts. MetricRule { consumer: CandidateForMemoGroupId, input: usize, producer_metric: AccuracyMetric, - consumer_carries: Vec, + /// The metrics accepted on that input (on a readout's own input: at + /// its output). + accepted_metrics: Vec, + }, + /// The consumer is a top-k readout whose `k` exceeds the producer's + /// heap size: the heap cannot hold the k ranked keys, whatever accuracy + /// is required. + HeapSmallerThanTopK { + consumer: CandidateForMemoGroupId, + input: usize, + k: usize, + }, + /// The readout `consumer` is reached by query `query`, whose target sets + /// a failure probability over the whole result while the number of + /// values one evaluation returns ($n_q$) is unknown: no per-value + /// failure probability follows, so no state meets it. + ValuesPerEvaluationUnknown { + consumer: CandidateForMemoGroupId, + query: usize, }, /// The candidate never had a consumer and is not a query output. A /// candidate whose consumers were eliminated is traced to their causes. diff --git a/crates/types/tests/memo_summary_coverage.rs b/crates/types/tests/memo_summary_coverage.rs index 1f29f31a0..c4bdde5a5 100644 --- a/crates/types/tests/memo_summary_coverage.rs +++ b/crates/types/tests/memo_summary_coverage.rs @@ -22,7 +22,7 @@ use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarVal use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{ DataType, ExactKind, Field, FieldDataType, MergeInputRule, Schema, SketchAlgorithm, - SketchStatistic, SummaryUpdate, + SketchParams, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, ScalarExpr, @@ -454,7 +454,7 @@ fn scan_predicates_lift_and_their_states_merge() { /// G1 Scan latency /// G2 TimeShift(1h)(G1); G3 TimeRange[5m](G2); G4 SummaryAgg[KLL](G3) (the query) /// G5 Chain[KLL, 1m](G1); G6 TimeShift(1h)(G5); G7 TimeRange[5m](G6) -/// G4 candidate: SummaryMerge(G7) +/// G4 candidate: SummaryMerge[k=269](G7) /// ``` #[test] fn a_merge_over_a_chains_windows_covers_the_direct_build() { @@ -478,7 +478,8 @@ fn a_merge_over_a_chains_windows_covers_the_direct_build() { operator: merge(vec![g7]), }; let merge_of_windows = CandidateForMemoGroup { - logical_properties: LogicalProperties::new(&expression, None).unwrap(), + logical_properties: LogicalProperties::new(&expression, Some(SketchParams::Kll { k: 269 })) + .unwrap(), expression, input_requirements: vec![LogicalRequirements::default()], fused_groups: vec![], diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index b5ed3ca5a..5d7529890 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -311,11 +311,9 @@ rule's origin, and gives each reader a candidate that reads that group. | `OriginOfCandidateForMemoGroup` | Added by | Candidates | |---|---|---| | `Pass1` | `try_populate_pass1` (`pass1/memo_population.rs`) | Every Pass 1 choice of the group's measures: exact, exact accumulator, each summary family and size, and fused computations. Each summary choice comes at every `accuracy_budget_parts` from 1 to the most approximate inputs a query reaching the group has. A fused candidate implements the groups it absorbs only when no other reader needs them; otherwise they stay groups of their own. | -| `TumblingWindows` | Window composition: `add_local_window_composition` and `extend_local_window_composition` (`pass2/memo/window_candidates.rs`) | A window summary computed from tumbling windows, at each tumbling-window width the reader's evaluation grid allows. The tumbling-window chain stays private to the candidate. | -| `WidenedScans` | Scan widening: `add_scan_label_candidates` (`pass2/memo/rules/scan_labels.rs`) | A reader of a metric's scans widened to a shared label set, read through a `Project`. | -| `SummaryInput` | `factor_summary_inputs` (`pass2/memo/rules/summary_inputs.rs`) | Canonical value preprocessing feeding a summary build. | -| `TopKReuse` | Exact top-k reuse: `add_topk_candidates` (`pass2/memo/rules/topk.rs`) | One ranking (`Limit(k_max)` over `Sort`), and each top-k reading its first `k` rows from it. | -| `Rollup` | Rollup: `add_finer_aggregate_candidates` (`pass2/memo/rules/finer_aggregate.rs`), `add_rollup_candidates` and `add_window_rollup_candidates` (`pass2/memo/rules/rollup.rs`) | A coarse group computed from a finer group's values or states. | +| `TumblingWindows` | Chain groups: `add_chain_groups` (`stage1-pass2-reuse-candidates/src/rules/chains.rs`) | A chain group `SummaryTumblingWindowChain[w](X)` per width the reaching queries' evaluation grids allow, one build per surviving θ, the `TimeShift`/`TimeRange` groups over it, and the `SummaryMerge` over them in the state group. | +| `TopKReuse` | Exact top-k reuse: `add_topk_candidates` (`stage1-pass2-reuse-candidates/src/rules/topk.rs`) | One ranking (`Limit(k_max)` over `Sort`), and each top-k reading its first `k` rows from it. | +| `Rollup` | Rollup from the finest common group: `add_rollup_candidates` (`stage1-pass2-reuse-candidates/src/rules/rollup.rs`) | A coarse group computed from a finer group's values or states (through a compensating `Filter` on the finer group's keys), and the builds of a cluster's finest common group. | | `SummaryResult` | `factor_summary_results` (`pass2/memo/rules/summary_results.rs`) | A summary readout factored out of a composite implementation, so that other readers can read it. | | `SharedBuild` | Summary capability: `add_shared_summary_state_groups` (`pass2/memo/state_groups.rs`), Pass 2's last step | For each build made two or more times (by different candidates, or twice within one), one concrete summary-state group with one `SharedBuild` candidate; and for each candidate making such builds, one `SharedBuild` copy that reads the groups instead of building its own states. | @@ -524,7 +522,7 @@ candidates is in `crates/stage3-plan-selection/src/` (`select.rs`, `groups/`, `m | Stage 0 | `memo_of_queries` → `DagOfLogicalMemoGroups`, `MemoGroup`, `MemoGroupId` | `stage0.rs` (the types: `asap_types::stages`); hash-consing is `share_common_sub_dags` in `stage0/cse.rs` | | Pass 1 | `pass1_memo` → `PerComputationLogicalMemoCandidates` (its `Memo` and its `PrunedCandidate`s; `ComputationTarget` is Pass 1's private working type), the one entry: inside it, the computation targets (walking the memo groups), the summaries the deployment does not support and those no query can use pruned, the candidates attached to the groups (`try_populate_pass1`), then the accuracy-scoped sizes | `pass1/logical_memo_candidates.rs`; `pass1/per_computation_candidates.rs`; `pass1/memo_population.rs` | | Memo types | `DagOfLogicalMemoGroups`, `PerComputationLogicalMemoCandidates`, `HowToComputeAggregateMeasure`, `Memo`, `CandidateForMemoGroup`, `OriginOfCandidateForMemoGroup`, `LogicalASAPMemoCandidates`, `CandidateGenerationTimeout`, `root_groups`, `queries_rooted_at` | `crates/types/src/stages/` (`asap_types::stages`) | -| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. window composition for every legal tumbling-window width (`add_local_window_composition`); 2–9. the rules in `pass2/memo/rules/`: scan widening, summary inputs, top-k reuse, finer aggregate, rollup, window rollup, summary results, state configurations; 10. `add_shared_summary_state_groups`, the shared builds | `pass2/logical_memo_candidates.rs` (`LogicalASAPMemoCandidates` itself: `asap_types::stages`); `pass2/memo/window_candidates.rs`, `pass2/memo/rules/`, `pass2/memo/state_groups.rs`, `pass2/memo/shared_configurations.rs` | +| Pass 2 | `generate_logical_asap_memo_candidates` → `LogicalASAPMemoCandidates`, from Pass 1's output. It calls in this order: 1. rollup from the finest common group; 2. top-k reuse; 3. reach, caps and readout requirements; 4. dead-candidate elimination; 5. chain groups; 6. the logical edges | `stage1-pass2-reuse-candidates/src/` (`logical_memo_candidates.rs`, `rules/{rollup,topk,chains}.rs`, `requirements.rs`, `dead_candidates.rs`; `LogicalASAPMemoCandidates` itself: `asap_types::stages`) | | Memo to candidate DAGs | `enumerate`, `build_logical_asap_dags_from_chosen_candidates`, `ChosenCandidatePerMemoGroup`, `AllChosenCandidatesPerMemoGroup`, `ErrorBuildingLogicalASAPDagsFromChosenCandidates` | `pass2/memo/enumeration.rs` (re-exported at the crate root) | | Production memo build and search | The facade's `run_planning` builds the DAG of logical memo groups and runs Pass 1 (`pass1_memo`) once; then `generate_logical_asap_memo_candidates`, `generate_physical_asap_memo_candidates` and Stage 3's `select_plan`, once each; `select_plan` builds the plan. Behind `select_plan`: the group selection MILP (#1095), `groups/` and `milp/` | `crates/planner/src/run.rs`; `crates/stage3-plan-selection/src/select.rs` | @@ -596,7 +594,7 @@ the memo path's entry points. |---|---|---|---| | 0. Frontends | Query text, language → one logical root per query, carrying its accuracy target | Same: a `QueryRoot` per query | `lower_promql_query_workload`, `lower_sql_dialect` (`asap-frontend-*`); `plan_installable` (`asap_planner`) runs the frontends and Stages 1–3 on the user input | | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | -| 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the memo rules (`pass2/memo/rules/`), `pass2::memo::window_candidates` | +| 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the rules of `stage1-pass2-reuse-candidates/src/rules/` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | | 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_stage3_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | From 8876072d7fcbb7f572267b63bf1a21ff609021cd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:43:09 +0000 Subject: [PATCH 30/30] refactor(stage2): physical expansion, physical edges and pruned alternatives; Count-Min weights via column_sign (#1207 PR 5) Stage 2 expands each logical candidate Pass 2 kept into its offered (timing, lifetime): ingestion time for time-independent groups and their inputs (counter states only over ordered rows), AcrossEvaluations for chains and grid-evaluated groups. Prune reasons: NoIngestionPipeline, RawDataNotRetained, QueryTimeResultsNotRetained, CountMinWeightsNotProvenNonNegative, InputHasNoBuildableProducer, ReachesNoQueryOutput. Physical edges carry alpha^log and alpha^phys; PhysicalASAPMemoCandidates indexes the admissible ones. Deletes WeightDomain, NonNegativeWeightProof and SummaryUpdate::weight_domain (the state group's column_sign replaces them), Pass 1's copy of reads_only_time_series, and per-crate memo test fixtures (asap_types::test_support). --- Cargo.lock | 1 - crates/cost/src/state_bytes.rs | 7 +- crates/executor/src/capability.rs | 47 +- crates/executor/src/physical_planner/mod.rs | 8 - .../executor/src/summary_kernels/factory.rs | 12 - crates/executor/tests/hydra_cms_execution.rs | 49 +- .../tests/keyed_summary_row_weight.rs | 3 - crates/executor/tests/main.rs | 1 - crates/executor/tests/plan_properties.rs | 1 - crates/executor/tests/raw_scan.rs | 1 - crates/executor/tests/univmon_execution.rs | 6 +- .../executor/tests/weighted_topk_binding.rs | 17 - .../src/per_computation_candidates.rs | 61 +- .../src/per_computation_candidates_tests.rs | 26 +- .../src/realization.rs | 72 +- .../src/test_fixtures.rs | 32 +- .../src/tests.rs | 124 +-- .../src/tests/decided_examples.rs | 12 +- .../stage2-physical-optimization/Cargo.toml | 12 +- .../src/catalog/configurations.rs | 214 +++++ .../src/catalog/mod.rs | 204 ++++- .../src/catalog/pruning.rs | 82 ++ .../src/catalog/tests.rs | 807 ++++++++++++++++++ .../stage2-physical-optimization/src/lib.rs | 2 +- crates/types/src/ir/schema/mod.rs | 9 +- crates/types/src/ir/schema/sketch.rs | 102 +-- crates/types/src/ir/schema/topk.rs | 2 - .../src/stages/physical/memo_candidates.rs | 41 +- crates/types/src/stages/pruning.rs | 12 +- crates/types/src/test_support.rs | 90 +- docs/design_docs/concepts/glossary.md | 2 +- .../proposals/asap-primitive-schema.md | 28 +- .../design_docs/proposals/planner-layering.md | 27 +- .../asap-aware-mapping-contracts.md | 4 +- 34 files changed, 1561 insertions(+), 557 deletions(-) delete mode 100644 crates/executor/tests/weighted_topk_binding.rs create mode 100644 crates/stage2-physical-optimization/src/catalog/configurations.rs create mode 100644 crates/stage2-physical-optimization/src/catalog/pruning.rs create mode 100644 crates/stage2-physical-optimization/src/catalog/tests.rs diff --git a/Cargo.lock b/Cargo.lock index afe947361..130fc7200 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -467,7 +467,6 @@ name = "asap-stage2-physical-optimization" version = "0.1.0" dependencies = [ "asap-types", - "thiserror 2.0.18", ] [[package]] diff --git a/crates/cost/src/state_bytes.rs b/crates/cost/src/state_bytes.rs index fb18c5431..008834b55 100644 --- a/crates/cost/src/state_bytes.rs +++ b/crates/cost/src/state_bytes.rs @@ -434,9 +434,7 @@ mod tests { use asap_executor::summary_kernels::HllSketchAccumulator; use asap_executor::values::Value; use asap_types::ir::scalar::expr_ir::ColumnRef; - use asap_types::ir::schema::{ - NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, - }; + use asap_types::ir::schema::{SummaryInputExpr, SummaryUpdate}; use std::mem::{align_of, size_of}; use SketchAlgorithm as A; /// How the priced bytes relate to the sketch's. @@ -576,9 +574,6 @@ mod tests { let labeled = SummaryUpdate { item, weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, }; // A 16-byte label value: its key has the assumed size. let label = |i: u32| format!("{i:016}"); diff --git a/crates/executor/src/capability.rs b/crates/executor/src/capability.rs index 279139cfb..c4060f8f7 100644 --- a/crates/executor/src/capability.rs +++ b/crates/executor/src/capability.rs @@ -53,22 +53,19 @@ pub fn validate_summary_kernel( } /// The only Hydra algorithm with a kernel: Hydra over Count-Min. Each update -/// adds one item column's value to its group, with weight 1 (a row count, -/// `UnitCount`), a positive integer constant (`PositiveConstant`) or a weight -/// column proven non-negative (a Count-Min point count only overestimates -/// when no weight is negative). +/// adds one item column's value to its group, with weight 1 (a row count), a +/// positive integer constant or a weight column. The planner offers a +/// Count-Min build only over weights proven non-negative (a Count-Min point +/// count only overestimates when no weight is negative), and the build +/// rejects a negative weight value. fn validate_hydra_cms(family: &SummaryFamilyType, input: &SummaryUpdate) -> Result<(), String> { - use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; + use planner_types::ir::schema::SummaryInputExpr; hydra_cms_shape(family)?; if !matches!(input.item, Some(SummaryInputExpr::Column(_))) { return Err("HydraCms requires one item column".into()); } - let WeightDomain::NonNegative { proof } = &input.weight_domain else { - return Err("HydraCms weights must be non-negative; the weight's sign is unknown".into()); - }; - match (&input.weight, proof) { - (SummaryInputExpr::Constant(w), NonNegativeWeightProof::UnitCount) if *w == 1.0 => Ok(()), - (SummaryInputExpr::Constant(w), NonNegativeWeightProof::PositiveConstant) => { + match &input.weight { + SummaryInputExpr::Constant(w) => { let weight = constant_row_weight(*w).map_err(|error| format!("HydraCms {error}"))?; i32::try_from(weight.get()).map(|_| ()).map_err(|_| { format!( @@ -77,11 +74,9 @@ fn validate_hydra_cms(family: &SummaryFamilyType, input: &SummaryUpdate) -> Resu ) }) } - (SummaryInputExpr::Column(_), _) => Ok(()), + SummaryInputExpr::Column(_) => Ok(()), _ => Err( - "HydraCms weight must be the unit count, a positive constant or a non-negative \ - weight column" - .into(), + "HydraCms weight must be the unit count, a positive constant or a weight column".into(), ), } } @@ -116,10 +111,8 @@ pub(crate) fn hydra_cms_shape( /// none of the query's columns for the filter predicate to read, so only a /// query-time build can apply it. The ingestion-time compiler and the /// binding of any ingestion-timed `SummaryAgg` apply this check. The planner -/// applies the same rule: Stage 2 never offers materializing at ingestion time for a -/// filtered build (`IngestionPruneReason::Filtered`), and materialization -/// timing rejects an explicit one (`FilteredSummaryAtIngestion`). This check -/// guards plans that did not come from the planner. +/// offers a filtered build at ingestion time when its input is produced at +/// ingestion time; feeding such a build its input rows is #1207 PR 6. pub fn validate_ingestion_time_summary( filter: Option<&planner_types::ir::Predicate>, ) -> Result<(), String> { @@ -147,15 +140,13 @@ fn valid_matrix(width: u32, depth: u32) -> bool { pub(crate) fn unit_frequency_column( update: &planner_types::ir::schema::SummaryUpdate, ) -> Option<&planner_types::ir::scalar::ColumnRef> { - use planner_types::ir::schema::{NonNegativeWeightProof, SummaryInputExpr, WeightDomain}; - match (&update.item, &update.weight, &update.weight_domain) { - ( - Some(SummaryInputExpr::Column(column)), - SummaryInputExpr::Constant(weight), - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - ) if *weight == 1.0 => Some(column), + use planner_types::ir::schema::SummaryInputExpr; + match (&update.item, &update.weight) { + (Some(SummaryInputExpr::Column(column)), SummaryInputExpr::Constant(weight)) + if *weight == 1.0 => + { + Some(column) + } _ => None, } } diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index d7369f00f..4828b10c6 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -1327,14 +1327,6 @@ fn summary_agg_build( let PlannerReduction::Reduce(keys) = reduction else { return Err(invalid("keyed summary requires explicit partitions")); }; - if matches!(family, FieldDataType::Sketch(kind) if kind == &planner_types::ir::schema::SketchAlgorithm::CmsWithHeap) - && !matches!( - update.weight_domain, - planner_types::ir::schema::WeightDomain::NonNegative { .. } - ) - { - return Err(invalid("CMS requires a nonnegative weight contract")); - } fn columns( expr: &SummaryInputExpr, input: &SchemaRef, diff --git a/crates/executor/src/summary_kernels/factory.rs b/crates/executor/src/summary_kernels/factory.rs index 1bd273372..8b5d88b15 100644 --- a/crates/executor/src/summary_kernels/factory.rs +++ b/crates/executor/src/summary_kernels/factory.rs @@ -470,18 +470,6 @@ pub fn create_planner_accumulator( parameters: Option<&SketchParams>, input: &planner_types::ir::schema::SummaryUpdate, ) -> Result, String> { - if input.item.is_some() - && matches!( - input.weight_domain, - planner_types::ir::schema::WeightDomain::NonNegative { - proof: - planner_types::ir::schema::NonNegativeWeightProof::ResetAwareCounterDerivative - } - ) - { - return Err("window-weighted summaries require typed DAG binding; integer heap updaters cannot consume rates".into()); - } - crate::capability::validate_summary_kernel(family, input)?; if matches!(family, SummaryFamilyType::Sketch(SketchAlgorithm::HydraCms)) { return Err( diff --git a/crates/executor/tests/hydra_cms_execution.rs b/crates/executor/tests/hydra_cms_execution.rs index eedc1d553..a301a46b6 100644 --- a/crates/executor/tests/hydra_cms_execution.rs +++ b/crates/executor/tests/hydra_cms_execution.rs @@ -91,9 +91,6 @@ fn unit_count() -> SummaryUpdate { SummaryUpdate { item: Some(SummaryInputExpr::Column(ColumnRef::Named("service".into()))), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, } } @@ -269,26 +266,19 @@ fn hydra_cms_merges_like_one_build() { ); } -fn weighted(weight_domain: WeightDomain) -> SummaryUpdate { +fn weighted() -> SummaryUpdate { SummaryUpdate { weight: SummaryInputExpr::Column(ColumnRef::Named("weight".into())), - weight_domain, ..unit_count() } } -fn non_negative() -> WeightDomain { - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - } -} - // A non-negative weight column sums each item's weight within its group // (e.g. bytes per service per job), within the Count-Min bound over the // stream's total weight; a split-stream merge answers as one build. #[test] fn weighted_hydra_cms_sums_item_weights_per_group() { - let points = run(weighted(non_negative()), None, vec![batch(&ROWS)]); + let points = run(weighted(), None, vec![batch(&ROWS)]); let total: f64 = ROWS.iter().map(|(_, _, w)| w).sum(); let slack = std::f64::consts::E * total * (1. / f64::from(SHARED_COLUMNS) + 1. / f64::from(WIDTH)); @@ -303,26 +293,18 @@ fn weighted_hydra_cms_sums_item_weights_per_group() { } let (left, right) = ROWS.split_at(5); assert_eq!( - run( - weighted(non_negative()), - None, - vec![batch(left), batch(right)] - ), + run(weighted(), None, vec![batch(left), batch(right)]), points ); } -// A positive constant weight, `Constant(2.0)` with its `PositiveConstant` -// proof, adds 2 per row: each job's point count is twice the unit count's. +// A positive constant weight, `Constant(2.0)`, adds 2 per row: each job's point count is twice the unit count's. // A non-integer constant is refused at binding, since sketchlib's Hydra adds // integer weights. #[test] fn positive_constant_weight_adds_the_constant_per_row() { let constant = |weight: f64| SummaryUpdate { weight: SummaryInputExpr::Constant(weight), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::PositiveConstant, - }, ..unit_count() }; let unit = run(unit_count(), None, vec![batch(&ROWS)]); @@ -346,12 +328,9 @@ fn positive_constant_weight_adds_the_constant_per_row() { fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { let constant = SummaryUpdate { weight: SummaryInputExpr::Constant(2.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::PositiveConstant, - }, ..unit_count() }; - let column = weighted(non_negative()); + let column = weighted(); for (update, expected) in [ (constant, serde_json::json!({"Constant": 2})), (column, serde_json::json!({"Column": 2})), @@ -393,9 +372,6 @@ fn row_weight_serializes_as_one_field_and_rejects_constants_above_i32() { fn constant_weight_above_i32_is_refused_at_validation() { let update = SummaryUpdate { weight: SummaryInputExpr::Constant(2_147_483_648.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::PositiveConstant, - }, ..unit_count() }; let error = asap_executor::capability::validate_summary_kernel(&family(), &update) @@ -413,19 +389,6 @@ fn constant_weight_above_i32_is_refused_at_validation() { ); } -// A weight column of unknown sign is rejected at binding: a Count-Min point -// count only overestimates when no weight is negative. -#[test] -fn unknown_sign_weight_is_rejected() { - let error = compile_node( - &summary_agg(weighted(WeightDomain::UnknownOrSigned), None), - &[input_schema()], - ) - .err() - .expect("unknown-sign weight"); - assert!(error.to_string().contains("non-negative"), "{error}"); -} - // sketchlib's Hydra adds integer weights, so a negative, fractional or // non-finite weight value fails the build with a clear error. #[test] @@ -437,7 +400,7 @@ fn invalid_weight_values_fail_the_build() { (f64::INFINITY, "integer"), ] { let rows = [("api", "checkout", 1.0), ("api", "checkout", weight)]; - let error = try_run(weighted(non_negative()), None, vec![batch(&rows)]) + let error = try_run(weighted(), None, vec![batch(&rows)]) .err() .unwrap_or_else(|| panic!("{weight} accepted")); assert!(error.contains(expected), "{weight}: {error}"); diff --git a/crates/executor/tests/keyed_summary_row_weight.rs b/crates/executor/tests/keyed_summary_row_weight.rs index ed5162e4f..121a44df9 100644 --- a/crates/executor/tests/keyed_summary_row_weight.rs +++ b/crates/executor/tests/keyed_summary_row_weight.rs @@ -140,9 +140,6 @@ fn query_time_count_heap_binds_a_constant_row_weight() { ColumnRef::Named("service".into()), )])), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![0])), filter: None, diff --git a/crates/executor/tests/main.rs b/crates/executor/tests/main.rs index ee0b768e5..3aad7004d 100644 --- a/crates/executor/tests/main.rs +++ b/crates/executor/tests/main.rs @@ -28,4 +28,3 @@ mod text_extremum_memory; mod thread_safety; mod time_series_window; mod univmon_execution; -mod weighted_topk_binding; diff --git a/crates/executor/tests/plan_properties.rs b/crates/executor/tests/plan_properties.rs index 15c30098c..eb3b6d8ae 100644 --- a/crates/executor/tests/plan_properties.rs +++ b/crates/executor/tests/plan_properties.rs @@ -145,7 +145,6 @@ fn summary_capability_levels_are_distinct() { ColumnRef::Named("host".into()), )), weight: planner_types::ir::schema::SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), }; assert!(validate_summary_kernel(&cms, &update).is_ok()); // Stored Count-Min state reads only its bare count natively. diff --git a/crates/executor/tests/raw_scan.rs b/crates/executor/tests/raw_scan.rs index aa222032a..cce16a233 100644 --- a/crates/executor/tests/raw_scan.rs +++ b/crates/executor/tests/raw_scan.rs @@ -855,7 +855,6 @@ fn windowed_count_plan(shift_ms: Option) -> PhysicalASAPDAG { input: SummaryUpdate { item: None, weight: SummaryInputExpr::Constant(1.0), - weight_domain: Default::default(), }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), filter: None, diff --git a/crates/executor/tests/univmon_execution.rs b/crates/executor/tests/univmon_execution.rs index cf0aed856..be5954670 100644 --- a/crates/executor/tests/univmon_execution.rs +++ b/crates/executor/tests/univmon_execution.rs @@ -15,8 +15,7 @@ use planner_types::ir::properties::ExecutionDataState; use planner_types::ir::scalar::ColumnRef; use planner_types::ir::schema::{DataType, Schema}; use planner_types::ir::schema::{ - Field, FieldDataType, NonNegativeWeightProof, SketchAlgorithm, SketchStatistic, - SummaryInputExpr, SummaryUpdate, WeightDomain, + Field, FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, SummaryUpdate, }; use planner_types::ir::ASAPOp; use std::sync::Arc; @@ -169,9 +168,6 @@ fn planner_unit_frequency_update_binds_as_a_value_frequency_build() -> Result<() input: SummaryUpdate { item: Some(SummaryInputExpr::Column(ColumnRef::Named("src_ip".into()))), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, }, reduction: Reduction::Reduce(GroupKeys::by(vec![])), filter: None, diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs deleted file mode 100644 index bb97d9c47..000000000 --- a/crates/executor/tests/weighted_topk_binding.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Planner output binds directly to the shared runtime over raw counter samples. -use planner_types::ir::schema::*; - -#[test] -fn rate_updates_cannot_enter_integer_heap_factory() { - let family = FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap); - let input = SummaryUpdate { - item: Some(SummaryInputExpr::Column( - planner_types::ir::scalar::ColumnRef::Named("service".into()), - )), - weight: SummaryInputExpr::Column(planner_types::ir::scalar::ColumnRef::SampleValue), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - }, - }; - assert!(asap_executor::factory::create_planner_accumulator(&family, None, &input).is_err()); -} diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs index 1bb79cb78..08fe8eca8 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates.rs @@ -55,8 +55,8 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{resolve_column_ref, ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{ - DataType, EntityIdentity, ExactKind, FieldDataType, NonNegativeWeightProof, Schema, - SketchAlgorithm, SketchParams, StatisticKind, SummaryInputExpr, SummaryUpdate, WeightDomain, + DataType, EntityIdentity, ExactKind, FieldDataType, Schema, SketchAlgorithm, SketchParams, + StatisticKind, SummaryInputExpr, SummaryUpdate, }; use asap_types::ir::{ ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorResultKind, Predicate, @@ -73,8 +73,7 @@ use asap_accuracy::sizing::sketch_statistic; use asap_types::deployment_capabilities::{summary_supported, SupportedSummary}; use crate::realization::{ - expression, raw_data_samples_of_metric, realize_keyed_additive_summary_input, schema, - summary_candidates, + expression, realize_keyed_additive_summary_input, schema, summary_candidates, }; /// One measure of an aggregate group with at least one realization besides @@ -713,23 +712,6 @@ fn whole_expression_input( Some((realized.child, input, partitions)) } -/// `update` with the open requirement that its weights are samples of -/// `metric` ([`WeightDomain::RawDataSamplesOfMetric`]), when it is a keyed heap -/// update whose weights are not otherwise proven. -fn with_raw_data_samples_requirement( - mut update: SummaryUpdate, - metric: Option<&str>, -) -> SummaryUpdate { - if let Some(metric) = metric { - if update.item.is_some() && update.weight_domain == WeightDomain::UnknownOrSigned { - update.weight_domain = WeightDomain::RawDataSamplesOfMetric { - metric: metric.to_string(), - }; - } - } - update -} - /// Adds Pass 1's candidates to `memo` (Stage 0's, without candidates): /// every group's own expression, then, for each aggregate group with /// `targets`, each way to compute it. A consumer `Filter` group whose @@ -819,10 +801,6 @@ struct SingleMeasure<'a> { reduction: &'a Reduction, intent: &'a AggIntent, filter: Option<&'a Predicate>, - /// The metric whose raw data samples, or plain sums of them, are every - /// value the aggregate reads: a heap-sketch weight read from them - /// carries the open requirement `WeightDomain::RawDataSamplesOfMetric`. - raw_data_samples_of: Option<&'a str>, } /// Adds `ways.result_group`'s candidates that compute it from @@ -858,13 +836,11 @@ fn add_aggregate_candidates( Operator::NonASAP(NonASAPOp::Filter { pred, child: input }), )?; } - let raw_data_samples = raw_data_samples_of_metric(memo, child); let measure = |index: usize| SingleMeasure { input, reduction: &reduction, intent: &measures[index], filter: filters.get(index).and_then(Option::as_ref), - raw_data_samples_of: raw_data_samples.as_deref(), }; let target_of = |index: usize| { ways.targets @@ -974,7 +950,7 @@ fn add_fused_candidate( memo, Operator::ASAP(ASAPOp::SummaryAgg { child: rows, - input: with_raw_data_samples_requirement(update, measure.raw_data_samples_of), + input: update, summary_type: FieldDataType::Sketch(*algorithm), reduction: partitions, filter: None, @@ -1006,7 +982,6 @@ fn add_fused_candidate( reduction: &partitions, intent: &inner, filter: None, - raw_data_samples_of: None, }; let read = measure_input(&inner, memo, rows, None)?; let merged = accumulator_build(memo, kind, &inner, &below_inner, read)?; @@ -1049,16 +1024,13 @@ fn measure_output( )?; let summary_type = FieldDataType::Sketch(*algorithm); let read = measure_input(measure.intent, memo, measure.input, measure.filter)?; - let update = with_raw_data_samples_requirement( - summary_update( - measure.intent, - &summary_type, - measure.reduction, - schema(memo, measure.input), - read, - )?, - measure.raw_data_samples_of, - ); + let update = summary_update( + measure.intent, + &summary_type, + measure.reduction, + schema(memo, measure.input), + read, + )?; let state = add_state_group( memo, Operator::ASAP(ASAPOp::SummaryAgg { @@ -1757,23 +1729,14 @@ fn summary_update( item: Some(item), // The ranked value ([`measure_input`]). weight, - // Not proven non-negative; selection decides whether CMS is admissible. - weight_domain: WeightDomain::UnknownOrSigned, } } (_, Some(SketchAlgorithm::UnivMon)) | (AggIntent::Count, Some(SketchAlgorithm::Cms)) => { SummaryUpdate { item: Some(weight), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, } } - _ => SummaryUpdate { - item: None, - weight, - weight_domain: WeightDomain::UnknownOrSigned, - }, + _ => SummaryUpdate { item: None, weight }, }) } diff --git a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs index fcd9a9c3b..5e8ab0e65 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/per_computation_candidates_tests.rs @@ -10,7 +10,7 @@ use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; use asap_types::ir::schema::{ DataType, ExactKind, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, - SketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, PROMQL_SERIES_IDENTITY, + SketchStatistic, SummaryInputExpr, SummaryUpdate, PROMQL_SERIES_IDENTITY, }; use asap_types::ir::{ ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ScalarExpr, @@ -25,9 +25,8 @@ use crate::pass1_memo; use crate::per_computation_candidates::{ computation_candidates_for_intent, computation_targets, PerComputationCandidateError, }; -use crate::test_fixtures::{ - aggregate_group, group, metric_schema, query, scan_group, scan_operator, -}; +use crate::test_fixtures::{aggregate_group, group, metric_schema, query, scan_group}; +use asap_types::test_support::time_series_scan; /// A compact printout of every group and its candidates: the expression's /// operator, then per candidate its operator, input groups, θ and fused @@ -309,7 +308,7 @@ fn an_open_per_series_aggregate_offers_no_renamed_readout() { let mut memo = Memo::default(); let scan = group( &mut memo, - scan_operator("latency", metric_schema(&["job"], vec![])), + time_series_scan("latency", metric_schema(&["job"], vec![]), vec![]), ); let rows = group( &mut memo, @@ -718,12 +717,12 @@ fn own_value_having_filters_each_readout() { assert_eq!(kinds, ["Aggregate", "Filter", "Filter"]); } -// ── Fused heaps, whole-expression top-k, ranked items, raw-data signs ────── +// ── Fused heaps, whole-expression top-k, ranked items ───────────────────── /// `topk(5, sum by (job) (sum_over_time(m[5m])))`: the top-k gets heaps /// over the range's raw data samples, keyed by `job` (the ranked item), -/// weighted by the sample value with the open requirement that the samples -/// of `m` are non-negative (`RawDataSamplesOfMetric`). Each heap build +/// weighted by the sample value (whether those weights are non-negative is +/// the state group's `column_sign`, which Stage 2 reads). Each heap build /// lists the absorbed sums' groups in `fused_groups`, and the top-k's own /// heaps over its one-row-per-item input are pruned. #[test] @@ -760,10 +759,6 @@ fn whole_expression_topk_heaps_read_the_raw_samples() { update.item, Some(SummaryInputExpr::Column(ColumnRef::Named("job".into()))) ); - assert_eq!( - update.weight_domain, - WeightDomain::RawDataSamplesOfMetric { metric: "m".into() } - ); for build in candidates(&memo, state) { let mut fused = build.fused_groups.clone(); fused.sort(); @@ -776,8 +771,7 @@ fn whole_expression_topk_heaps_read_the_raw_samples() { } } -/// A top-k over a `count` ranks the counted items with unit weight, proven -/// non-negative. +/// A top-k over a `count` ranks the counted items with unit weight. #[test] fn count_ranked_topk_counts_with_unit_weight() { let mut memo = Memo::default(); @@ -800,10 +794,6 @@ fn count_ranked_topk_counts_with_unit_weight() { let memo = pass1_memo(memo, None).unwrap(); let (_, update) = state_group(&memo, SketchAlgorithm::CmsWithHeap); assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - assert!(matches!( - update.weight_domain, - WeightDomain::NonNegative { .. } - )); } /// The ranked item over several group keys is their tuple; the top-k's own diff --git a/crates/stage1-pass1-per-computation-candidates/src/realization.rs b/crates/stage1-pass1-per-computation-candidates/src/realization.rs index 16ef12ac6..95cb0f462 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/realization.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/realization.rs @@ -5,15 +5,14 @@ use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::non_asap::any_measure_filtered; -use asap_types::ir::operator::operator_properties::Source; use asap_types::ir::operator::Reduction; use asap_types::ir::scalar::column_resolution::{resolve_column_ref, ResolveError}; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::topk::ranked_column_of_group; use asap_types::ir::schema::ColumnId; use asap_types::ir::schema::{ - EntityIdentity, Field, FieldDataType, NonNegativeWeightProof, Schema, SketchAlgorithm, - SummaryInputExpr, SummaryTypeCapability, SummaryUpdate, WeightDomain, + EntityIdentity, Field, FieldDataType, Schema, SketchAlgorithm, SummaryInputExpr, + SummaryTypeCapability, SummaryUpdate, }; use asap_types::ir::{NonASAPOp, Operator}; use asap_types::stages::{Memo, MemoGroupId}; @@ -177,26 +176,17 @@ pub(crate) fn realize_keyed_additive_summary_input( if any_measure_filtered(filters) { return None; } - // A grouped sum of per-series `rate`/`increase`/`irate` values: the - // metric they derive from, whose declared type decides their sign. - let derivative_of = match (measures.as_slice(), expression(memo, raw_child)) { + // A grouped sum of per-series `rate`/`increase`/`irate` values. + let counter_input = matches!( + (measures.as_slice(), expression(memo, raw_child)), ( [AggIntent::Sum { .. }], - Operator::NonASAP(NonASAPOp::Aggregate { - measures: derivative, - child: samples, - .. - }), + Operator::NonASAP(NonASAPOp::Aggregate { measures: derivative, .. }), ) if matches!( derivative.as_slice(), [AggIntent::Rate | AggIntent::Increase | AggIntent::IRate] - ) => - { - Some(raw_data_samples_of_metric(memo, *samples)) - } - _ => None, - }; - let counter_input = derivative_of.is_some(); + ) + ); // `sum by (G) (sum_over_time(m[W]))` or `... count_over_time ...`: the // grouped sum of per-series sums (counts) of raw data samples is their sum // (count) by `G`, so the heap reads those samples. @@ -227,22 +217,6 @@ pub(crate) fn realize_keyed_additive_summary_input( }), _ => return None, }; - let weight_domain = match derivative_of { - _ if counted => WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - Some(Some(metric)) => WeightDomain::CounterDerivativeOfMetric { metric }, - _ => WeightDomain::UnknownOrSigned, - }; - if matches!(heap_algorithm, SketchAlgorithm::CmsWithHeap) - && !matches!( - weight_domain, - WeightDomain::NonNegative { .. } | WeightDomain::CounterDerivativeOfMetric { .. } - ) - { - // Unsupported: value-weighted CMS requires non-negative update evidence; use CountSketch for arbitrary values. - return None; - } let subpopulation_columns = match output_reduction { Reduction::PerEntity => vec![], Reduction::Reduce(keys) => keys @@ -301,7 +275,6 @@ pub(crate) fn realize_keyed_additive_summary_input( input: SummaryUpdate { item: Some(item), weight, - weight_domain, }, }) } @@ -380,35 +353,6 @@ pub(crate) fn schema(memo: &Memo, group: MemoGroupId) -> &Schema { &memo.groups()[group.0 as usize].group_properties.schema } -/// The metric whose raw data samples, or plain sums of them, are every value -/// group `group` outputs, or `None`: walking down the groups' expressions -/// through a time range, an `offset`-only time shift (it moves the window, -/// not the samples) and a plain `sum`, to a time-series scan. Any other -/// operator (rate and arithmetic included) ends it. Whether the metric's -/// samples are non-negative is plan selection's question -/// ([`WeightDomain::RawDataSamplesOfMetric`]). -pub(crate) fn raw_data_samples_of_metric(memo: &Memo, group: MemoGroupId) -> Option { - let Operator::NonASAP(operator) = expression(memo, group) else { - return None; - }; - match operator { - NonASAPOp::Scan { - source: Source::TimeSeries { metric }, - .. - } => Some(metric.clone()), - NonASAPOp::TimeRange { child, .. } => raw_data_samples_of_metric(memo, *child), - NonASAPOp::TimeShift { shift, child } if shift.at.is_none() => { - raw_data_samples_of_metric(memo, *child) - } - NonASAPOp::Aggregate { - measures, child, .. - } if matches!(measures.as_slice(), [AggIntent::Sum { col: None }]) => { - raw_data_samples_of_metric(memo, *child) - } - _ => None, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs index fa648844e..370365b6d 100644 --- a/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs +++ b/crates/stage1-pass1-per-computation-candidates/src/test_fixtures.rs @@ -2,12 +2,12 @@ //! groups through `Memo::find_or_add_group` and `Memo::add_query`, as //! Stage 0 does, so no test calls another stage crate. -use asap_types::ir::operator::operator_properties::{Reduction, Source}; +use asap_types::ir::operator::operator_properties::Reduction; use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate, QueryRoot}; +use asap_types::ir::{NonASAPOp, Operator, Predicate, QueryRoot}; use asap_types::stages::{Memo, MemoGroupId, MemoQuery}; +use asap_types::test_support::{series_schema, time_series_scan}; /// `[ts(0), value(1), labels...]`, time index 0, with `unique_keys`. pub fn metric_schema(labels: &[&str], unique_keys: Vec>) -> Schema { @@ -23,31 +23,19 @@ pub fn metric_schema(labels: &[&str], unique_keys: Vec>) -> Schema Schema::with_time_index(columns, 0, unique_keys) } -/// A predicate-free `TimeSeries(metric)` scan with `schema`. -pub fn scan_operator(metric: &str, schema: Schema) -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema, - }) -} - /// The group of `operator`, added as Stage 0 adds it. pub fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { memo.find_or_add_group(operator).unwrap() } -/// The group of a `TimeSeries(metric)` scan over [`metric_schema`] with its -/// series identity realized as the planner realizes it before Stage 0 -/// (`with_promql_series_identity`): a closed schema whose last column is -/// `$promql_series_identity`, keyed by it and the timestamp. +/// The group of a `TimeSeries(metric)` scan listing `labels`, its series +/// identity realized as the planner realizes it before Stage 0 +/// (`test_support::series_schema`). pub fn scan_group(memo: &mut Memo, metric: &str, labels: &[&str]) -> MemoGroupId { - let open = - OperatorNode::new_shared(scan_operator(metric, metric_schema(labels, vec![]))).unwrap(); - let realized = with_promql_series_identity(&open).unwrap(); - group(memo, scan_operator(metric, realized.schema.clone())) + group( + memo, + time_series_scan(metric, series_schema(labels), vec![]), + ) } /// The group of an `Aggregate` of `measures` (named by their intents) over diff --git a/crates/stage1-pass2-reuse-candidates/src/tests.rs b/crates/stage1-pass2-reuse-candidates/src/tests.rs index 9e86b2e20..9f25f9a65 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests.rs @@ -8,32 +8,33 @@ use std::time::Duration; use asap_accuracy::NoMeasuredGuarantees; use asap_types::ir::operator::operator_properties::{ - AtModifier, BinaryOpKind, Reduction, Source, TimeShift, + AtModifier, BinaryOpKind, Reduction, TimeShift, }; use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; use asap_types::ir::scalar::{ColumnRef, CompareOpKind, ScalarValue}; -use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{ - DataType, ExactKind, Field, FieldDataType, Schema, SketchAlgorithm, SketchParams, - SketchStatistic, SummaryUpdate, + ExactKind, FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic, SummaryUpdate, }; use asap_types::ir::{ - ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, OperatorNode, Predicate, - ProjectItem, QueryRoot, ScalarExpr, TimeRangeKind, + ASAPOp, BinaryOperator, ExprSemantics, NonASAPOp, Operator, Predicate, ProjectItem, QueryRoot, + ScalarExpr, TimeRangeKind, }; use asap_types::stages::{ - parameter_list, CandidateError, CandidateExpression, CandidateForMemoGroup, - CandidateForMemoGroupId, LogicalASAPMemoCandidates, LogicalProperties, LogicalRequirements, - Memo, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, PruneCause, - PrunedLogicalCandidate, RequiredAccuracy, RootPruneReason, + parameter_list, CandidateError, CandidateForMemoGroup, CandidateForMemoGroupId, + LogicalASAPMemoCandidates, Memo, MemoGroupId, MemoQuery, OriginOfCandidateForMemoGroup, + PruneCause, PrunedLogicalCandidate, RequiredAccuracy, RootPruneReason, }; use asap_types::workload::{ Predictability, Query, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, RepeatedDemand, RepetitionInterval, TimestampMs, }; +use asap_types::test_support::{ + add_candidate, group_with_own_candidate, series_schema, time_series_scan, +}; + use crate::generate_logical_asap_memo_candidates; /// The examples decided on 2026-11-07. @@ -41,79 +42,12 @@ mod decided_examples; // ── Fixtures ────────────────────────────────────────────────────────────── -/// Adds `operator` with `parameters` to `group`, as Pass 1 does. -fn try_candidate( - memo: &mut Memo, - group: MemoGroupId, - operator: Operator, - parameters: Option, -) -> Result { - let expression = CandidateExpression { operator }; - let logical_properties = LogicalProperties::new(&expression, parameters).unwrap(); - let input_requirements = - vec![LogicalRequirements::default(); expression.operator.children().len()]; - memo.add_candidate( - group, - CandidateForMemoGroup { - expression, - logical_properties, - input_requirements, - fused_groups: Vec::new(), - origin: OriginOfCandidateForMemoGroup::Pass1, - }, - ) -} - -fn candidate( - memo: &mut Memo, - group: MemoGroupId, - operator: Operator, - parameters: Option, -) -> CandidateForMemoGroupId { - try_candidate(memo, group, operator, parameters).unwrap() -} - -/// The group of `operator`, with `operator` as its exact candidate. -fn group(memo: &mut Memo, operator: Operator) -> MemoGroupId { - let group = memo.find_or_add_group(operator.clone()).unwrap(); - candidate(memo, group, operator, None); - group -} - -/// `[ts, value, labels...]`, the series identity realized as the planner -/// realizes it before Stage 0. -fn series_schema(labels: &[&str]) -> Schema { - let mut fields = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ]; - fields.extend( - labels - .iter() - .map(|n| Field::plain(*n, DataType::Utf8, true)), - ); - let open = OperatorNode::new_shared(scan_operator( - "m", - Schema::with_time_index(fields, 0, vec![]), - vec![], - )) - .unwrap(); - with_promql_series_identity(&open).unwrap().schema.clone() -} - -fn scan_operator(metric: &str, schema: Schema, predicates: Vec>) -> Operator { - Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates, - schema, - }) -} - /// A time-series scan of `metric` listing `labels`. fn scan(memo: &mut Memo, metric: &str, labels: &[&str]) -> MemoGroupId { - group(memo, scan_operator(metric, series_schema(labels), vec![])) + group_with_own_candidate( + memo, + time_series_scan(metric, series_schema(labels), vec![]), + ) } /// `column = value`, PromQL semantics. @@ -133,12 +67,15 @@ fn scan_where( labels: &[&str], matchers: Vec>, ) -> MemoGroupId { - group(memo, scan_operator(metric, series_schema(labels), matchers)) + group_with_own_candidate( + memo, + time_series_scan(metric, series_schema(labels), matchers), + ) } /// `child[seconds]`. fn range(memo: &mut Memo, child: MemoGroupId, seconds: u64) -> MemoGroupId { - group( + group_with_own_candidate( memo, Operator::NonASAP(NonASAPOp::TimeRange { range: Duration::from_secs(seconds), @@ -150,7 +87,7 @@ fn range(memo: &mut Memo, child: MemoGroupId, seconds: u64) -> MemoGroupId { /// `child offset `, or `child @ at`. fn shift(memo: &mut Memo, child: MemoGroupId, seconds: i64, at: Option) -> MemoGroupId { - group( + group_with_own_candidate( memo, Operator::NonASAP(NonASAPOp::TimeShift { shift: TimeShift { @@ -188,7 +125,7 @@ fn sketch_state( let operator = build(child, FieldDataType::Sketch(algorithm), reduction); let state = memo.find_or_add_group(operator.clone()).unwrap(); for parameters in parameter_list(algorithm) { - candidate(memo, state, operator.clone(), Some(parameters.clone())); + add_candidate(memo, state, operator.clone(), Some(parameters.clone())).unwrap(); } state } @@ -203,8 +140,9 @@ fn exact_state( kind: ExactKind, reduction: Reduction, ) -> (MemoGroupId, MemoGroupId) { - let state = group(memo, build(window, FieldDataType::exact(kind), reduction)); - let finalized = group( + let state = + group_with_own_candidate(memo, build(window, FieldDataType::exact(kind), reduction)); + let finalized = group_with_own_candidate( memo, Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: state }), ); @@ -213,7 +151,7 @@ fn exact_state( /// A readout group of `statistic` over `state`. fn estimate(memo: &mut Memo, state: MemoGroupId, statistic: SketchStatistic) -> MemoGroupId { - group( + group_with_own_candidate( memo, Operator::ASAP(ASAPOp::SummaryEstimate { summary_input: state, @@ -228,7 +166,7 @@ fn arithmetic( lhs: MemoGroupId, rhs: MemoGroupId, ) -> MemoGroupId { - group( + group_with_own_candidate( memo, Operator::NonASAP(NonASAPOp::BinaryOp { operator: BinaryOperator { @@ -600,7 +538,7 @@ fn a_coarse_exact_sum_rolls_up_the_finer_sum() { let mut memo = Memo::default(); let input = scan(&mut memo, "x", &["job", "instance"]); let window = range(&mut memo, input, 60); - let rate = group( + let rate = group_with_own_candidate( &mut memo, build( window, @@ -608,12 +546,12 @@ fn a_coarse_exact_sum_rolls_up_the_finer_sum() { Reduction::PerEntity, ), ); - let rows = group( + let rows = group_with_own_candidate( &mut memo, Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: rate }), ); let sum_by = |memo: &mut Memo, keys: Vec, name: &str| { - group( + group_with_own_candidate( memo, Operator::NonASAP(NonASAPOp::Aggregate { child: rows, @@ -657,7 +595,7 @@ fn a_smaller_topk_reads_the_largest_topk_of_the_same_input() { let mut memo = Memo::default(); let input = scan(&mut memo, "x", &["job"]); let topk = |memo: &mut Memo, k: usize| { - group( + group_with_own_candidate( memo, Operator::NonASAP(NonASAPOp::Aggregate { child: input, diff --git a/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs b/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs index f05798f19..4e450dbe3 100644 --- a/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs +++ b/crates/stage1-pass2-reuse-candidates/src/tests/decided_examples.rs @@ -24,7 +24,7 @@ fn quantile_over_time_readouts( q: f64, algorithms: &[SketchAlgorithm], ) -> QuantileReadouts { - let result = group( + let result = group_with_own_candidate( memo, Operator::NonASAP(NonASAPOp::Aggregate { child: window, @@ -42,7 +42,7 @@ fn quantile_over_time_readouts( summary_input: state, query: SketchStatistic::Quantile { q }, }); - let id = match try_candidate(memo, result, direct, None) { + let id = match add_candidate(memo, result, direct, None) { Ok(id) => id, Err(CandidateError::OutputDiffersFromGroup { .. }) => { let readout = estimate(memo, state, SketchStatistic::Quantile { q }); @@ -59,7 +59,7 @@ fn quantile_over_time_readouts( .collect(), child: readout, }); - candidate(memo, result, renamed, None) + add_candidate(memo, result, renamed, None).unwrap() } Err(error) => panic!("{error}"), }; @@ -356,7 +356,7 @@ fn e1_a_key_only_filter_keeps_the_ddsketch_readout_admissible() { .iter() .position(|field| field.name == "job") .expect("the result keeps the job label"); - let filtered = group( + let filtered = group_with_own_candidate( &mut memo, Operator::NonASAP(NonASAPOp::Filter { pred: Predicate(equals(job, "api")), @@ -388,7 +388,7 @@ fn h2_avg_of_readouts_under_a_relative_target_reads_exact_only() { let window = range(&mut memo, input, 300); let algorithms = [SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]; let p99 = quantile_over_time_readouts(&mut memo, window, 0.99, &algorithms); - let avg = group( + let avg = group_with_own_candidate( &mut memo, Operator::NonASAP(NonASAPOp::Aggregate { child: p99.result, @@ -481,7 +481,7 @@ fn a2_the_metric_rule_cuts_the_readout_edges_and_keeps_the_exact_division() { let p90 = quantile_over_time_readouts(&mut memo, window, 0.9, &algorithms); let ratio = arithmetic(&mut memo, ArithmeticOpKind::Div, p50.result, p50.result); let sum = arithmetic(&mut memo, ArithmeticOpKind::Add, ratio, p90.result); - let avg = group( + let avg = group_with_own_candidate( &mut memo, Operator::NonASAP(NonASAPOp::Aggregate { child: sum, diff --git a/crates/stage2-physical-optimization/Cargo.toml b/crates/stage2-physical-optimization/Cargo.toml index db1f94888..e0b801eea 100644 --- a/crates/stage2-physical-optimization/Cargo.toml +++ b/crates/stage2-physical-optimization/Cargo.toml @@ -3,10 +3,12 @@ name = "asap-stage2-physical-optimization" version = "0.1.0" edition = "2021" -# #509 Stage 2, physical ASAP-aware optimization: materialization and -# physical operator implementation of each Stage 1 candidate, and the -# PhysicalASAPMemoCandidates Stage 3 selects from (`catalog`). Depends only on -# asap-types, never on Stage 1's crate, plan selection or the executor. +# #509 Stage 2, physical ASAP-aware optimization: each Stage 1 candidate's +# timing and lifetime, and the PhysicalASAPMemoCandidates Stage 3 selects +# from (`catalog`). Depends only on asap-types, never on Stage 1's crate, +# plan selection or the executor. [dependencies] asap-types = { path = "../types" } -thiserror = "2" + +[dev-dependencies] +asap-types = { path = "../types", features = ["test-support"] } diff --git a/crates/stage2-physical-optimization/src/catalog/configurations.rs b/crates/stage2-physical-optimization/src/catalog/configurations.rs new file mode 100644 index 000000000..993b47529 --- /dev/null +++ b/crates/stage2-physical-optimization/src/catalog/configurations.rs @@ -0,0 +1,214 @@ +//! Decision-tree levels 3 and 4 (concepts doc, section 2): each logical +//! candidate's physical properties $p^{\mathrm{phys}} = (\tau, \ell)$, the +//! product of the values each dimension offers the candidate less the +//! invalid combination (`PhysicalProperties::new` rejects it), each with +//! the reason a prune rule removes it. Each prune rule reads only the value +//! it removes and the values decided above it. + +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::operator::operator_properties::AtModifier; +use asap_types::ir::schema::{ExactKind, FieldDataType, SketchAlgorithm}; +use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; +use asap_types::physical::{ExecutionTiming, Lifetime, PhysicalProperties, StorageMedium}; +use asap_types::stages::{ + CandidateForMemoGroupId, LogicalASAPMemoCandidates, Memo, MemoGroupId, PruneReason, ValueSign, +}; +use asap_types::workload::{DataArrival, DataWorkload}; + +/// Level 3's values, $\tau$. +const TIMINGS: [ExecutionTiming; 2] = [ExecutionTiming::QueryTime, ExecutionTiming::IngestionTime]; + +/// Level 4's values, $\ell$. A materialized result is stored in memory +/// only: the deployment capabilities offer no disk storage (#1217). +const LIFETIMES: [Lifetime; 3] = [ + Lifetime::None, + Lifetime::OneEvaluation(StorageMedium::Memory), + Lifetime::AcrossEvaluations(StorageMedium::Memory), +]; + +/// Every valid $(\tau, \ell)$ offered to `candidate`, in [`TIMINGS`] × +/// [`LIFETIMES`] order, each with the reason a prune rule removes it +/// (`None` for a kept one). Offered are: +/// +/// - `IngestionTime` only when `ingestion_time_offered` (the candidate's +/// group is time-independent or an input of one) and, for a counter +/// state, when its rows arrive in timestamp order or within a known +/// bound ([`counter_state_rows_ordered`]); +/// - `AcrossEvaluations` only for a chain and for a candidate of a group +/// evaluated on a known grid (`evaluation_grid_of_group`), and not for one +/// read under an absolute `@ T` ([`read_at_absolute_time`]); +/// - (IngestionTime, None) not for a summary build (`SummaryAgg`, a chain, +/// `MaintainPopulation`): fed at ingestion time, it keeps its state +/// between batches, so it is materialized. +/// +/// A value not offered is not recorded, and neither is the invalid +/// (IngestionTime, OneEvaluation). +pub(super) fn enumerate_candidate_configurations( + logical: &LogicalASAPMemoCandidates, + candidate: CandidateForMemoGroupId, + ingestion_time_offered: bool, + data: &DataWorkload, + capabilities: &DeploymentCapabilities, +) -> Vec<(PhysicalProperties, Option)> { + let memo = &logical.memo; + let operator = &memo.candidate(candidate).expression.operator; + let ingestion_time_offered = + ingestion_time_offered && counter_state_rows_ordered(memo, operator, data); + let across_evaluations_offered = + (matches!( + operator, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { .. }) + ) || logical.evaluation_grid_of_group[candidate.group.0 as usize].is_some()) + && !read_at_absolute_time(memo, candidate.group); + let count_min_weights_not_proven_non_negative = + count_min_weights_not_proven_non_negative(memo, candidate.group, operator); + let has_ingestion_pipeline = data.arrival.rows_per_second().is_some(); + let scans_raw_data = matches!(operator, Operator::NonASAP(NonASAPOp::Scan { .. })); + let builds_summary = matches!( + operator, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::MaintainPopulation { .. } + ) + ); + let mut configurations = Vec::new(); + for timing in TIMINGS { + if timing == ExecutionTiming::IngestionTime && !ingestion_time_offered { + continue; + } + for lifetime in LIFETIMES { + let across_evaluations = matches!(lifetime, Lifetime::AcrossEvaluations(_)); + if across_evaluations && !across_evaluations_offered { + continue; + } + if builds_summary + && timing == ExecutionTiming::IngestionTime + && lifetime == Lifetime::None + { + continue; + } + let Ok(properties) = PhysicalProperties::new(timing, lifetime) else { + continue; + }; + let query_time = timing == ExecutionTiming::QueryTime; + let reason = if count_min_weights_not_proven_non_negative { + Some(PruneReason::CountMinWeightsNotProvenNonNegative) + } else if !query_time && !has_ingestion_pipeline { + Some(PruneReason::NoIngestionPipeline) + } else if (query_time || across_evaluations) + && scans_raw_data + && !capabilities.raw_data_retained + { + Some(PruneReason::RawDataNotRetained) + } else if query_time && across_evaluations && !capabilities.query_time_results_retained + { + Some(PruneReason::QueryTimeResultsNotRetained) + } else { + None + }; + configurations.push((properties, reason)); + } + } + configurations +} + +/// Whether `group` is read under an absolute `@ T`: a `TimeShift` anchored +/// at a timestamp reads it, directly or through `TimeShift`s without `@`. +/// With no known end of the horizon, how far back that reader reaches (the +/// horizon end − T + range) is unbounded, so `group` is not materialized +/// across evaluations (concepts doc, section 5); `x @ 1700000000 [5m]` +/// reads `x` at query time. +fn read_at_absolute_time(memo: &Memo, group: MemoGroupId) -> bool { + memo.groups()[group.0 as usize] + .parent_groups() + .iter() + .any( + |&parent| match &memo.groups()[parent.0 as usize].expression { + Operator::NonASAP(NonASAPOp::TimeShift { shift, child }) if *child == group => { + match shift.at { + Some(AtModifier::Timestamp(_)) => true, + None => read_at_absolute_time(memo, parent), + Some(_) => false, + } + } + _ => false, + }, + ) +} + +/// Whether `operator`, when it builds a counter state (`rate`, `increase`, +/// `irate`), gets its rows in timestamp order at ingestion time: the data +/// declares how late a row may arrive (`max_out_of_order_ms`), so a reorder +/// buffer restores the order, or its rows are PromQL samples, taken to +/// arrive in order. True for any other operator. +fn counter_state_rows_ordered( + memo: &Memo, + operator: &Operator, + data: &DataWorkload, +) -> bool { + let (Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type, + .. + }) + | Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child, + summary_type, + .. + })) = operator + else { + return true; + }; + let counter_state = matches!( + summary_type, + FieldDataType::ExactAggregate(ExactKind::Rate | ExactKind::Increase | ExactKind::IRate, _) + ); + let out_of_order_bound_known = matches!( + data.arrival, + DataArrival::Ingesting { + max_out_of_order_ms: Some(_), + .. + } | DataArrival::Mixed { + max_out_of_order_ms: Some(_), + .. + } + ); + !counter_state + || out_of_order_bound_known + || memo.groups()[child.0 as usize] + .group_properties + .reads_only_time_series +} + +/// Whether `operator` builds a Count-Min state (`Cms`, `CmsWithHeap`, +/// `HydraCms`) whose weights are not proven non-negative: the state column +/// of its group, which records the sign of the weights it is updated with +/// (`column_sign`), is not `NonNegative`. +fn count_min_weights_not_proven_non_negative( + memo: &Memo, + group: MemoGroupId, + operator: &Operator, +) -> bool { + let (Operator::ASAP(ASAPOp::SummaryAgg { summary_type, .. }) + | Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { summary_type, .. })) = operator + else { + return false; + }; + if !matches!( + summary_type, + FieldDataType::Sketch( + SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::HydraCms + ) + ) { + return false; + } + let properties = &memo.groups()[group.0 as usize].group_properties; + let weight_sign = properties + .schema + .fields + .iter() + .position(|field| !field.is_plain()) + .map(|state| properties.column_sign[state]); + weight_sign != Some(ValueSign::NonNegative) +} diff --git a/crates/stage2-physical-optimization/src/catalog/mod.rs b/crates/stage2-physical-optimization/src/catalog/mod.rs index 391775557..b6b1f706e 100644 --- a/crates/stage2-physical-optimization/src/catalog/mod.rs +++ b/crates/stage2-physical-optimization/src/catalog/mod.rs @@ -1,28 +1,216 @@ //! Stage 2's physical candidates: expand each logical candidate into one -//! physical candidate per valid $(\tau, \ell)$, and each logical edge -//! into the physical edges between the two candidates' alternatives +//! physical candidate per offered $(\tau, \ell)$ (decision-tree levels 3 +//! and 4, [`configurations`]), and each logical edge into the physical +//! edges between the two candidates' alternatives //! ([`generate_physical_asap_memo_candidates`]). The output type, //! `PhysicalASAPMemoCandidates`, lives in `asap_types::stages`; Stage 3 //! selects from it. +mod configurations; +mod pruning; +#[cfg(test)] +mod tests; + +use std::collections::{HashMap, HashSet}; +use std::ops::Range; + use asap_types::deployment_capabilities::DeploymentCapabilities; -use asap_types::stages::{LogicalASAPMemoCandidates, PhysicalASAPMemoCandidates}; +use asap_types::stages::{ + producer_satisfies_physical_requirements, CandidateForMemoGroupId, LogicalASAPMemoCandidates, + MemoGroupId, PhysicalASAPMemoCandidates, PhysicalCandidate, PhysicalCandidateId, PhysicalEdge, + PrunedCandidate, +}; use asap_types::workload::DataWorkload; +use configurations::enumerate_candidate_configurations; + /// Stage 2: expand every logical candidate of `logical` that survived Pass 2 /// into its physical alternatives (decision-tree levels 3–4: timing and /// lifetime) the data and the deployment's `capabilities` allow, and every /// logical edge into its physical edges with $\alpha^{\log}$ and /// $\alpha^{\mathrm{phys}}$. Every alternative rejected is recorded in the -/// output's `pruned_candidates` with its `PruneReason`; a value outside a +/// output's `pruned_candidates` with its `PruneReason`: first those a +/// decision-tree prune rule removes, then those that cannot be built or +/// that no query output can read ([`pruning`]). A value outside a /// dimension's declared value set (`AcrossEvaluations` on a candidate of a -/// group with no known grid) is not offered and not recorded. Costs nothing, and -/// cannot fail. +/// group with no known grid) is not offered and not recorded. Costs nothing, +/// and cannot fail; alignment with each execution context is Stage 3's +/// check (`PhysicalCandidate::delivers_on`). pub fn generate_physical_asap_memo_candidates( logical: &LogicalASAPMemoCandidates, data: &DataWorkload, capabilities: &DeploymentCapabilities, ) -> PhysicalASAPMemoCandidates { - let _ = (logical, data, capabilities); - todo!("#1207 PR 5: physical expansion and physical edges") + let memo = &logical.memo; + let eliminated: HashSet = logical + .pruned + .iter() + .map(|pruned| pruned.candidate) + .collect(); + let ingestion_time_offered = ingestion_time_offered_per_group(logical, &eliminated); + let mut pruned_candidates = Vec::new(); + // Levels 3 and 4: the offered alternatives, before pruning, and the + // alternatives of each logical candidate (a range of `alternatives`). + let mut alternatives = Vec::new(); + let mut alternatives_of_logical_candidate: HashMap> = + HashMap::new(); + for (group_index, group) in memo.groups().iter().enumerate() { + for index_in_group in 0..group.candidates.len() { + let logical_candidate_id = CandidateForMemoGroupId { + group: MemoGroupId(group_index as u32), + index_in_group, + }; + if eliminated.contains(&logical_candidate_id) { + continue; + } + let first = alternatives.len(); + for (physical_properties, reason) in enumerate_candidate_configurations( + logical, + logical_candidate_id, + ingestion_time_offered[group_index], + data, + capabilities, + ) { + let alternative = PhysicalCandidate { + logical_candidate_id, + physical_properties, + }; + match reason { + Some(reason) => pruned_candidates.push(PrunedCandidate { + candidate: alternative, + reason, + }), + None => alternatives.push(alternative), + } + } + alternatives_of_logical_candidate + .insert(logical_candidate_id, first..alternatives.len()); + } + } + // Every physical edge between the alternatives, over provisional IDs + // (indices into `alternatives`), and each (consumer, input)'s + // admissible producers, which pruning reads. Pass 2's edges are grouped + // by (consumer, input), and those into one (c, i) are exactly the + // candidates of $G_{c,i}$ that survived Pass 2. + let mut edges = Vec::new(); + let mut admissible_producers_per_input: Vec>> = alternatives + .iter() + .map(|alternative| { + let input_count = memo + .candidate(alternative.logical_candidate_id) + .expression + .operator + .children() + .len(); + vec![Vec::new(); input_count] + }) + .collect(); + for edges_into_input in logical.edges.chunk_by(|a, b| { + (a.consumer_candidate_id, a.input_index_of_consumer) + == (b.consumer_candidate_id, b.input_index_of_consumer) + }) { + let consumer = edges_into_input[0].consumer_candidate_id; + let input_index_of_consumer = edges_into_input[0].input_index_of_consumer; + for consumer_index in alternatives_of_logical_candidate[&consumer].clone() { + let consumer_timing = alternatives[consumer_index].physical_properties.timing(); + for logical_edge in edges_into_input { + for producer_index in + alternatives_of_logical_candidate[&logical_edge.producer_candidate_id].clone() + { + let edge = PhysicalEdge { + producer_candidate_id: PhysicalCandidateId(producer_index), + consumer_candidate_id: PhysicalCandidateId(consumer_index), + input_index_of_consumer, + satisfies_logical_requirement: logical_edge.satisfies_logical_requirement, + satisfies_physical_requirement: producer_satisfies_physical_requirements( + &alternatives[producer_index], + consumer_timing, + ), + }; + if edge.is_logically_and_physically_admissible() { + admissible_producers_per_input[consumer_index][input_index_of_consumer] + .push(producer_index); + } + edges.push(edge); + } + } + } + } + // Candidates that cannot be built or that no query output reads. + let prune_reasons = + pruning::prune_reasons(&alternatives, &admissible_producers_per_input, memo); + let mut physical_candidate_id = vec![None; alternatives.len()]; + let mut physical_candidates = Vec::new(); + for ((alternative, reason), id) in alternatives + .into_iter() + .zip(prune_reasons) + .zip(&mut physical_candidate_id) + { + match reason { + Some(reason) => pruned_candidates.push(PrunedCandidate { + candidate: alternative, + reason, + }), + None => { + *id = Some(PhysicalCandidateId(physical_candidates.len())); + physical_candidates.push(alternative); + } + } + } + // $\mathcal E$ over $\mathcal C$: the edges between emitted candidates, + // renumbered and grouped by (consumer, input). + let mut edges: Vec = edges + .into_iter() + .filter_map(|edge| { + Some(PhysicalEdge { + producer_candidate_id: physical_candidate_id[edge.producer_candidate_id.0]?, + consumer_candidate_id: physical_candidate_id[edge.consumer_candidate_id.0]?, + ..edge + }) + }) + .collect(); + edges.sort_by_key(|edge| (edge.consumer_candidate_id, edge.input_index_of_consumer)); + PhysicalASAPMemoCandidates::new( + logical.clone(), + physical_candidates, + edges, + pruned_candidates, + ) +} + +/// Per group, by `MemoGroupId`: whether level 3 offers its candidates +/// `IngestionTime`. It does when the group is time-independent (it does +/// not depend on the evaluation time, as a chain does not), or when it is +/// an input, directly or through other offered groups, of a candidate of +/// an offered group: an ingestion-time candidate reads only ingestion-time +/// producers. Candidates Pass 2 eliminated read nothing. +fn ingestion_time_offered_per_group( + logical: &LogicalASAPMemoCandidates, + eliminated: &HashSet, +) -> Vec { + let groups = logical.memo.groups(); + let mut offered: Vec = groups + .iter() + .map(|group| !group.group_properties.depends_on_evaluation_time) + .collect(); + let mut pending: Vec = (0..groups.len()).filter(|&group| offered[group]).collect(); + while let Some(group) = pending.pop() { + for (index_in_group, candidate) in groups[group].candidates.iter().enumerate() { + let id = CandidateForMemoGroupId { + group: MemoGroupId(group as u32), + index_in_group, + }; + if eliminated.contains(&id) { + continue; + } + for input in candidate.expression.operator.children() { + let input = input.0 as usize; + if !offered[input] { + offered[input] = true; + pending.push(input); + } + } + } + } + offered } diff --git a/crates/stage2-physical-optimization/src/catalog/pruning.rs b/crates/stage2-physical-optimization/src/catalog/pruning.rs new file mode 100644 index 000000000..6a996ea0b --- /dev/null +++ b/crates/stage2-physical-optimization/src/catalog/pruning.rs @@ -0,0 +1,82 @@ +//! Physical alternatives no plan can use, removed before Stage 3: one with +//! an input no buildable producer can supply, or one no query output reads. +//! Both follow from the admissible edges alone; no cost is involved. + +use asap_types::physical::ExecutionTiming; +use asap_types::stages::{ + producer_satisfies_physical_requirements, root_groups, Memo, PhysicalCandidate, PruneReason, +}; + +/// Why each of `alternatives` cannot be used, `None` for a usable one. +/// `admissible_producers_per_input[c][i]` lists the alternatives (by +/// index) on admissible edges into input `i` of alternative `c`. +/// +/// - [`PruneReason::InputHasNoBuildableProducer`]: not buildable, where +/// buildable is the least fixed point of "every input has a buildable +/// admissible producer". A candidate without inputs is buildable; a cycle +/// without a buildable way out is not. +/// - [`PruneReason::ReachesNoQueryOutput`]: buildable, but not reached from +/// a query output over admissible edges into buildable producers. A query +/// reads its output as a query-time consumer, so a query output is an +/// alternative of a query's root group that $\alpha^{\mathrm{phys}}$ +/// lets a query-time consumer read. +pub(super) fn prune_reasons( + alternatives: &[PhysicalCandidate], + admissible_producers_per_input: &[Vec>], + memo: &Memo, +) -> Vec> { + let mut buildable = vec![false; alternatives.len()]; + loop { + let mut changed = false; + for (candidate, inputs) in admissible_producers_per_input.iter().enumerate() { + if !buildable[candidate] + && inputs + .iter() + .all(|producers| producers.iter().any(|&producer| buildable[producer])) + { + buildable[candidate] = true; + changed = true; + } + } + if !changed { + break; + } + } + let root_groups = root_groups(&memo.queries); + let mut reached = vec![false; alternatives.len()]; + let mut pending: Vec = alternatives + .iter() + .enumerate() + .filter(|(index, alternative)| { + buildable[*index] + && root_groups.contains(&alternative.logical_candidate_id.group) + && producer_satisfies_physical_requirements(alternative, ExecutionTiming::QueryTime) + }) + .map(|(index, _)| index) + .collect(); + while let Some(candidate) = pending.pop() { + if !reached[candidate] { + reached[candidate] = true; + pending.extend( + admissible_producers_per_input[candidate] + .iter() + .flatten() + .copied() + .filter(|&producer| buildable[producer]), + ); + } + } + buildable + .into_iter() + .zip(reached) + .map(|(buildable, reached)| { + if !buildable { + Some(PruneReason::InputHasNoBuildableProducer) + } else if !reached { + Some(PruneReason::ReachesNoQueryOutput) + } else { + None + } + }) + .collect() +} diff --git a/crates/stage2-physical-optimization/src/catalog/tests.rs b/crates/stage2-physical-optimization/src/catalog/tests.rs new file mode 100644 index 000000000..5b8c27894 --- /dev/null +++ b/crates/stage2-physical-optimization/src/catalog/tests.rs @@ -0,0 +1,807 @@ +//! Stage 2's tests. Each builds Pass 2's output, the memo and its logical +//! edges, through `Memo::find_or_add_group`, `Memo::add_candidate` and +//! `Memo::add_query`, and reads the physical candidates, edges and pruned +//! alternatives; no other stage crate is called. +use std::collections::BTreeMap; +use std::time::Duration; + +use asap_types::deployment_capabilities::DeploymentCapabilities; +use asap_types::ir::operator::operator_properties::{ + AtModifier, Reduction, Source, SourceColumn, TimeShift, +}; +use asap_types::ir::scalar::expr_ir::ArithmeticOpKind; +use asap_types::ir::scalar::{ColumnRef, ScalarValue}; +use asap_types::ir::schema::{ + EntityIdentity, ExactKind, FieldDataType, SketchAlgorithm, SketchStatistic, SummaryInputExpr, + SummaryUpdate, +}; +use asap_types::ir::{ + ASAPOp, ExprSemantics, NonASAPOp, Operator, QueryRoot, ScalarExpr, TimeRangeKind, +}; +use asap_types::physical::{ExecutionTiming, Lifetime, PhysicalProperties, StorageMedium}; +use asap_types::stages::{ + parameter_list, CandidateForMemoGroupId, EvaluationGrid, LogicalASAPMemoCandidates, + LogicalEdge, Memo, MemoGroupId, MemoQuery, PhysicalASAPMemoCandidates, PhysicalCandidateId, + PruneCause, PruneReason, PrunedLogicalCandidate, RootPruneReason, ValueSign, +}; +use asap_types::test_support::{ + add_candidate, group_with_own_candidate, series_schema, time_series_scan, +}; +use asap_types::workload::{DataArrival, DataWorkload}; +use ExecutionTiming::{IngestionTime, QueryTime}; + +use super::generate_physical_asap_memo_candidates; + +// ── Fixtures ────────────────────────────────────────────────────────────── + +const MEMORY: StorageMedium = StorageMedium::Memory; +const QT_NONE: (ExecutionTiming, Lifetime) = (QueryTime, Lifetime::None); +const QT_ONE: (ExecutionTiming, Lifetime) = (QueryTime, Lifetime::OneEvaluation(MEMORY)); +const QT_ACROSS: (ExecutionTiming, Lifetime) = (QueryTime, Lifetime::AcrossEvaluations(MEMORY)); +const IT_NONE: (ExecutionTiming, Lifetime) = (IngestionTime, Lifetime::None); +const IT_ACROSS: (ExecutionTiming, Lifetime) = (IngestionTime, Lifetime::AcrossEvaluations(MEMORY)); + +const GRID_60S: Option = Some(EvaluationGrid { + period_ms: 60_000, + phase_ms: 0, +}); + +/// The candidate `index_in_group` of `group`. +fn id(group: MemoGroupId, index_in_group: usize) -> CandidateForMemoGroupId { + CandidateForMemoGroupId { + group, + index_in_group, + } +} + +/// The edge from `producer` into input `input` of `consumer`. +fn edge( + producer: CandidateForMemoGroupId, + consumer: CandidateForMemoGroupId, + input: usize, + satisfies_logical_requirement: bool, +) -> LogicalEdge { + LogicalEdge { + producer_candidate_id: producer, + consumer_candidate_id: consumer, + input_index_of_consumer: input, + satisfies_logical_requirement, + } +} + +/// `child[seconds]`, a range relative to the evaluation time. +fn range(seconds: u64, child: MemoGroupId) -> Operator { + Operator::NonASAP(NonASAPOp::TimeRange { + range: Duration::from_secs(seconds), + kind: TimeRangeKind::Range, + child, + }) +} + +fn kll_build(child: MemoGroupId) -> Operator { + Operator::ASAP(ASAPOp::SummaryAgg { + child, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + }) +} + +/// The group of `operator`, with one candidate of `operator` per θ in +/// `parameters` (the first `count` of the algorithm's list). +fn sketch_group( + memo: &mut Memo, + operator: Operator, + algorithm: SketchAlgorithm, + count: usize, +) -> MemoGroupId { + let group = memo.find_or_add_group(operator.clone()).unwrap(); + for parameters in ¶meter_list(algorithm)[..count] { + add_candidate(memo, group, operator.clone(), Some(parameters.clone())).unwrap(); + } + group +} + +fn query(memo: &mut Memo, root: QueryRoot) { + memo.add_query(MemoQuery { + root, + accuracy_target: None, + values_per_evaluation: None, + }); +} + +/// The running example's query A, `quantile_over_time(0.95, latency[1m])` +/// evaluated every 60 s at phase 0, after Pass 2: the scan; its last +/// minute; the KLL state group, with candidate 0 a KLL build over that +/// minute and candidate 1 a merge of the windows a `TimeRange` selects +/// from a 60 s chain over the scan; and the p95 readout, the query's root. +/// Each edge's $\alpha^{\log}$ is true. Groups read by the query evaluate +/// on its 60 s grid; the scan and the chain do not depend on the +/// evaluation time. +struct Example { + logical: LogicalASAPMemoCandidates, + scan: CandidateForMemoGroupId, + last_minute: CandidateForMemoGroupId, + build: CandidateForMemoGroupId, + merge: CandidateForMemoGroupId, + chain: CandidateForMemoGroupId, + windows: CandidateForMemoGroupId, + readout: CandidateForMemoGroupId, +} + +fn example() -> Example { + let mut memo = Memo::default(); + let scan = group_with_own_candidate( + &mut memo, + time_series_scan("latency", series_schema(&["job"]), vec![]), + ); + let last_minute = group_with_own_candidate(&mut memo, range(60, scan)); + let state = sketch_group(&mut memo, kll_build(last_minute), SketchAlgorithm::Kll, 1); + let chain = sketch_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: FieldDataType::Sketch(SketchAlgorithm::Kll), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }), + SketchAlgorithm::Kll, + 1, + ); + // The windows and the merge carry the chain build's θ. + let windows = sketch_group(&mut memo, range(60, chain), SketchAlgorithm::Kll, 1); + let merge = add_candidate( + &mut memo, + state, + Operator::ASAP(ASAPOp::SummaryMerge { + children: vec![windows], + group_by: None, + }), + Some(parameter_list(SketchAlgorithm::Kll)[0].clone()), + ) + .unwrap(); + let readout = group_with_own_candidate( + &mut memo, + Operator::ASAP(ASAPOp::SummaryEstimate { + summary_input: state, + query: SketchStatistic::Quantile { q: 0.95 }, + }), + ); + query(&mut memo, QueryRoot::Operator(readout)); + let [scan, last_minute, build, chain, windows, readout] = + [scan, last_minute, state, chain, windows, readout].map(|group| id(group, 0)); + let edges = vec![ + edge(scan, last_minute, 0, true), + edge(last_minute, build, 0, true), + edge(windows, merge, 0, true), + edge(scan, chain, 0, true), + edge(chain, windows, 0, true), + edge(build, readout, 0, true), + edge(merge, readout, 0, true), + ]; + let mut evaluation_grid_of_group = vec![GRID_60S; memo.groups().len()]; + evaluation_grid_of_group[scan.group.0 as usize] = None; + evaluation_grid_of_group[chain.group.0 as usize] = None; + Example { + logical: LogicalASAPMemoCandidates { + memo, + edges, + pruned: vec![], + evaluation_grid_of_group, + }, + scan, + last_minute, + build, + merge, + chain, + windows, + readout, + } +} + +fn ingesting(max_out_of_order_ms: Option) -> DataWorkload { + DataWorkload { + arrival: DataArrival::Ingesting { + rows_per_second: 1000.0, + max_out_of_order_ms, + }, + ..DataWorkload::default() + } +} + +/// Raw data and query-time results retained. +fn retaining_everything() -> DeploymentCapabilities { + DeploymentCapabilities { + query_time_results_retained: true, + ..DeploymentCapabilities::default() + } +} + +/// Stage 2 over `logical`, checking what Stage 3 relies on: the edges are +/// grouped by (consumer, input) over emitted candidates, the admissible +/// indexes list exactly the admissible edges, and every emitted candidate +/// has an admissible edge into each input. +fn generate( + logical: &LogicalASAPMemoCandidates, + data: &DataWorkload, + capabilities: &DeploymentCapabilities, +) -> PhysicalASAPMemoCandidates { + let physical = generate_physical_asap_memo_candidates(logical, data, capabilities); + let count = physical.physical_candidates.len(); + let mut consumer_inputs = Vec::new(); + for (edge_id, edge) in physical.edges() { + assert!(edge.producer_candidate_id.0 < count && edge.consumer_candidate_id.0 < count); + consumer_inputs.push((edge.consumer_candidate_id, edge.input_index_of_consumer)); + assert_eq!( + physical + .admissible_edges_into(edge.consumer_candidate_id, edge.input_index_of_consumer) + .contains(&edge_id), + edge.is_logically_and_physically_admissible() + ); + assert_eq!( + physical + .admissible_edges_out_of(edge.producer_candidate_id) + .contains(&edge_id), + edge.is_logically_and_physically_admissible() + ); + } + assert!(consumer_inputs.is_sorted(), "grouped by (consumer, input)"); + for (id, candidate) in physical.physical_candidates.iter().enumerate() { + let inputs = logical + .memo + .candidate(candidate.logical_candidate_id) + .expression + .operator + .children() + .len(); + for input in 0..inputs { + assert!( + !physical + .admissible_edges_into(PhysicalCandidateId(id), input) + .is_empty(), + "{candidate:?} input {input}" + ); + } + } + physical +} + +fn properties((timing, lifetime): (ExecutionTiming, Lifetime)) -> PhysicalProperties { + PhysicalProperties::new(timing, lifetime).unwrap() +} + +/// The physical properties of `logical_candidate`'s emitted alternatives. +fn offered( + physical: &PhysicalASAPMemoCandidates, + logical_candidate: CandidateForMemoGroupId, +) -> Vec { + physical + .physical_candidates + .iter() + .filter(|candidate| candidate.logical_candidate_id == logical_candidate) + .map(|candidate| candidate.physical_properties) + .collect() +} + +/// The pruned alternatives of `logical_candidate`, with their reasons. +fn pruned( + physical: &PhysicalASAPMemoCandidates, + logical_candidate: CandidateForMemoGroupId, +) -> Vec<(PhysicalProperties, PruneReason)> { + physical + .pruned_candidates + .iter() + .filter(|pruned| pruned.candidate.logical_candidate_id == logical_candidate) + .map(|pruned| (pruned.candidate.physical_properties, pruned.reason)) + .collect() +} + +fn list(states: &[(ExecutionTiming, Lifetime)]) -> Vec { + states.iter().copied().map(properties).collect() +} + +fn with_reason( + states: &[(ExecutionTiming, Lifetime)], + reason: PruneReason, +) -> Vec<(PhysicalProperties, PruneReason)> { + states + .iter() + .map(|&state| (properties(state), reason)) + .collect() +} + +// ── Levels 3 and 4 ──────────────────────────────────────────────────────── + +/// Query A over ingesting data, everything retained. Time-dependent +/// candidates (the last minute, the build over it, the merge, the windows, +/// the readout) run only at query time; on the 60 s grid they may be +/// materialized across evaluations. The scan and the chain are +/// time-independent, so ingestion time is offered too; the scan has no +/// known grid, so it is not offered `AcrossEvaluations`. A summary build is +/// not offered (IT, None) (A4): fed at ingestion time it keeps its state, +/// so the chain is offered (IT, AcrossEvaluations) only; the scan's +/// (IT, None) feeds the chain built at ingestion time. +#[test] +fn offers_each_candidate_its_timings_and_lifetimes() { + let example = example(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + let query_time = list(&[QT_NONE, QT_ONE, QT_ACROSS]); + for candidate in [ + example.last_minute, + example.build, + example.merge, + example.windows, + example.readout, + ] { + assert_eq!(offered(&physical, candidate), query_time, "{candidate:?}"); + assert_eq!(pruned(&physical, candidate), vec![], "{candidate:?}"); + } + assert_eq!( + offered(&physical, example.scan), + list(&[QT_NONE, QT_ONE, IT_NONE]) + ); + assert_eq!( + offered(&physical, example.chain), + list(&[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS]) + ); + assert_eq!(pruned(&physical, example.chain), vec![]); +} + +/// Decided 2026-11-09 (A4): no summary build (`SummaryAgg`, chain, +/// `MaintainPopulation`) is offered (IngestionTime, None), offered or +/// pruned; the scan, which keeps nothing between batches, is. +#[test] +fn no_summary_build_is_offered_at_ingestion_time_without_a_lifetime() { + let example = example(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + let memo = &example.logical.memo; + let ingestion_time_none = properties(IT_NONE); + let alternatives = physical + .physical_candidates + .iter() + .map(|candidate| { + ( + candidate.logical_candidate_id, + candidate.physical_properties, + ) + }) + .chain(physical.pruned_candidates.iter().map(|pruned| { + ( + pruned.candidate.logical_candidate_id, + pruned.candidate.physical_properties, + ) + })); + for (logical_candidate, physical_properties) in alternatives { + let builds = matches!( + memo.candidate(logical_candidate).expression.operator, + Operator::ASAP( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::MaintainPopulation { .. } + ) + ); + assert!( + !(builds && physical_properties == ingestion_time_none), + "{logical_candidate:?}" + ); + } + assert!(offered(&physical, example.scan).contains(&ingestion_time_none)); +} + +/// Data at rest (or of unknown arrival) has no ingestion pipeline: every +/// ingestion-time alternative is pruned. +#[test] +fn without_an_ingestion_pipeline_nothing_runs_at_ingestion_time() { + let example = example(); + for arrival in [DataArrival::AtRest, DataArrival::Unknown] { + let data = DataWorkload { + arrival, + ..DataWorkload::default() + }; + let physical = generate(&example.logical, &data, &retaining_everything()); + assert_eq!( + pruned(&physical, example.scan), + with_reason(&[IT_NONE], PruneReason::NoIngestionPipeline) + ); + assert_eq!( + pruned(&physical, example.chain), + with_reason(&[IT_ACROSS], PruneReason::NoIngestionPipeline) + ); + assert!(physical + .physical_candidates + .iter() + .all(|candidate| candidate.physical_properties.timing() == QueryTime)); + } +} + +/// Without raw data retained, the scan cannot run at query time, nor keep +/// its raw rows across evaluations at ingestion time; what reads it at +/// query time then has no buildable producer, so only the ingestion-time +/// chain serves the query. +#[test] +fn without_raw_data_only_ingestion_time_builds_read_the_scan() { + let mut example = example(); + let capabilities = DeploymentCapabilities { + raw_data_retained: false, + ..retaining_everything() + }; + let physical = generate(&example.logical, &ingesting(None), &capabilities); + assert_eq!( + pruned(&physical, example.scan), + with_reason(&[QT_NONE, QT_ONE], PruneReason::RawDataNotRetained) + ); + assert_eq!(offered(&physical, example.scan), list(&[IT_NONE])); + assert_eq!(offered(&physical, example.last_minute), vec![]); + assert_eq!(offered(&physical, example.build), vec![]); + assert_eq!(offered(&physical, example.chain), list(&[IT_ACROSS])); + assert_eq!( + offered(&physical, example.merge), + list(&[QT_NONE, QT_ONE, QT_ACROSS]) + ); + assert_eq!( + pruned(&physical, example.build), + with_reason( + &[QT_NONE, QT_ONE, QT_ACROSS], + PruneReason::InputHasNoBuildableProducer + ) + ); + + // A scan on an evaluation grid is offered `AcrossEvaluations`; at + // ingestion time that retains the raw rows it scans, so it is pruned + // too. + example.logical.evaluation_grid_of_group[example.scan.group.0 as usize] = GRID_60S; + let physical = generate(&example.logical, &ingesting(None), &capabilities); + assert_eq!( + pruned(&physical, example.scan), + with_reason( + &[QT_NONE, QT_ONE, QT_ACROSS, IT_ACROSS], + PruneReason::RawDataNotRetained + ) + ); + assert_eq!(offered(&physical, example.scan), list(&[IT_NONE])); +} + +/// The default deployment retains no query-time result across +/// evaluations: every (QueryTime, AcrossEvaluations) is pruned, and +/// `OneEvaluation` stays. +#[test] +fn query_time_results_across_evaluations_need_the_deployment_to_retain_them() { + let example = example(); + let physical = generate( + &example.logical, + &ingesting(None), + &DeploymentCapabilities::default(), + ); + for candidate in [example.build, example.merge, example.readout] { + assert_eq!( + pruned(&physical, candidate), + with_reason(&[QT_ACROSS], PruneReason::QueryTimeResultsNotRetained), + "{candidate:?}" + ); + assert_eq!(offered(&physical, candidate), list(&[QT_NONE, QT_ONE])); + } +} + +/// `sum_over_time(m[1m] @ 1000)`: the `@ T` result is time-independent, so +/// it and its time-dependent input, the range under it, are offered +/// ingestion time; neither has a known grid, so neither is offered +/// `AcrossEvaluations`. +#[test] +fn an_at_t_subtree_is_offered_ingestion_time() { + let mut memo = Memo::default(); + let scan = + group_with_own_candidate(&mut memo, time_series_scan("m", series_schema(&[]), vec![])); + let last_minute = group_with_own_candidate(&mut memo, range(60, scan)); + let pinned = group_with_own_candidate( + &mut memo, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(AtModifier::Timestamp(1_000)), + }, + child: last_minute, + }), + ); + query(&mut memo, QueryRoot::Operator(pinned)); + let [scan, last_minute, pinned] = [scan, last_minute, pinned].map(|group| id(group, 0)); + let logical = LogicalASAPMemoCandidates { + evaluation_grid_of_group: vec![None; memo.groups().len()], + memo, + edges: vec![ + edge(scan, last_minute, 0, true), + edge(last_minute, pinned, 0, true), + ], + pruned: vec![], + }; + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + for candidate in [last_minute, pinned] { + assert_eq!( + offered(&physical, candidate), + list(&[QT_NONE, QT_ONE]), + "{candidate:?}" + ); + assert_eq!( + pruned(&physical, candidate), + with_reason(&[IT_NONE], PruneReason::ReachesNoQueryOutput), + "{candidate:?}" + ); + } +} + +/// Decided 2026-11-09 (A7): `x @ 1700000000 [5m]` with no known horizon +/// end reads `x` back without bound, so the scan under the `@ T` is not +/// materialized across evaluations; it still runs at query time. +#[test] +fn a_subtree_under_an_absolute_anchor_is_not_materialized_across_evaluations() { + let mut memo = Memo::default(); + let scan = + group_with_own_candidate(&mut memo, time_series_scan("x", series_schema(&[]), vec![])); + let pinned = group_with_own_candidate( + &mut memo, + Operator::NonASAP(NonASAPOp::TimeShift { + shift: TimeShift { + offset_ms: 0, + at: Some(AtModifier::Timestamp(1_700_000_000_000)), + }, + child: scan, + }), + ); + let window = group_with_own_candidate(&mut memo, range(300, pinned)); + query(&mut memo, QueryRoot::Operator(window)); + let [scan, pinned, window] = [scan, pinned, window].map(|group| id(group, 0)); + let mut evaluation_grid_of_group = vec![GRID_60S; memo.groups().len()]; + evaluation_grid_of_group[pinned.group.0 as usize] = None; + evaluation_grid_of_group[window.group.0 as usize] = None; + let logical = LogicalASAPMemoCandidates { + evaluation_grid_of_group, + memo, + edges: vec![edge(scan, pinned, 0, true), edge(pinned, window, 0, true)], + pruned: vec![], + }; + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + let scan_offered = offered(&physical, scan); + assert!(scan_offered.contains(&properties(QT_NONE))); + assert!(!scan_offered + .iter() + .any(|p| matches!(p.lifetime(), Lifetime::AcrossEvaluations(_)))); + assert!(pruned(&physical, scan) + .iter() + .all(|(p, _)| !matches!(p.lifetime(), Lifetime::AcrossEvaluations(_)))); +} + +/// A rate state built over a SQL table (rows without a series order) is +/// offered ingestion time only when the data bounds how late a row +/// arrives; over PromQL samples, taken to arrive in order, it always is. +#[test] +fn a_counter_state_runs_at_ingestion_time_over_ordered_rows() { + let rate_state = |memo: &mut Memo, source: Source| { + let scan = group_with_own_candidate( + memo, + Operator::NonASAP(NonASAPOp::Scan { + source, + predicates: vec![], + schema: series_schema(&[]), + }), + ); + let state = group_with_own_candidate( + memo, + Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + child: scan, + summary_type: FieldDataType::exact(ExactKind::Rate), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + filter: None, + tumbling_window_width_ms: 60_000, + }), + ); + query(memo, QueryRoot::Operator(state)); + (id(scan, 0), id(state, 0)) + }; + let mut memo = Memo::default(); + let (table, table_state) = rate_state( + &mut memo, + Source::Table { + table_ref: "events".into(), + }, + ); + let (series, series_state) = rate_state( + &mut memo, + Source::TimeSeries { + metric: "c_total".into(), + }, + ); + let logical = LogicalASAPMemoCandidates { + evaluation_grid_of_group: vec![None; memo.groups().len()], + memo, + edges: vec![ + edge(table, table_state, 0, true), + edge(series, series_state, 0, true), + ], + pruned: vec![], + }; + let at_ingestion = |physical: &PhysicalASAPMemoCandidates, state| { + offered(physical, state).contains(&properties(IT_ACROSS)) + }; + let unbounded = generate(&logical, &ingesting(None), &retaining_everything()); + assert!(!at_ingestion(&unbounded, table_state)); + assert!(at_ingestion(&unbounded, series_state)); + let bounded = generate(&logical, &ingesting(Some(2_000)), &retaining_everything()); + assert!(at_ingestion(&bounded, table_state)); + assert!(at_ingestion(&bounded, series_state)); +} + +/// `topk(5, count by (job) (m))` as a Count-Min heap keyed by `job` and +/// weighted by the sample value: pruned in every alternative unless the +/// data declares `m`'s values non-negative, and then offered. +#[test] +fn a_count_min_build_needs_weights_proven_non_negative() { + let heap = |declared: BTreeMap| { + let mut memo = Memo::new(declared); + let scan = group_with_own_candidate( + &mut memo, + time_series_scan("m", series_schema(&["job"]), vec![]), + ); + let state = sketch_group( + &mut memo, + Operator::ASAP(ASAPOp::SummaryAgg { + child: scan, + summary_type: FieldDataType::Sketch(SketchAlgorithm::CmsWithHeap), + input: SummaryUpdate { + item: Some(SummaryInputExpr::EntityIdentity( + EntityIdentity::PromqlLabelSet { excluding: vec![] }, + )), + weight: SummaryInputExpr::Column(ColumnRef::SampleValue), + }, + reduction: Reduction::Reduce(Default::default()), + filter: None, + }), + SketchAlgorithm::CmsWithHeap, + 1, + ); + query(&mut memo, QueryRoot::Operator(state)); + let (scan, state) = (id(scan, 0), id(state, 0)); + let logical = LogicalASAPMemoCandidates { + evaluation_grid_of_group: vec![None; memo.groups().len()], + memo, + edges: vec![edge(scan, state, 0, true)], + pruned: vec![], + }; + let physical = generate(&logical, &ingesting(None), &retaining_everything()); + (offered(&physical, state), pruned(&physical, state)) + }; + let (kept, unproven) = heap(BTreeMap::new()); + assert_eq!(kept, vec![]); + assert_eq!( + unproven, + with_reason( + &[QT_NONE, QT_ONE], + PruneReason::CountMinWeightsNotProvenNonNegative + ) + ); + let (kept, unproven) = heap(BTreeMap::from([( + SourceColumn { + source: "m".into(), + column: "value".into(), + }, + ValueSign::NonNegative, + )])); + assert_eq!(kept, list(&[QT_NONE, QT_ONE])); + assert_eq!(unproven, vec![]); +} + +// ── Edges and pruning ───────────────────────────────────────────────────── + +/// Each logical edge becomes an edge between every pair of the two +/// candidates' emitted alternatives, with $\alpha^{\log}$ copied and +/// $\alpha^{\mathrm{phys}}$ from the producer's timing and lifetime and the +/// consumer's timing: the chain's four alternatives read the scan's three +/// (12 edges); a query-time chain reads only the query-time scan, and the +/// ingestion-time chain only the ingestion-time scan, which is not +/// materialized. +#[test] +fn edges_carry_both_requirements() { + let example = example(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + let candidate = |id: PhysicalCandidateId| physical.physical_candidates[id.0]; + let into_chain: Vec<_> = physical + .edges() + .map(|(_, edge)| edge) + .filter(|edge| candidate(edge.consumer_candidate_id).logical_candidate_id == example.chain) + .collect(); + assert_eq!(into_chain.len(), 12); + for edge in into_chain { + let producer = candidate(edge.producer_candidate_id).physical_properties; + let consumer = candidate(edge.consumer_candidate_id).physical_properties; + assert!(edge.satisfies_logical_requirement); + assert_eq!( + edge.satisfies_physical_requirement, + producer.timing() == consumer.timing(), + "{producer:?} -> {consumer:?}" + ); + } +} + +/// A logical edge with $\alpha^{\log}$ false is kept but not admissible: +/// a readout whose only producer fails it has no buildable producer, and +/// what only it reads reaches no query output. +#[test] +fn an_input_without_an_admissible_producer_is_not_buildable() { + let mut example = example(); + for logical_edge in &mut example.logical.edges { + if logical_edge.consumer_candidate_id == example.readout { + logical_edge.satisfies_logical_requirement = false; + } + } + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + assert!(physical.physical_candidates.is_empty()); + assert_eq!( + pruned(&physical, example.readout), + with_reason( + &[QT_NONE, QT_ONE, QT_ACROSS], + PruneReason::InputHasNoBuildableProducer + ) + ); + assert_eq!( + pruned(&physical, example.build), + with_reason( + &[QT_NONE, QT_ONE, QT_ACROSS], + PruneReason::ReachesNoQueryOutput + ) + ); +} + +/// A candidate Pass 2 eliminated is not expanded: the build over the last +/// minute, here eliminated with its input, leaves the merge as the +/// readout's only producer. +#[test] +fn candidates_pass_2_eliminated_are_not_expanded() { + let mut example = example(); + let (build, last_minute) = (example.build, example.last_minute); + example.logical.edges.retain(|edge| { + ![edge.producer_candidate_id, edge.consumer_candidate_id].contains(&build) + && edge.consumer_candidate_id != last_minute + }); + example.logical.pruned = [build, last_minute] + .map(|candidate| PrunedLogicalCandidate { + candidate, + causes: vec![PruneCause { + through_pruned_candidates: vec![], + root: RootPruneReason::NoConsumer, + }], + }) + .to_vec(); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + for candidate in [build, last_minute] { + assert_eq!(offered(&physical, candidate), vec![]); + assert_eq!(pruned(&physical, candidate), vec![]); + } + assert_eq!( + offered(&physical, example.readout), + list(&[QT_NONE, QT_ONE, QT_ACROSS]) + ); +} + +/// A scalar query's operand is a query output too: `latency_p95 * 2` over +/// the readout keeps the readout's query-time alternatives. +#[test] +fn a_scalar_roots_operand_is_a_query_output() { + let mut example = example(); + example.logical.memo.queries.clear(); + query( + &mut example.logical.memo, + QueryRoot::Scalar(ScalarExpr::Arithmetic { + op: ArithmeticOpKind::Mul, + left: Box::new(ScalarExpr::PromqlScalarFromVector(example.readout.group)), + right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(2.0))), + semantics: ExprSemantics::Promql, + }), + ); + let physical = generate(&example.logical, &ingesting(None), &retaining_everything()); + assert_eq!( + offered(&physical, example.readout), + list(&[QT_NONE, QT_ONE, QT_ACROSS]) + ); +} diff --git a/crates/stage2-physical-optimization/src/lib.rs b/crates/stage2-physical-optimization/src/lib.rs index 5ef0d90d1..f22b0c377 100644 --- a/crates/stage2-physical-optimization/src/lib.rs +++ b/crates/stage2-physical-optimization/src/lib.rs @@ -3,7 +3,7 @@ //! //! The planner facade calls [`generate_physical_asap_memo_candidates`] on //! Pass 2's output (`asap_types::stages::LogicalASAPMemoCandidates`): each -//! logical candidate expanded into one physical candidate per valid +//! logical candidate expanded into one physical candidate per offered //! $(\tau, \ell)$, each logical edge into its physical edges with //! $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives //! (`asap_types::stages::PhysicalASAPMemoCandidates`). Stage 3 selects from diff --git a/crates/types/src/ir/schema/mod.rs b/crates/types/src/ir/schema/mod.rs index a7457a8c1..ba15d0c8e 100644 --- a/crates/types/src/ir/schema/mod.rs +++ b/crates/types/src/ir/schema/mod.rs @@ -22,10 +22,9 @@ pub mod topk; pub use aggregate_schema::aggregate_output_schema; pub use sketch::{ - EntityIdentity, ExactKind, ExactParams, HydraParams, NonNegativeWeightProof, SamplingKind, - SamplingParams, SketchAlgorithm, SketchParams, SketchStatistic, StatModelKind, StatModelParams, - SummaryInputExpr, SummaryUpdate, WaveletKind, WaveletParams, WeightDomain, - EXACT_ACCUMULATOR_STATE_BYTES, + EntityIdentity, ExactKind, ExactParams, HydraParams, SamplingKind, SamplingParams, + SketchAlgorithm, SketchParams, SketchStatistic, StatModelKind, StatModelParams, + SummaryInputExpr, SummaryUpdate, WaveletKind, WaveletParams, EXACT_ACCUMULATOR_STATE_BYTES, }; pub use summary_type_capability::{ MergeInputRule, MergedError, StatisticKind, SummaryTypeCapability, SupportedSummary, @@ -132,7 +131,7 @@ impl From> for Field { /// /// "Sample" means two things in the planner, and the names keep them apart: /// a *raw data sample* is one timestamped value of a time series as it -/// arrives (`RawDataSamplesOfMetric`), and a *summary sample* is the subset of +/// arrives (a time-series `Scan`'s rows), and a *summary sample* is the subset of /// input rows a sampling summary family keeps as its state /// ([`FieldDataType::SummarySample`], [`SamplingKind`]). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/types/src/ir/schema/sketch.rs b/crates/types/src/ir/schema/sketch.rs index c94c4fd0f..3fb48776a 100644 --- a/crates/types/src/ir/schema/sketch.rs +++ b/crates/types/src/ir/schema/sketch.rs @@ -259,11 +259,6 @@ impl HydraParams { pub struct SummaryUpdate { pub item: Option, pub weight: SummaryInputExpr, - /// Planner proof for algorithms whose error/update semantics require a - /// non-negative stream. Absence of proof is never interpreted as zero or - /// non-negative. - #[serde(default)] - pub weight_domain: WeightDomain, } impl SummaryUpdate { @@ -271,77 +266,10 @@ impl SummaryUpdate { Self { item: None, weight: SummaryInputExpr::Column(column), - weight_domain: WeightDomain::UnknownOrSigned, } } } -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum WeightDomain { - #[default] - UnknownOrSigned, - NonNegative { - proof: NonNegativeWeightProof, - }, - /// An open requirement, not a proof: the weights are raw data samples of - /// `metric`, or plain sums of them (no rate, no other arithmetic), so they - /// are non-negative when every raw data sample of `metric` is: `metric` is - /// declared a counter, or declared non-negative. Logical optimization - /// records it without reading the data workload; plan selection confirms - /// it from the declared source facts - /// ([`SourceFacts::metric_type`](crate::workload::SourceFacts::metric_type), - /// [`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)). - /// Unconfirmed, it counts as not proven. Serialized as - /// `samples_of_metric`. - #[serde(rename = "samples_of_metric")] - RawDataSamplesOfMetric { - metric: String, - }, - /// An open requirement, not a proof: the weights are reset-corrected - /// `rate`, `increase` or `irate` values of `metric`'s raw data samples, or plain - /// sums of them. Reset correction keeps them non-negative exactly when - /// the samples are, that is when `metric` is a counter; over a gauge a - /// negative sample can give a negative value. Plan selection confirms - /// it from the declared metric types, as - /// [`RawDataSamplesOfMetric`](Self::RawDataSamplesOfMetric). Unconfirmed, it - /// counts as not proven. - CounterDerivativeOfMetric { - metric: String, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum NonNegativeWeightProof { - UnitCount, - /// `rate`, `increase` or `irate` of a counter's samples, or plain sums - /// of them: reset correction over non-negative samples gives a - /// non-negative increase, and a rate divides it by a positive duration. - /// Plan selection gives it on confirming a - /// [`WeightDomain::CounterDerivativeOfMetric`] requirement. - ResetAwareCounterDerivative, - /// Raw data samples of a metric declared a counter - /// ([`PrometheusMetricType::Counter`](crate::workload::PrometheusMetricType::Counter)), or - /// plain sums of them: a counter sample is never negative. Plan selection - /// gives it on confirming a - /// [`WeightDomain::RawDataSamplesOfMetric`] requirement. Serialized as - /// `counter_samples`. - #[serde(rename = "counter_samples")] - CounterRawDataSamples, - /// Raw data samples of a metric declared non-negative - /// ([`SourceFacts::column_signs`](crate::workload::SourceFacts::column_signs)), - /// such as a gauge, or plain sums of them. Plan selection gives it on - /// confirming a [`WeightDomain::RawDataSamplesOfMetric`] requirement for a - /// metric not declared a counter. Serialized as - /// `declared_non_negative_samples`. - #[serde(rename = "declared_non_negative_samples")] - DeclaredNonNegativeRawDataSamples, - /// A constant weight greater than 0, such as `Constant(2.0)`: every row - /// adds the same positive amount, so no weight is negative. - PositiveConstant, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum EntityIdentity { PromqlLabelSet { excluding: Vec }, @@ -395,16 +323,12 @@ mod tests { EntityIdentity::PromqlLabelSet { excluding: vec![] }, )), weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, }, SummaryUpdate { item: Some(SummaryInputExpr::Tuple(vec![SummaryInputExpr::Column( ColumnRef::Named("service".into()), )])), weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, }, ] { let input_json = serde_json::to_string(&input).unwrap(); @@ -421,34 +345,12 @@ mod tests { } } - /// The raw-data-sample and summary-sample renames keep their JSON names, - /// so committed plans and workloads still read. + /// The summary-sample rename keeps its JSON name, so committed plans + /// and workloads still read. #[test] fn renamed_sample_terms_keep_their_wire_names() { use crate::ir::schema::FieldDataType; let json = |value: serde_json::Value| value.to_string(); - let raw = WeightDomain::RawDataSamplesOfMetric { - metric: "m".to_string(), - }; - assert_eq!( - serde_json::to_string(&raw).unwrap(), - json(serde_json::json!({"kind": "samples_of_metric", "metric": "m"})) - ); - for (proof, name) in [ - ( - NonNegativeWeightProof::CounterRawDataSamples, - "counter_samples", - ), - ( - NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples, - "declared_non_negative_samples", - ), - ] { - assert_eq!( - serde_json::to_string(&proof).unwrap(), - format!("\"{name}\"") - ); - } let summary_sample = FieldDataType::SummarySample( SamplingKind::Reservoir, SamplingParams::Reservoir { size: 64 }, diff --git a/crates/types/src/ir/schema/topk.rs b/crates/types/src/ir/schema/topk.rs index 89d9f819d..e2af20f14 100644 --- a/crates/types/src/ir/schema/topk.rs +++ b/crates/types/src/ir/schema/topk.rs @@ -245,7 +245,6 @@ mod tests { use crate::ir::schema::schema_support::with_promql_series_identity; use crate::ir::schema::{ DataType, Field, FieldDataType, Schema, SketchAlgorithm, SketchStatistic, SummaryUpdate, - WeightDomain, }; use crate::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, ProjectItem, ScalarExpr, TimeRangeKind, @@ -319,7 +318,6 @@ mod tests { input: SummaryUpdate { item: Some(super::series_identity_item(schema_of(&memo, window), &[2])), weight: crate::ir::schema::SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, }, reduction: Reduction::by(vec![2]), filter: None, diff --git a/crates/types/src/stages/physical/memo_candidates.rs b/crates/types/src/stages/physical/memo_candidates.rs index 11357f3f3..6684c3441 100644 --- a/crates/types/src/stages/physical/memo_candidates.rs +++ b/crates/types/src/stages/physical/memo_candidates.rs @@ -132,6 +132,14 @@ pub struct PhysicalEdge { pub satisfies_physical_requirement: bool, } +impl PhysicalEdge { + /// Whether a plan may use the edge: $\alpha^{\log}_{u,c,i} = + /// \alpha^{\mathrm{phys}}_{u,c,i} = 1$. + pub fn is_logically_and_physically_admissible(&self) -> bool { + self.satisfies_logical_requirement && self.satisfies_physical_requirement + } +} + /// Stage 2's output, which Stage 3 selects from. #[derive(Debug, Clone)] pub struct PhysicalASAPMemoCandidates { @@ -163,13 +171,26 @@ impl PhysicalASAPMemoCandidates { edges: Vec, pruned_candidates: Vec>, ) -> Self { - let _ = ( + let mut admissible_edges_into_input: HashMap<_, Vec> = HashMap::new(); + let mut admissible_edges_out_of_candidate = vec![Vec::new(); physical_candidates.len()]; + for (id, edge) in edges.iter().enumerate() { + if edge.is_logically_and_physically_admissible() { + admissible_edges_into_input + .entry((edge.consumer_candidate_id, edge.input_index_of_consumer)) + .or_default() + .push(PhysicalEdgeId(id)); + admissible_edges_out_of_candidate[edge.producer_candidate_id.0] + .push(PhysicalEdgeId(id)); + } + } + Self { logical_candidates, physical_candidates, - edges, pruned_candidates, - ); - todo!("#1207 PR 5: index the admissible edges") + edges, + admissible_edges_into_input, + admissible_edges_out_of_candidate, + } } /// $\mathcal E$: every edge, in (consumer, input) order. @@ -188,18 +209,14 @@ impl PhysicalASAPMemoCandidates { consumer: PhysicalCandidateId, input_index_of_consumer: usize, ) -> &[PhysicalEdgeId] { - let _ = ( - consumer, - input_index_of_consumer, - &self.admissible_edges_into_input, - ); - todo!("#1207 PR 5") + self.admissible_edges_into_input + .get(&(consumer, input_index_of_consumer)) + .map_or(&[], Vec::as_slice) } /// The admissible edges out of `producer`: every consumer that may share it. pub fn admissible_edges_out_of(&self, producer: PhysicalCandidateId) -> &[PhysicalEdgeId] { - let _ = (producer, &self.admissible_edges_out_of_candidate); - todo!("#1207 PR 5") + &self.admissible_edges_out_of_candidate[producer.0] } } diff --git a/crates/types/src/stages/pruning.rs b/crates/types/src/stages/pruning.rs index fb994d1e4..6c885b0a6 100644 --- a/crates/types/src/stages/pruning.rs +++ b/crates/types/src/stages/pruning.rs @@ -28,12 +28,14 @@ pub enum PruneReason { /// Stage 2: it runs at ingestion time, and the source has no ingestion /// pipeline (its data is at rest or its arrival unknown). NoIngestionPipeline, - /// Stage 2: it scans raw data at query time, and the deployment does - /// not retain raw data, so no window of it is there to scan. + /// Stage 2: it scans raw data at query time, or retains the raw rows it + /// scans across evaluations, and the deployment does not retain raw + /// data, so no window of it is there to scan or keep. RawDataNotRetained, - /// Stage 2: it is a query-time result materialized (`OneEvaluation` or - /// `AcrossEvaluations`), and the deployment retains no query-time - /// results. + /// Stage 2: it is a query-time result materialized across evaluations + /// (`AcrossEvaluations`), and the deployment retains no query-time + /// result across evaluations + /// (`DeploymentCapabilities::query_time_results_retained`). QueryTimeResultsNotRetained, /// Stage 2: one of its inputs has no producer candidate that can be /// built: none is left, or every one depends, directly or through a diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index f39e5ccc5..861b6f3c3 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -1,17 +1,25 @@ //! Helpers that only tests call (this crate's integration tests and other //! crates' tests), exposed under the `test-support` feature so the normal //! public API stays as narrow as its production users need. Each forwards -//! to the crate-private function of the same name, except the fixture -//! [`population_over_selector`], which the executor's tests build. +//! to the crate-private function of the same name, except the fixtures: +//! [`population_over_selector`], which the executor's tests build, and the +//! memo fixtures the stage crates' tests share ([`series_schema`], +//! [`time_series_scan`], [`add_candidate`], [`group_with_own_candidate`]). use crate::ir::materialized_view::RawDataWindow; use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; use crate::ir::properties::summary_coverage::SummaryCoverage; -use crate::ir::schema::Schema; -use crate::ir::{ASAPOp, NonASAPOp}; +use crate::ir::operator::operator_properties::Source; +use crate::ir::schema::schema_support::with_promql_series_identity; +use crate::ir::schema::{DataType, Field, Schema, SketchParams}; +use crate::ir::{ASAPOp, NonASAPOp, Predicate}; use crate::ir::{OperatorNode, SchemaDerivationError}; +use crate::stages::{ + CandidateError, CandidateExpression, CandidateForMemoGroup, CandidateForMemoGroupId, + LogicalProperties, LogicalRequirements, Memo, MemoGroupId, OriginOfCandidateForMemoGroup, +}; use std::rc::Rc; /// `SummaryCoverage::definition_id`. @@ -140,3 +148,77 @@ pub fn population_over_selector( }); Rc::new(OperatorNode::with_schema(evaluation, root.schema.clone())) } + +/// `[ts, value, labels...]`, time index 0, with the series identity realized +/// as the planner realizes it before Stage 0 (`with_promql_series_identity`): +/// a closed schema whose last column is `$promql_series_identity`. +pub fn series_schema(labels: &[&str]) -> Schema { + let mut fields = vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ]; + fields.extend( + labels + .iter() + .map(|label| Field::plain(*label, DataType::Utf8, true)), + ); + let open = OperatorNode::new_shared(time_series_scan( + "m", + Schema::with_time_index(fields, 0, vec![]), + vec![], + )) + .expect("a scan schema with a time index"); + with_promql_series_identity(&open) + .expect("a PromQL scan") + .schema + .clone() +} + +/// A `TimeSeries(metric)` scan with `schema`, filtered by `predicates`. +pub fn time_series_scan( + metric: &str, + schema: Schema, + predicates: Vec>, +) -> Operator { + Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates, + schema, + }) +} + +/// Adds `operator` with `parameters` to `group` as Pass 1 does: no input +/// requirement and no fused group. +pub fn add_candidate( + memo: &mut Memo, + group: MemoGroupId, + operator: Operator, + parameters: Option, +) -> Result { + let expression = CandidateExpression { operator }; + let logical_properties = LogicalProperties::new(&expression, parameters) + .expect("parameters from the algorithm's list"); + let input_requirements = + vec![LogicalRequirements::default(); expression.operator.children().len()]; + memo.add_candidate( + group, + CandidateForMemoGroup { + expression, + logical_properties, + input_requirements, + fused_groups: Vec::new(), + origin: OriginOfCandidateForMemoGroup::Pass1, + }, + ) +} + +/// The group of `operator`, with `operator` as its own candidate. +pub fn group_with_own_candidate(memo: &mut Memo, operator: Operator) -> MemoGroupId { + let group = memo + .find_or_add_group(operator.clone()) + .expect("an expression the node rules accept"); + add_candidate(memo, group, operator, None).expect("the group's own expression"); + group +} diff --git a/docs/design_docs/concepts/glossary.md b/docs/design_docs/concepts/glossary.md index 8b52e6ef6..0e8ccfc15 100644 --- a/docs/design_docs/concepts/glossary.md +++ b/docs/design_docs/concepts/glossary.md @@ -42,7 +42,7 @@ A node of the physical DAG whose output is materialized across evaluations (life ## Raw data sample -One timestamped value of a time series as it arrives, such as one Prometheus Remote Write sample. Raw data samples are the rows a time-series scan reads. Code names them with `raw_data_sample`, e.g. `WeightDomain::RawDataSamplesOfMetric`, `raw_data_samples_of_metric`. +One timestamped value of a time series as it arrives, such as one Prometheus Remote Write sample. Raw data samples are the rows a time-series scan reads. Code names them with `raw_data_sample`, e.g. `RawDataWindow`. ## Summary sample diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index d28a66a93..30d19f311 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -1271,13 +1271,13 @@ This part of the code answers two questions about summary state: **Update input examples** (as the planner builds them in Stage 1, `asap-stage1-pass1-per-computation-candidates`): -| Query | Sketch | `item` | `weight` | `weight_domain` | -|---|---|---|---|---| -| p99 of `latency` | KLL | none: KLL has no keys | `Column(latency)`: the value itself | `UnknownOrSigned` | -| how often each `endpoint` occurs | Count-Min | `Column(endpoint)` | `Constant(1.0)`: each row counts once | `NonNegative(UnitCount)` | -| `topk(5, rate(http_requests_total[5m]))` | Count-Min with heap | `Tuple(every column except value and the group columns)`: one item per series, including its timestamp | `Column(value)`: the rate | `NonNegative(ResetAwareCounterDerivative)` | +| Query | Sketch | `item` | `weight` | +|---|---|---|---| +| p99 of `latency` | KLL | none: KLL has no keys | `Column(latency)`: the value itself | +| how often each `endpoint` occurs | Count-Min | `Column(endpoint)` | `Constant(1.0)`: each row counts once | +| `topk(5, rate(http_requests_total[5m]))` | Count-Min with heap | `Tuple(every column except value and the group columns)`: one item per series, including its timestamp | `Column(value)`: the rate | -`weight_domain` matters because some sketches (e.g. Count-Min) are only accurate when weights are never negative. The planner records why a weight is non-negative; if it cannot prove it, the weight counts as possibly negative. +Some sketches (e.g. Count-Min) are only accurate when weights are never negative. The sign of the weights is not part of the update: it is the state group's `column_sign`, derived from the input groups and the declared source signs, and Stage 2 prunes a Count-Min build whose weights are not proven non-negative (`CountMinWeightsNotProvenNonNegative`). **Evaluation examples:** `Quantile { q: 0.99 }` reads p99 from a KLL; `PointCount { key: endpoint, value: Some("checkout") }` reads how often `checkout` occurred from a Count-Min; `TopK { k: 5 }` reads the 5 heaviest items. @@ -1290,25 +1290,11 @@ pub struct SummaryUpdate { /// The value added to the state: the observed value for a KLL, or the /// count to add for a Count-Min (often `Constant(1.0)`). pub weight: SummaryInputExpr, - /// Whether `weight` is proven never negative. Some algorithms need that. - /// Missing proof is never treated as non-negative. - pub weight_domain: WeightDomain, } impl SummaryUpdate { - /// Add the value of column `c` for each row: no item, and the weight is - /// not known to be non-negative. + /// Add the value of column `c` for each row, with no item. pub fn column(c: ColumnRef) -> Self; } -pub enum WeightDomain { - /// Nothing is known; the weight may be negative. The default. - UnknownOrSigned, - /// The weight is never negative. `proof`: why. - NonNegative { proof: NonNegativeWeightProof }, -} -pub enum NonNegativeWeightProof { - UnitCount, // every row adds 1 - ResetAwareCounterDerivative, // PromQL increase/rate over counters with reset correction -} /// An expression that computes an item or a weight from the input row. pub enum SummaryInputExpr { Constant(f64), // the same number for every row diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 5d7529890..520ef05a9 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -563,16 +563,14 @@ The sections below describe each stage. | 0. Frontends | nothing: a query that cannot be lowered fails planning with an error | — | — | | 1, Pass 1 | what the measure or the deployment cannot compute: `RequiresExact`, `NullableAvgInput`, `NoRelationalCountSketchRealization`, `UnsupportedByDeployment`, `QuantileOutsideUnitInterval`, `TopKHeapOverOneRowPerItem`; and, only when the deployment's accuracy model and evidence cannot accept an output without its guarantee, a summary no query reading it can use: `NoErrorBoundForQueryTarget` | `PerComputationLogicalMemoCandidates::pruned_candidates`, each on its aggregate's memo group | the way to compute the measure (`HowToComputeAggregateMeasure`) | | 1, Pass 2 | nothing: it only adds candidates | — | — | -| 2 | what cannot be built or materialized: `NoIngestionPipeline`, `RawDataNotRetained` (a query-time scan of raw data the deployment does not keep), `QueryTimeResultsNotRetained`, `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`; the planner does not check executor support | `PhysicalASAPMemoCandidates::pruned_candidates` | the `PhysicalCandidate` it would have been | +| 2 | what cannot be built or materialized: `NoIngestionPipeline`, `RawDataNotRetained` (a scan of raw data the deployment does not keep, at query time or materialized across evaluations), `QueryTimeResultsNotRetained`, `CountMinWeightsNotProvenNonNegative`, `InputHasNoBuildableProducer`, `ReachesNoQueryOutput`; the planner does not check executor support | `PhysicalASAPMemoCandidates::pruned_candidates` | the `PhysicalCandidate` it would have been | | 3 | valid candidates that lose, and the checks only a whole plan can make (accuracy, latency, cluster capacity) | its own `RejectionCause` | — | -Stage 2's key names what a pruned candidate would have been, since kept -candidates are renumbered after pruning and a pruned one has no -`PhysicalCandidateId`. Stage 2 emits one candidate per operator node, -timing and lifetime, and makes no copies of it, so (`operator_node`, -`properties`) is unique. Operator -nodes are kept whole, so `operator_node` indexes the output's -`operator_nodes_of_candidates_for_memo_groups`. Pass 1's accuracy pruning +Stage 2 records a pruned candidate as the `PhysicalCandidate` it would +have been, since kept candidates are renumbered after pruning and a pruned +one has no `PhysicalCandidateId`. Stage 2 emits one candidate per logical +candidate, timing and lifetime, and makes no copies of it, so +(`logical_candidate_id`, `physical_properties`) is unique. Pass 1's accuracy pruning (`NoErrorBoundForQueryTarget`) runs inside Pass 1's entry, before the candidates are attached to the memo groups, and records a `PrunedCandidate` like every other Pass 1 prune. @@ -596,7 +594,7 @@ the memo path's entry points. | 1, Pass 1 | Logical roots; the deployment's supported summaries → the Pass 1 inventory, every computation's candidates (`PerComputationCandidates`, now Pass 1's crate-private working form) | Logical roots → `DagOfLogicalMemoGroups` (hash-consed `MemoGroup`s); with the Pass 1 inventory, less the summaries the deployment does not support and those no query reading them can use → `PerComputationLogicalMemoCandidates`: the `Memo` (each group's `CandidateForMemoGroup`s, with the accuracy-scoped sizes) and the pruned candidates (`PrunedCandidate`) | `memo_of_queries`, `pass1_memo` | | 1, Pass 2 | One Pass 1 choice per computation → that choice rewritten by each sharing rule, every subset of sharing choices (no longer in the code) | `Memo`; the entries' recurrence and range steps → the same memo with each rule's candidates added to the groups they implement | the rules of `stage1-pass2-reuse-candidates/src/rules/` | | 1, concrete summary-state groups | (part of Pass 2) | `Memo` → the memo with concrete summary-state groups: one group per shared size or tumbling-window layout that readers can read | `add_shared_summary_state_groups`, Pass 2's last step; the Pass 2 output is `LogicalASAPMemoCandidates` (`generate_logical_asap_memo_candidates`) | -| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates`; entries, `DataWorkload`, query-time and raw-data retention → `PhysicalASAPMemoCandidates`: every operator node of every candidate with each allowed timing and lifetime (one physical candidate each, no copies), the producers of each input, reader constraints (each applied to every selected run of its candidate), and the pruned candidates (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | +| 2 | One whole-workload logical candidate; entries (recurrence, predictability), `DataWorkload`, query-time retention → its physical candidates, one per physical properties assignment | `LogicalASAPMemoCandidates` (with each group's evaluation grid); `DataWorkload` (arrival), `DeploymentCapabilities` (raw-data and query-time retention) → `PhysicalASAPMemoCandidates`: every logical candidate Pass 2 kept with each offered timing and lifetime (one physical candidate each, no copies), every logical edge expanded into physical edges with $\alpha^{\log}$ and $\alpha^{\mathrm{phys}}$, and the pruned alternatives (`PrunedCandidate`) | `generate_physical_asap_memo_candidates` (`asap_stage2_physical_optimization`) | | 3 | Every physical candidate; accuracy targets, accuracy model and evidence, cost model → one plan | `PhysicalASAPMemoCandidates`; entries, `DataWorkload`, `AccuracyCheckInputs`, `CostInputs`, Stage 2's realizer → `Selection`: the selected plan (`selected_dag`, a `DagOfSelectedCandidates`: one node per run of a candidate, so a candidate recomputed by two readers or shared as two instances over different inputs is two nodes, with an edge per input), its cost, the accepted accuracy per query and the search report | `select_plan` (`asap_stage3_plan_selection`); the facade (`run_planning`, output `PlanningRun`) runs it once and builds the selected `PhysicalWorkloadPlan` with `AssignmentRealizer::realize`, one plan node per selected node | | 4. Execution | The selected `PhysicalASAPDAG` → query results | Same: an `InstallablePlan` (`{dag, query_texts}`) → query results | The deployment runtime installs the `InstallablePlan` | @@ -1283,10 +1281,13 @@ On the memo, Stage 2 prunes per operator node and records each pruned candidate in `PhysicalASAPMemoCandidates::pruned_candidates` ([Pruning by stage](#pruning-by-stage)). It does not generate a configuration the deployment cannot run: an ingestion-time alternative -without an ingestion pipeline (`NoIngestionPipeline`), a query-time scan -of raw data the deployment does not retain (`RawDataNotRetained`), and a -materialized query-time result the deployment does not retain -(`QueryTimeResultsNotRetained`). It does not check executor support. It +without an ingestion pipeline (`NoIngestionPipeline`), a scan of raw data +the deployment does not retain, at query time or materialized across +evaluations (`RawDataNotRetained`), a +query-time result materialized across evaluations the deployment does not +retain (`QueryTimeResultsNotRetained`), and a Count-Min build whose weights +are not proven non-negative (`CountMinWeightsNotProvenNonNegative`). It does +not check executor support. It then removes the candidates no buildable input or query output can use (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`), repeating until nothing more is removed. Stage 2 makes no execution-scope diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 1e29571d3..011c86bd6 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -95,8 +95,8 @@ asap-stage1-pass2-reuse-candidates -p asap-stage2-physical-optimization -p asap- `PhysicalASAPMemoCandidates::pruned_candidates` as `PrunedCandidate` with a `PruneReason`: an ingestion-time alternative without an ingestion pipeline - (`NoIngestionPipeline`), a query-time scan of raw data the deployment does - not retain (`RawDataNotRetained`) and a materialized query-time result the + (`NoIngestionPipeline`), a scan of raw data the deployment does not retain, + at query time or materialized across evaluations (`RawDataNotRetained`) and a materialized query-time result the deployment does not retain (`QueryTimeResultsNotRetained`) are not generated; a candidate no buildable input or query output can use is removed (`InputHasNoBuildableProducer`, `ReachesNoQueryOutput`). The