From fc1db61348483f559643a94a363f8908d2215e02 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:25:43 +0000 Subject: [PATCH 1/2] refactor(types, stages): physical properties off the logical IR OperatorNode has no timing and no lifetime: a logical candidate has only its logical properties (#1200), so frontends, Stage 0, Stage 1 and accuracy propagation cannot see physical properties. A node's (timing, lifetime) lives only in PhysicalPropertiesAssignment, keyed by node: Stage 2's choice in, the timing pass's completed assignment out. - apply_physical_properties[_to_roots] return the completed assignment instead of rebuilding timed copies; NodeToTimedNodeCache is deleted. - Export, tumbling-window expansion and with_confirmed_weights take or carry the assignment; PhysicalWorkloadPlan, PlanOutput and WholePlanSelection hold the selected plan's physical_properties, which replace the properties pinned on Stage 3's selected nodes. - An entry with lifetime None fixes a node's timing (what a pin did); the pin conflict and invalid-pin errors are deleted, as are the Stage 0/1/2 comparisons, copies and checks of the two fields. - Logical FlatDag nodes lose their timing/lifetime keys; the DAG viewer and its fixtures change with them. Wire format unchanged (INSTALLABLE_PLAN_WIRE_VERSION 12); corpus and sketch-plan snapshots unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K1M6br9PL4SDeVS1XpezKJ --- crates/accuracy/src/propagation.rs | 16 +- crates/cost/tests/ingestion_refresh_runs.rs | 2 +- crates/devtools/src/bin/asap_planner.rs | 20 +- crates/devtools/src/memo_pricing.rs | 1 + crates/devtools/src/planning_stages.rs | 11 +- crates/devtools/tests/asap_planner.rs | 1 + .../executor/src/physical_planner/logical.rs | 2 - crates/executor/src/physical_planner/mod.rs | 22 +- crates/executor/tests/common/mod.rs | 11 +- crates/executor/tests/time_series_window.rs | 14 +- .../executor/tests/weighted_topk_binding.rs | 23 +- crates/frontend-promql/tests/support.rs | 14 +- crates/frontend-sql/tests/pearson_corr.rs | 13 +- crates/integration-tests/src/lib.rs | 27 +- .../tests/exact_composition.rs | 14 +- .../tests/frequency_execution.rs | 13 +- .../tests/kll_tumbling_window_execution.rs | 17 +- .../tests/materialized_views.rs | 12 +- .../tests/operator_design_examples.rs | 13 +- .../tests/per_series_exact_execution.rs | 13 +- .../tests/physical_common/mod.rs | 5 +- .../tests/physical_properties_e2e.rs | 8 +- .../tests/precompute_raw_samples.rs | 7 +- .../tests/promql_to_post_asap.rs | 10 +- crates/integration-tests/tests/reuse_rules.rs | 25 +- .../tests/runtime_handoff.rs | 3 +- .../tests/tumbling_window_merge.rs | 35 +- .../src/pass1/per_computation_candidates.rs | 5 - .../pass1/per_computation_candidates_tests.rs | 45 +- .../src/pass2/memo/rules/reuse.rs | 6 - .../src/pass2/memo/rules/scan_labels.rs | 2 - .../rules/shared_configuration_readers.rs | 18 +- .../src/pass2/memo/state_groups.rs | 4 - .../src/pass2/memo/window_candidates.rs | 6 +- .../src/stage0/cse.rs | 13 +- .../src/test_support.rs | 25 +- .../src/catalog/configurations.rs | 7 - .../src/catalog/factoring.rs | 4 - .../src/catalog/inventory.rs | 7 - .../src/catalog/realize.rs | 22 +- crates/physical-asap-optimization/src/lib.rs | 43 +- .../src/split_by_timing.rs | 119 +-- .../src/groups/accuracy_positions.rs | 9 +- crates/plan-selection/src/lib.rs | 63 +- crates/plan-selection/src/report.rs | 4 +- crates/plan-selection/src/selector.rs | 4 +- crates/planner/src/installable.rs | 61 +- crates/planner/src/pass.rs | 27 +- crates/planner/src/run.rs | 14 +- crates/planner/tests/runtime_install.rs | 81 +- crates/types/src/ir/flat.rs | 7 - crates/types/src/ir/mod.rs | 2 +- crates/types/src/ir/operator/node.rs | 63 +- .../src/ir/operator/tumbling_window_chain.rs | 87 +- crates/types/src/ir/physical_export.rs | 49 +- .../src/ir/properties/execution_data_state.rs | 32 +- crates/types/src/ir/properties/timing.rs | 962 +++++++----------- crates/types/src/ir/schema/schema_support.rs | 2 - crates/types/src/physical/properties.rs | 8 + crates/types/src/physical/workload_plan.rs | 9 +- crates/types/src/test_support.rs | 13 +- crates/types/src/workload/bounds.rs | 42 +- crates/types/tests/materialized_view.rs | 32 +- crates/types/tests/physical_export.rs | 165 ++- crates/types/tests/schema_rebuilding.rs | 39 +- crates/types/tests/structure_contract.rs | 31 - crates/types/tests/summary_coverage.rs | 25 +- crates/types/tests/summary_merge_structure.rs | 19 +- crates/types/tests/tumbling_window_chain.rs | 40 +- docs/design_docs/concepts/post-asap-ir.md | 28 +- docs/develop_docs/library-api.md | 4 +- docs/develop_docs/pre-asap-ir.md | 13 +- tools/dag-viewer/stages.js | 3 +- .../test-fixtures/memo-enumeration-small.json | 2 +- .../dag-viewer/test-fixtures/memo-small.json | 2 +- 75 files changed, 1052 insertions(+), 1568 deletions(-) diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index cace7af85..2c15ba4b8 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -122,7 +122,7 @@ pub fn root_guarantee(root: &QueryRoot) -> Option { /// Bind known statistics in existing contracts, then recover compositions /// that previously required constant child bounds. Physical state contracts, -/// schemas, timing, lifetime, and sharing are preserved. This does not +/// schemas and sharing are preserved. This does not /// infer facts: `bind` must leave unsupported statistics unknown. pub fn bind_guarantees( root: &QueryRoot, @@ -137,9 +137,7 @@ pub fn bind_guarantees( return done.clone(); } let operator = node.operator.map_children(|child| walk(child, bind, cache)); - let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime); + let mut rebuilt = OperatorNode::with_schema(operator, node.schema.clone()); rebuilt.guarantee = node .guarantee .as_ref() @@ -162,8 +160,8 @@ pub fn bind_guarantees( /// `roots` with every sketch summary state rebuilt with the parameters /// `resize` gives it (same algorithm; `None` keeps them), and every node /// above one rebuilt over its new inputs with its guarantee composed again -/// ([`composed_guarantee`]; a node that had none keeps none). Timing, -/// materialization and sharing are kept. Pass 1 sizes summaries +/// ([`composed_guarantee`]; a node that had none keeps none). Sharing is +/// kept. Pass 1 sizes summaries /// for a failure probability this way. pub fn with_resized_summaries( roots: &[QueryRoot], @@ -212,11 +210,7 @@ pub fn with_resized_summaries( .guarantee .as_ref() .and_then(|_| composed_guarantee(&rebuilt)); - Rc::new( - rebuilt - .with_timing(node.timing) - .with_lifetime(node.lifetime), - ) + Rc::new(rebuilt) } else { node.clone() }; diff --git a/crates/cost/tests/ingestion_refresh_runs.rs b/crates/cost/tests/ingestion_refresh_runs.rs index e41dfdcee..5b5bc6d48 100644 --- a/crates/cost/tests/ingestion_refresh_runs.rs +++ b/crates/cost/tests/ingestion_refresh_runs.rs @@ -97,7 +97,7 @@ fn dag_of( }) .collect(); let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - compile_physical_asap_workload(&timed).unwrap() + compile_physical_asap_workload(&roots, &timed).unwrap() } /// `(multiplier, per_second)` of `dag`'s `Scan`, each query every 10 s. diff --git a/crates/devtools/src/bin/asap_planner.rs b/crates/devtools/src/bin/asap_planner.rs index 3aebed273..000be26ab 100644 --- a/crates/devtools/src/bin/asap_planner.rs +++ b/crates/devtools/src/bin/asap_planner.rs @@ -395,19 +395,29 @@ fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value let mut stage3 = selection_json(&run.run.selection); match (&run.run.selection, &run.run.selected_plan) { (Ok(selection), Some(roots)) => { - let tumbling_window_retention_ms = run + let (tumbling_window_retention_ms, physical_properties) = run .run .selected_physical_plan .as_ref() - .map(|realized| realized.tumbling_window_retention_ms.clone()) + .map(|realized| { + ( + realized.tumbling_window_retention_ms.clone(), + realized.plan.physical_properties.clone(), + ) + }) .unwrap_or_default(); stage3["tumbling_window_retention_ms"] = tumbling_window_retention_ms .iter() .map(|(node, ms)| (node.to_string(), json!(ms))) .collect::>() .into(); - match materialized_views_json(roots, selection, &tumbling_window_retention_ms, entries) - { + match materialized_views_json( + roots, + &physical_properties, + selection, + &tumbling_window_retention_ms, + entries, + ) { Ok(views) => stage3["materialized_views"] = Value::Array(views), Err(error) => { stage3["materialized_views"] = json!([]); @@ -646,6 +656,7 @@ fn arrival_together(earlier: DataArrival, later: DataArrival) -> Option Result, String> { let (plan, report) = installable_plan_of( roots, + physical_properties, Some(selection), tumbling_window_retention_ms, entries, diff --git a/crates/devtools/src/memo_pricing.rs b/crates/devtools/src/memo_pricing.rs index d099adf41..1e45af73f 100644 --- a/crates/devtools/src/memo_pricing.rs +++ b/crates/devtools/src/memo_pricing.rs @@ -145,6 +145,7 @@ fn price_choice( Ok(selection) => { match installable_plan_of( &selection.plan, + &selection.physical_properties, None, &selection.tumbling_window_retention_ms, entries, diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index b40f36dea..c401d5d22 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -431,7 +431,7 @@ pub fn run_stages_until( ) { Ok(selection) => { match realizer.realize(&physical, &selection.selected_dag) { - Ok(realized) => { + Ok(mut realized) => { // As the facade does: the plan to run carries the weight // proofs Stage 3 made. let proven: HashSet<*const asap_types::ir::OperatorNode> = selection @@ -440,9 +440,12 @@ pub fn run_stages_until( .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) .map(Rc::as_ptr) .collect(); - let plan = with_confirmed_weights_at(&realized.plan.roots, data, &|node| { - proven.contains(&Rc::as_ptr(node)) - }); + let plan = with_confirmed_weights_at( + &realized.plan.roots, + data, + &|node| proven.contains(&Rc::as_ptr(node)), + &mut realized.plan.physical_properties, + ); validate(&plan)?; run.selected_plan = Some(plan); run.selected_physical_plan = Some(realized); diff --git a/crates/devtools/tests/asap_planner.rs b/crates/devtools/tests/asap_planner.rs index 2fda7acad..7bbed98db 100644 --- a/crates/devtools/tests/asap_planner.rs +++ b/crates/devtools/tests/asap_planner.rs @@ -1630,6 +1630,7 @@ fn scans_projecting_different_labels_are_shared() { let (_, reference) = plan("partial-sharing", &partial); let (installed, _) = asap_planner::installable_plan_of( &physical.roots, + &physical.physical_properties, None, &offered.tumbling_window_retention_ms, &entries, diff --git a/crates/executor/src/physical_planner/logical.rs b/crates/executor/src/physical_planner/logical.rs index 774a38843..4b4d58ff1 100644 --- a/crates/executor/src/physical_planner/logical.rs +++ b/crates/executor/src/physical_planner/logical.rs @@ -36,8 +36,6 @@ pub(super) fn restore(dag: &PhysicalASAPDAG) -> Result = windows - .iter() - .map(|window| { - let timed = apply_physical_properties( - window, - &PhysicalPropertiesAssignment::default(), - &mut cache, - ) + let roots: Vec<_> = windows.into_iter().map(QueryRoot::Operator).collect(); + let timed = + apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) .unwrap(); - QueryRoot::Operator(timed) - }) - .collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); + let dag = compile_physical_asap_workload(&roots, &timed).unwrap(); let roots = dag.roots.iter().map(|&id| id as NodeId).collect(); (dag, roots) } diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index 0d9ff5820..ada7a1ef4 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -1,20 +1,15 @@ #![allow(dead_code)] use planner_types::ir::physical_export::PhysicalASAPDAG; use planner_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, OperatorNode, PhysicalPropertiesAssignment, - QueryRoot, + apply_physical_properties, OperatorNode, PhysicalPropertiesAssignment, QueryRoot, }; use std::rc::Rc; pub fn compile_physical_asap_dag( root: &Rc, ) -> Result> { - let root = apply_physical_properties( - root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::default(), - )?; - Ok(planner_types::ir::physical_export::compile_physical_asap_dag(&root)?) + let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default())?; + Ok(planner_types::ir::physical_export::compile_physical_asap_dag(root, &timed)?) } /// The logical DAG the planner pipeline (`asap_planner::run_planning`) selects for the diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index 91bcd78f0..6f91223ef 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -12,8 +12,8 @@ use planner_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAP use planner_types::ir::scalar::{CompareOpKind, ScalarValue}; use planner_types::ir::schema::{DataType, Field, Schema, PROMQL_SERIES_IDENTITY}; use planner_types::ir::{ - apply_physical_properties, ExprSemantics, NodeToTimedNodeCache, NonASAPOp, Operator, - OperatorNode, PhysicalPropertiesAssignment, Predicate, ScalarExpr, TimeRangeKind, + apply_physical_properties, ExprSemantics, NonASAPOp, Operator, OperatorNode, + PhysicalPropertiesAssignment, Predicate, ScalarExpr, TimeRangeKind, }; use std::collections::BTreeMap; use std::sync::Arc; @@ -84,13 +84,9 @@ fn window_over_rows( child, })) .unwrap(); - let timed = apply_physical_properties( - &range, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() + let timed = + apply_physical_properties(&range, &PhysicalPropertiesAssignment::default()).unwrap(); + compile_physical_asap_dag(&range, &timed).unwrap() } /// `(instance, ts, value)` samples of `latency_ms` as a store's raw data. diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index 5f3ff145f..0dc3bc6d1 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -69,6 +69,7 @@ fn confirmed_as_counter( match planner_types::workload::bounds::with_confirmed_weights( &[QueryRoot::Operator(plan)], &data, + &mut Default::default(), ) .remove(0) { @@ -274,24 +275,12 @@ fn rate_updates_cannot_enter_integer_heap_factory() { /// it upstream of a query-time node, so the pin is lifted first. fn continuously_maintained_dag(candidate: &Rc) -> PhysicalASAPDAG { use planner_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, Operator as IrOperator, OperatorNode, + apply_physical_properties, Operator as IrOperator, OperatorNode, PhysicalPropertiesAssignment, }; use planner_types::physical::PhysicalProperties; - fn unpinned(node: &Rc) -> Rc { - let mut copy = (**node).clone(); - copy.operator = node.operator.map_children(unpinned); - if matches!( - copy.operator, - IrOperator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - ) { - copy.timing = None; - } - Rc::new(copy) - } - let candidate = unpinned(candidate); let mut assignment = PhysicalPropertiesAssignment::default(); - for node in OperatorNode::reachable(&candidate) { + for node in OperatorNode::reachable(candidate) { if matches!(node.operator, IrOperator::ASAP(ASAPOp::SummaryAgg { .. })) { assignment.set( &node, @@ -299,8 +288,6 @@ fn continuously_maintained_dag(candidate: &Rc) ); } } - let timed = - apply_physical_properties(&candidate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - planner_types::ir::physical_export::compile_physical_asap_dag(&timed).unwrap() + let timed = apply_physical_properties(candidate, &assignment).unwrap(); + planner_types::ir::physical_export::compile_physical_asap_dag(candidate, &timed).unwrap() } diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 715878589..53a5d8d8c 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -83,16 +83,10 @@ pub fn promql_scalar(node: &ScalarExpr) -> Option { pub fn physical_asap_dag( root: &Rc, ) -> asap_types::ir::physical_export::PhysicalASAPDAG { - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, PhysicalPropertiesAssignment, - }; - let timed = apply_physical_properties( - root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default physical properties"); - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) + use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment}; + let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) + .expect("default physical properties"); + asap_types::ir::physical_export::compile_physical_asap_dag(root, &timed) .expect("post-ASAP DAG export") } diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 143dcba11..8e6648bdd 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -6,8 +6,8 @@ use asap_types::ir::operator::AggIntent; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::{ - apply_physical_properties, physical_export::compile_physical_asap_dag, NodeToTimedNodeCache, - NonASAPOp, OperatorNode, PhysicalPropertiesAssignment, ScalarExpr, + apply_physical_properties, physical_export::compile_physical_asap_dag, NonASAPOp, OperatorNode, + PhysicalPropertiesAssignment, ScalarExpr, }; fn catalog() -> SqlCatalog { @@ -179,11 +179,6 @@ async fn corr_survives_exact_plan_compilation() { // operator DAG, no ASAP node added. assert!(!plan.contains_asap(), "expected the exact plan"); assert_eq!(aggregate(plan).0, aggregate(&query).0); - let timed = apply_physical_properties( - plan, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap(); + let timed = apply_physical_properties(plan, &PhysicalPropertiesAssignment::default()).unwrap(); + compile_physical_asap_dag(plan, &timed).unwrap(); } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index a179eb582..3c735bb8b 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -98,20 +98,19 @@ pub mod fixtures { /// Timing and physical ASAP DAG export helpers for planned candidates. pub mod physical_export { use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPDAG}; - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, OperatorNode, PhysicalPropertiesAssignment, - }; + use asap_types::ir::{apply_physical_properties, OperatorNode, PhysicalPropertiesAssignment}; use std::rc::Rc; - /// Time `root` under the default assignment (no sub-DAG materialized, so every - /// summary computed at query time). Returns the timed copy; read `node.timing` on it. - pub fn timed(root: &Rc) -> Rc { + /// The physical properties of every node of `root` under the default + /// assignment (no sub-DAG materialized, so every summary computed at + /// query time). + pub fn timed(root: &Rc) -> PhysicalPropertiesAssignment { timed_with(root, &PhysicalPropertiesAssignment::default()) } - /// Time `root` with every sub-DAG materialized at ingestion time where the DAG - /// allows it. - pub fn at_ingestion(root: &Rc) -> Rc { + /// The physical properties of every node of `root` with every sub-DAG + /// materialized at ingestion time where the DAG allows it. + pub fn at_ingestion(root: &Rc) -> PhysicalPropertiesAssignment { timed_with( root, &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( @@ -123,20 +122,20 @@ pub mod physical_export { fn timed_with( root: &Rc, assignment: &PhysicalPropertiesAssignment, - ) -> Rc { - apply_physical_properties(root, assignment, &mut NodeToTimedNodeCache::new()) - .expect("applying physical properties failed") + ) -> PhysicalPropertiesAssignment { + apply_physical_properties(root, assignment).expect("applying physical properties failed") } /// Time `root` (default assignment), then export the physical DAG. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&timed(root)).expect("physical ASAP DAG export failed") + compile_physical_asap_dag(root, &timed(root)).expect("physical ASAP DAG export failed") } /// Time `root` with every sub-DAG materialized at ingestion time where the DAG allows /// it, then export the physical DAG. pub fn at_ingestion_physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(&at_ingestion(root)).expect("physical ASAP DAG export failed") + compile_physical_asap_dag(root, &at_ingestion(root)) + .expect("physical ASAP DAG export failed") } } diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index 63d051298..738fc115f 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -197,8 +197,8 @@ fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { }) .unwrap_or_else(|| panic!("{kind:?}: a KLL over the exact accumulator")); // Timing is not stored on the plan: time it with the outer summary - // maintained (which also validates every edge) and inspect the copy. - let root = at_ingestion(&root); + // maintained (which also validates every edge) and inspect it. + let timed = at_ingestion(&root); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { panic!("expected KLL evaluation, got {:?}", root.operator); }; @@ -210,7 +210,7 @@ fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { panic!("{kind:?}: missing maintenance finalization"); }; assert_eq!( - child.timing, + timed.get(child).map(|properties| properties.timing()), Some(ExecutionTiming::IngestionTime), "{kind:?}: finalization runs at maintenance time" ); @@ -254,7 +254,9 @@ fn max_and_avg_over_quantile_compose_at_query_time() { "the composed plan's schema is the pre-ASAP target's own" ); assert_eq!( - timed(&composed).timing, + timed(&composed) + .get(&composed) + .map(|properties| properties.timing()), Some(ExecutionTiming::QueryTime), "{intent:?}: the exact fold runs at query time" ); @@ -274,7 +276,9 @@ fn identity_and_genuine_multi_row_folds_both_compose() { let root = agg(vec![0], AggIntent::Max { col: None }, inner); let composed = composed(&root); assert_eq!( - timed(&composed).timing, + timed(&composed) + .get(&composed) + .map(|properties| properties.timing()), Some(ExecutionTiming::QueryTime), "{label}" ); diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs index f48adf2c6..a4fe275a3 100644 --- a/crates/integration-tests/tests/frequency_execution.rs +++ b/crates/integration-tests/tests/frequency_execution.rs @@ -14,8 +14,7 @@ use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOpe use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm}; use asap_types::ir::{ - apply_physical_properties, ASAPOp, NodeToTimedNodeCache, NonASAPOp, OperatorNode, - PhysicalPropertiesAssignment, + apply_physical_properties, ASAPOp, NonASAPOp, OperatorNode, PhysicalPropertiesAssignment, }; use crate::physical_common; @@ -36,13 +35,9 @@ fn built_family(candidate: &Rc) -> Option { /// Execute `candidate` at query time over `SAMPLES`, all inside the window. fn execute(candidate: &Rc) -> Vec { - let timed = apply_physical_properties( - candidate, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = + apply_physical_properties(candidate, &PhysicalPropertiesAssignment::default()).unwrap(); + let dag = compile_physical_asap_dag(candidate, &timed).unwrap(); let ranges = dag .nodes .iter() diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index c81bfd754..852fc2c95 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -120,15 +120,14 @@ fn fixture() -> Fixture { query: SketchStatistic::Quantile { q }, })) }; - let roots = apply_physical_properties_to_roots( - &[ - QueryRoot::Operator(estimate(0.5)), - QueryRoot::Operator(estimate(0.99)), - ], - &PhysicalPropertiesAssignment::default(), - ) - .unwrap(); - let compiled = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); + let roots = [ + QueryRoot::Operator(estimate(0.5)), + QueryRoot::Operator(estimate(0.99)), + ]; + let timed = + apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) + .unwrap(); + let compiled = compile_physical_asap_workload_with_node_ids(&roots, &timed).unwrap(); let mut tumbling_window_ids = vec![0; 5]; let (mut merge, mut p50, mut p99) = (0, 0, 0); for node in &compiled.dag.nodes { diff --git a/crates/integration-tests/tests/materialized_views.rs b/crates/integration-tests/tests/materialized_views.rs index ffd7652e2..60eddd576 100644 --- a/crates/integration-tests/tests/materialized_views.rs +++ b/crates/integration-tests/tests/materialized_views.rs @@ -81,7 +81,8 @@ async fn raw_data_retention_is_the_lookback_stage3_prices() { [RawDataRetention::new(m.clone(), Some(65 * 60_000))] ); let output = e2e_plan(input()).await.unwrap(); - let selected = compile_physical_asap_workload(&output.roots()).unwrap(); + let selected = + compile_physical_asap_workload(&output.roots(), &output.physical_properties).unwrap(); assert_eq!(stage3_raw_lookback_ms(&selected), [(m, Some(65 * 60_000))]); let capabilities = DeploymentCapabilities { @@ -113,7 +114,14 @@ fn a_range_evaluated_query_time_scan_keeps_its_window_plus_the_span() { ); let roots = asap_devtools::planning_stages::lower_promql_roots(&workload).unwrap(); let entries: Vec<_> = workload.query_workload.entries().collect(); - let (plan, _) = installable_plan_of(&roots, None, &Default::default(), &entries).unwrap(); + let (plan, _) = installable_plan_of( + &roots, + &Default::default(), + None, + &Default::default(), + &entries, + ) + .unwrap(); plan.validate().unwrap(); assert!(plan.materialized_views().unwrap().is_empty()); assert_eq!( diff --git a/crates/integration-tests/tests/operator_design_examples.rs b/crates/integration-tests/tests/operator_design_examples.rs index fac5da210..a0130fbeb 100644 --- a/crates/integration-tests/tests/operator_design_examples.rs +++ b/crates/integration-tests/tests/operator_design_examples.rs @@ -309,10 +309,15 @@ async fn batch_planning_replaces_summary_operators() { panic!("one SUM state for both queries: {states:?}") }; assert!(output.operators().into_iter().any(|n| { - matches!(n.lifetime, asap_types::physical::Lifetime::OneEvaluation(_)) - && OperatorNode::reachable(&n) - .iter() - .any(|m| Rc::ptr_eq(m, state)) + matches!( + output + .physical_properties + .get(&n) + .map(|properties| properties.lifetime()), + Some(asap_types::physical::Lifetime::OneEvaluation(_)) + ) && OperatorNode::reachable(&n) + .iter() + .any(|m| Rc::ptr_eq(m, state)) })); for (plan, expected) in output.plans.iter().zip([31.0, 60.0]) { let root = &plan.root; diff --git a/crates/integration-tests/tests/per_series_exact_execution.rs b/crates/integration-tests/tests/per_series_exact_execution.rs index 964449b2c..1d1e54e7f 100644 --- a/crates/integration-tests/tests/per_series_exact_execution.rs +++ b/crates/integration-tests/tests/per_series_exact_execution.rs @@ -18,8 +18,7 @@ use asap_types::ir::physical_export::{compile_physical_asap_dag, PhysicalASAPOpe use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::schema::{Schema, PROMQL_SERIES_IDENTITY}; use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, NonASAPOp, OperatorNode, - PhysicalPropertiesAssignment, + apply_physical_properties, NonASAPOp, OperatorNode, PhysicalPropertiesAssignment, }; use futures::{executor::block_on, StreamExt}; @@ -96,13 +95,9 @@ fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap { /// Run `query`'s exact per-entity plan at query time over `samples`, bound /// at its `Scan` of `x`: each answered series' `instance` and value. fn execute(query: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { - let timed = apply_physical_properties( - &per_entity_plan(query), - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let plan = per_entity_plan(query); + let timed = apply_physical_properties(&plan, &PhysicalPropertiesAssignment::default()).unwrap(); + let dag = compile_physical_asap_dag(&plan, &timed).unwrap(); let node = |matches: fn(&PhysicalASAPOperatorPayload) -> bool| { let found: Vec<_> = dag .nodes diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 00fa6b554..4eaab0083 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -61,10 +61,9 @@ fn compile_with( root: &std::rc::Rc, assignment: &asap_types::ir::PhysicalPropertiesAssignment, ) -> Result> { - let root = - asap_types::ir::apply_physical_properties(root, assignment, &mut Default::default())?; + let timed = asap_types::ir::apply_physical_properties(root, assignment)?; Ok(asap_types::ir::physical_export::compile_physical_asap_dag( - &root, + root, &timed, )?) } diff --git a/crates/integration-tests/tests/physical_properties_e2e.rs b/crates/integration-tests/tests/physical_properties_e2e.rs index 0747269c4..0057d8455 100644 --- a/crates/integration-tests/tests/physical_properties_e2e.rs +++ b/crates/integration-tests/tests/physical_properties_e2e.rs @@ -21,8 +21,7 @@ use asap_types::ir::physical_export::{ use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::{ - apply_physical_properties, ASAPOp, NodeToTimedNodeCache, NonASAPOp, OperatorNode, - PhysicalPropertiesAssignment, + apply_physical_properties, ASAPOp, NonASAPOp, OperatorNode, PhysicalPropertiesAssignment, }; use asap_types::physical::{Lifetime, Medium, PhysicalProperties}; @@ -93,9 +92,8 @@ fn timed_dag( assignment: &PhysicalPropertiesAssignment, ) -> PhysicalASAPDAG { let (split, assignment) = split_shared_by_timing(std::slice::from_ref(root), assignment); - let timed = apply_physical_properties(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() + let timed = apply_physical_properties(&split[0], &assignment).unwrap(); + compile_physical_asap_dag(&split[0], &timed).unwrap() } fn states(dag: &PhysicalASAPDAG) -> Vec { diff --git a/crates/integration-tests/tests/precompute_raw_samples.rs b/crates/integration-tests/tests/precompute_raw_samples.rs index 9db3c4947..73e19dae8 100644 --- a/crates/integration-tests/tests/precompute_raw_samples.rs +++ b/crates/integration-tests/tests/precompute_raw_samples.rs @@ -70,7 +70,12 @@ fn candidates(query: &str, accuracy: AccuracyTarget) -> Vec> { explicit_local_candidates(root) .into_iter() .map(|candidate| { - match &with_confirmed_weights(&[QueryRoot::Operator(candidate)], &data)[..] { + match &with_confirmed_weights( + &[QueryRoot::Operator(candidate)], + &data, + &mut Default::default(), + )[..] + { [QueryRoot::Operator(confirmed)] => Rc::clone(confirmed), _ => unreachable!("one operator root"), } diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index ab8c07357..334a9b785 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -640,10 +640,10 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { if matches!(child.asap(), Some(ASAPOp::FinalizeExactAccumulator { .. })))) }) .expect("a KLL over the finalized Sum accumulator"); - // Stored timings are gone: time the plan with its outer summary - // maintained and read the timed copy. - let timed_plan = at_ingestion(&plan); - let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = timed_plan.asap() else { + // Timings are not stored on the plan: time it with its outer summary + // maintained and read each node's timing. + let timed = at_ingestion(&plan); + let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = plan.asap() else { panic!("expected selected quantile summary"); }; let Some(ASAPOp::SummaryAgg { child, .. }) = summary_input.asap() else { @@ -656,7 +656,7 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { ); }; assert_eq!( - child.timing, + timed.get(child).map(|properties| properties.timing()), Some(asap_types::ir::properties::ExecutionTiming::IngestionTime) ); assert!(matches!( diff --git a/crates/integration-tests/tests/reuse_rules.rs b/crates/integration-tests/tests/reuse_rules.rs index c39eda65f..1e3cbc531 100644 --- a/crates/integration-tests/tests/reuse_rules.rs +++ b/crates/integration-tests/tests/reuse_rules.rs @@ -17,11 +17,10 @@ use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::planning::{ cheapest_of, compose_memo_plan, expanded_memo, select_among, }; -use asap_types::ir::physical_export::compile_physical_asap_dag; use asap_types::ir::properties::AccuracyTarget; use asap_types::ir::{ - apply_physical_properties, ASAPOp, NodeToTimedNodeCache, NonASAPOp, OperatorNode, - PhysicalPropertiesAssignment, QueryRoot, + apply_physical_properties, ASAPOp, NonASAPOp, OperatorNode, PhysicalPropertiesAssignment, + QueryRoot, }; use asap_types::physical::Lifetime; use asap_types::stages::{root_groups, MemoGroupId}; @@ -112,17 +111,19 @@ fn samples() -> Vec<(&'static str, &'static str, i64, f64)> { /// `Scan`; each `TimeRange` keeps its own window (a tumbling window's moved /// back by the `TimeShift` below it). fn execute(root: &Rc) -> Vec { - let timed = apply_physical_properties( - root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), + let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default()).unwrap(); + // Tumbling windows run as their tumbling windows, each over its own window. + let expansion = + asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload( + &[QueryRoot::Operator(Rc::clone(root))], + &timed, + ) + .unwrap(); + let dag = asap_types::ir::physical_export::compile_physical_asap_workload( + &expansion.roots, + &expansion.physical_properties, ) .unwrap(); - // Tumbling windows run as their tumbling windows, each over its own window. - let timed = - asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains(&timed) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); let store = physical_common::raw_data_store(&dag, |_, schema| { samples() .into_iter() diff --git a/crates/integration-tests/tests/runtime_handoff.rs b/crates/integration-tests/tests/runtime_handoff.rs index d937dcd6c..d3f1ba3d1 100644 --- a/crates/integration-tests/tests/runtime_handoff.rs +++ b/crates/integration-tests/tests/runtime_handoff.rs @@ -88,7 +88,8 @@ async fn export_carries_the_selected_lifetime() { assert!(output.selection.is_some(), "Stage 3 selected"); assert_eq!( dag, - compile_physical_asap_workload(&output.roots()).expect("the selected plan exports") + compile_physical_asap_workload(&output.roots(), &output.physical_properties) + .expect("the selected plan exports") ); } diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs index 64ffd6604..a9f205622 100644 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ b/crates/integration-tests/tests/tumbling_window_merge.rs @@ -20,16 +20,17 @@ use asap_logical_asap_optimization::test_support::{ window_composition_shared_chain_candidates, }; use asap_types::ir::operator::operator_properties::TimeShift; -use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains; +use asap_types::ir::operator::tumbling_window_chain::{ + expand_tumbling_window_chains, expand_tumbling_window_chains_in_workload, +}; use asap_types::ir::physical_export::PhysicalASAPDAG; -use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::properties::{AccuracyMetric, AccuracyTarget}; use asap_types::ir::schema::Schema; use asap_types::ir::{ - apply_physical_properties, ASAPOp, NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, + apply_physical_properties, ASAPOp, NonASAPOp, Operator, OperatorNode, PhysicalPropertiesAssignment, QueryRoot, }; -use asap_types::physical::{Lifetime, Medium, PhysicalProperties}; +use asap_types::physical::PhysicalProperties; use asap_types::test_support::compile_physical_asap_dag_with_node_ids; const EVALUATION_MS: i64 = 300_000; @@ -178,9 +179,8 @@ struct Exported { } fn export(root: &Rc, assignment: &PhysicalPropertiesAssignment) -> Exported { - let timed = - apply_physical_properties(root, assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); + let timed = apply_physical_properties(root, assignment).unwrap(); + let compiled = compile_physical_asap_dag_with_node_ids(root, &timed).unwrap(); let tumbling_window = |node: &OperatorNode| -> i64 { match node.children()[0].non_asap() { Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms / TUMBLING_WINDOW_WIDTH_MS, @@ -304,20 +304,25 @@ fn execute_maintained_chain( &chain, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = - apply_physical_properties(planned, &assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); - for tumbling_window in OperatorNode::reachable(&expanded) + let timed = apply_physical_properties(planned, &assignment).unwrap(); + let expansion = expand_tumbling_window_chains_in_workload( + &[QueryRoot::Operator(Rc::clone(planned))], + &timed, + ) + .unwrap(); + let [QueryRoot::Operator(expanded)] = &expansion.roots[..] else { + unreachable!("one operator root") + }; + for tumbling_window in OperatorNode::reachable(expanded) .iter() .filter(|n| matches!(n.asap(), Some(ASAPOp::SummaryAgg { .. }))) { assert_eq!( - tumbling_window.lifetime, - Lifetime::AcrossEvaluations(Medium::Memory) + expansion.physical_properties.get(tumbling_window), + Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION) ); - assert_eq!(tumbling_window.timing, Some(ExecutionTiming::IngestionTime)); } - execute_retained_as(&expanded, &PhysicalPropertiesAssignment::default(), samples) + execute_retained_as(expanded, &expansion.physical_properties, samples) } fn execute_retained_as( diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index 30a8c5fe3..cac277c9b 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -666,8 +666,6 @@ fn absorbed_targets(targets: &[ComputationTarget]) -> impl Iterator bool { left.operator.map_children(|_| ()) == right.operator.map_children(|_| ()) && left.schema == right.schema && left.result_kind == right.result_kind - && left.timing == right.timing - && left.lifetime == right.lifetime && left .children() .iter() diff --git a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs index c3fcde7d0..29f7f5b9a 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/rules/shared_configuration_readers.rs @@ -52,9 +52,7 @@ fn with_child_nodes( false => Rc::clone(resized), true => Rc::new( OperatorNode::with_schema(operator, resized.schema.clone()) - .with_guarantee(resized.guarantee.clone()) - .with_timing(resized.timing) - .with_lifetime(resized.lifetime), + .with_guarantee(resized.guarantee.clone()), ), }; done.insert(Rc::as_ptr(resized), Rc::clone(&out)); @@ -132,11 +130,6 @@ impl SharedProducer for SummaryStates { let Some((build, _)) = simple_build(memo, entry) else { continue; }; - // Configuration expansion precedes Stage 2. The rewrite helpers - // create unassigned state; preserve any already assigned build. - if build.timing.is_some() || build.lifetime.is_materialized() { - continue; - } let family = match build.asap() { Some( ASAPOp::SummaryAgg { family, .. } @@ -480,14 +473,7 @@ fn compatible(left: &StateNeed, right: &StateNeed) -> bool { field.dtype = left.family.clone(); } } - serde_json::to_string(&( - operator, - schema, - reader.build.result_kind, - reader.build.timing, - reader.build.lifetime, - )) - .ok() + serde_json::to_string(&(operator, schema, reader.build.result_kind)).ok() }; signature(left).is_some_and(|signature_left| Some(signature_left) == signature(right)) } diff --git a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs b/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs index 5c8b70270..9c018905e 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/state_groups.rs @@ -383,8 +383,6 @@ fn local_fingerprint(node: &OperatorNode) -> Option { &node.schema, node.result_kind, &node.guarantee, - node.timing, - node.lifetime, )) .ok() } @@ -449,8 +447,6 @@ fn same_local_fields(left: &OperatorNode, right: &OperatorNode) -> bool { node.result_kind, node.schema.clone(), node.guarantee.clone(), - node.timing, - node.lifetime, ) }; let left = signature(left); diff --git a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs index 57ae7f531..01cdf1813 100644 --- a/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs +++ b/crates/logical-asap-optimization/src/pass2/memo/window_candidates.rs @@ -109,8 +109,6 @@ fn local_tumbling_window_candidates( original.operator == rewritten.operator && original.schema == rewritten.schema && original.result_kind == rewritten.result_kind - && original.timing == rewritten.timing - && original.lifetime == rewritten.lifetime }); if let Some(original) = matches.next() { if matches.next().is_none() { @@ -156,9 +154,7 @@ fn restore_scan_boundaries( let rebuilt = if changed { Rc::new( OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) diff --git a/crates/logical-asap-optimization/src/stage0/cse.rs b/crates/logical-asap-optimization/src/stage0/cse.rs index f8243c9d2..8f7c639fd 100644 --- a/crates/logical-asap-optimization/src/stage0/cse.rs +++ b/crates/logical-asap-optimization/src/stage0/cse.rs @@ -85,7 +85,7 @@ fn own_fields(node: &OperatorNode) -> Operator<()> { /// hash is SipHash over two parts: /// /// 1. the canonical JSON of [`own_fields`] plus `result_kind`, `schema`, -/// `guarantee`, `timing` and `lifetime` — every field `PartialEq` compares except the +/// `guarantee` — every field `PartialEq` compares except the /// children. A scalar expression is serialized as data with each operator /// node it reads replaced by `()`, so a reference to an /// interned sub-DAG contributes nothing of its own here; @@ -116,8 +116,6 @@ pub(crate) fn structural_hash(node: &OperatorNode, cache: &mut NodeToStructuralH node.result_kind, &node.schema, &node.guarantee, - node.timing, - node.lifetime, ); serde_json::to_string(&own) .unwrap_or_default() @@ -177,8 +175,6 @@ fn same_node( children_equal && left.result_kind == right.result_kind && left.schema == right.schema - && left.timing == right.timing - && left.lifetime == right.lifetime && same_value(&left.guarantee, &right.guarantee) && same_value(&own_fields(left), &own_fields(right)) } @@ -242,8 +238,7 @@ impl InternTable { type NodeToInternedNodeCache = HashMap<*const OperatorNode, (Rc, Rc)>; /// Intern `node`'s children (recursively), then `node` itself. The rebuilt -/// node keeps `node`'s retained schema, result kind, guarantee, timing and -/// materialization: +/// node keeps `node`'s retained schema, result kind and guarantee: /// every child is replaced by an equal node, so each derived property stays /// valid, and the result is `PartialEq`-equal to the input. fn intern_bottom_up( @@ -258,9 +253,7 @@ fn intern_bottom_up( .operator .map_children(|child| intern_bottom_up(table, visited, child)); let rebuilt = OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime); + .with_guarantee(node.guarantee.clone()); let interned = table.intern(rebuilt); visited.insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&interned))); interned diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index 0476b09ea..c3a7d8b09 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -108,9 +108,7 @@ use std::time::Duration; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::properties::timing::{ - apply_physical_properties, NodeToTimedNodeCache, PhysicalPropertiesAssignment, -}; +use asap_types::ir::properties::timing::{apply_physical_properties, PhysicalPropertiesAssignment}; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; @@ -219,25 +217,22 @@ pub fn time_range(range: Duration, child: Rc) -> Rc .unwrap() } -/// `root` timed under the default (every sub-DAG not materialized) -/// assignment — the shape export and the post-ASAP validators consume. -pub fn timed(root: &Rc) -> Rc { - apply_physical_properties( - root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .expect("default physical properties apply") +/// The physical properties of every node of `root` under the default +/// (every sub-DAG not materialized) assignment — what export and the +/// post-ASAP validators read. +pub fn timed(root: &Rc) -> PhysicalPropertiesAssignment { + apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) + .expect("default physical properties apply") } -/// `root` timed with every sub-DAG materialized at ingestion time where the DAG allows it. -pub fn at_ingestion(root: &Rc) -> Rc { +/// The physical properties of every node of `root` with every sub-DAG +/// materialized at ingestion time where the DAG allows it. +pub fn at_ingestion(root: &Rc) -> PhysicalPropertiesAssignment { apply_physical_properties( root, &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(root), ), - &mut NodeToTimedNodeCache::new(), ) .expect("at-ingestion physical properties apply") } diff --git a/crates/physical-asap-optimization/src/catalog/configurations.rs b/crates/physical-asap-optimization/src/catalog/configurations.rs index 450dca8b2..ef47a6475 100644 --- a/crates/physical-asap-optimization/src/catalog/configurations.rs +++ b/crates/physical-asap-optimization/src/catalog/configurations.rs @@ -61,8 +61,6 @@ pub fn may_keep_across_evaluations_per_sole_reader_query( pub enum ConfigurationError { #[error("invalid source candidate inventory: {0}")] InvalidInventory(String), - #[error("candidate {candidate} already has timing or lifetime assigned")] - AssignedExecution { candidate: usize }, } /// Each operator node of `source` with every timing and lifetime @@ -122,11 +120,6 @@ pub fn enumerate_candidate_configurations( let mut source_configurations = vec![Vec::new(); source_count]; for source_id in 0..source_count { let node = &source.operator_nodes_of_candidates_for_memo_groups[source_id].physical_node; - if node.timing.is_some() || node.lifetime.is_materialized() { - return Err(ConfigurationError::AssignedExecution { - candidate: source_id, - }); - } let summary = matches!( node.asap(), Some(ASAPOp::SummaryAgg { .. } | ASAPOp::SummaryTumblingWindowChain { .. }) diff --git a/crates/physical-asap-optimization/src/catalog/factoring.rs b/crates/physical-asap-optimization/src/catalog/factoring.rs index abe4706b1..5c0abe95c 100644 --- a/crates/physical-asap-optimization/src/catalog/factoring.rs +++ b/crates/physical-asap-optimization/src/catalog/factoring.rs @@ -140,13 +140,9 @@ pub fn factor_terminal_readers( &template.schema, template.result_kind, &template.guarantee, - template.timing, - template.lifetime, &input.schema, input.result_kind, &input.guarantee, - input.timing, - input.lifetime, )) .map_err(|error| invalid(error.to_string()))?; if let Some(&previous) = classes.get(&key) { diff --git a/crates/physical-asap-optimization/src/catalog/inventory.rs b/crates/physical-asap-optimization/src/catalog/inventory.rs index 7990284a9..cbb8f74eb 100644 --- a/crates/physical-asap-optimization/src/catalog/inventory.rs +++ b/crates/physical-asap-optimization/src/catalog/inventory.rs @@ -277,13 +277,6 @@ impl ImmediateCandidateBuilder<'_> { if let Some(&existing) = self.private_nodes.get(&Rc::as_ptr(node)) { return Ok(existing); } - if node.timing.is_some() || node.lifetime.is_materialized() { - return Err(CandidateInventoryError::Unsupported { - group: self.owner.group.0 as usize, - candidate: self.owner.index, - reason: "assigned execution timing or lifetime", - }); - } let mut requirements = Vec::new(); for input in super::reconstruction::ordered_inputs(node) { let producers = if let Some(&group) = self.boundaries.get(&Rc::as_ptr(&input)) { diff --git a/crates/physical-asap-optimization/src/catalog/realize.rs b/crates/physical-asap-optimization/src/catalog/realize.rs index eeca63db6..e81b8efdb 100644 --- a/crates/physical-asap-optimization/src/catalog/realize.rs +++ b/crates/physical-asap-optimization/src/catalog/realize.rs @@ -108,6 +108,7 @@ pub fn realize_selected_candidates( ))); } } + let realized_properties = &physical.physical_properties; let operator_node_of_selected_node = rebuilt .operator_node_of_selected_node .into_iter() @@ -117,7 +118,10 @@ pub fn realize_selected_candidates( .get(&Rc::as_ptr(&operator_node)) .cloned() .ok_or_else(|| format!("selected node {node} has no physical identity"))?; - if physical.lifetime != assignments[node].lifetime() { + let realized = realized_properties + .get(&physical) + .ok_or_else(|| format!("selected node {node} has no physical properties"))?; + if realized.lifetime() != assignments[node].lifetime() { return Err(format!( "selected node {node} lifetime changed during physical reconstruction" )); @@ -170,10 +174,15 @@ impl AssignmentRealizer for Stage2AssignmentRealizer<'_> { .zip(&realized.operator_node_of_selected_node) { let timing = physical.candidates[id.0].properties.timing(); - if node.timing != Some(timing) { + let produced = realized + .plan + .physical_properties + .get(node) + .map(|properties| properties.timing()); + if produced != Some(timing) { return Err(PhysicalCandidateError::InvalidOrUnsupported(format!( - "candidate {} requested timing {timing:?}, Stage 2 produced {:?}", - id.0, node.timing + "candidate {} requested timing {timing:?}, Stage 2 produced {produced:?}", + id.0 ))); } } @@ -319,9 +328,10 @@ mod tests { .unwrap(); let through_trait = realizer.realize(&physical, &selected).unwrap(); assert_eq!( - format!("{:?}", direct.plan), - format!("{:?}", through_trait.plan) + format!("{:?}", direct.plan.roots), + format!("{:?}", through_trait.plan.roots) ); + assert_eq!(direct.plan.dag, through_trait.plan.dag); assert_eq!( direct.operator_node_of_selected_node, through_trait.operator_node_of_selected_node diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index c5008008b..76b116c42 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -128,8 +128,8 @@ use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_id use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; use asap_types::ir::schema::{ExactKind, FieldDataType}; use asap_types::ir::{ - apply_physical_properties, is_filtered_build, ASAPOp, NodeToTimedNodeCache, NonASAPOp, - Operator, OperatorNode, PhysicalPropertiesAssignment, QueryRoot, ScalarExpr, + apply_physical_properties, apply_physical_properties_to_roots, is_filtered_build, ASAPOp, + NonASAPOp, Operator, OperatorNode, PhysicalPropertiesAssignment, QueryRoot, ScalarExpr, SchemaDerivationError, SortKey, }; use asap_types::physical::{PhysicalProperties, PhysicalWorkloadPlan}; @@ -542,15 +542,13 @@ fn assigned_candidate( let split_roots = map_roots(implemented, &mut |node| { Ok(replaced[&Rc::as_ptr(node)].clone()) })?; - let mut timing = NodeToTimedNodeCache::new(); - let timed = map_roots(&split_roots, &mut |node| { - Ok(apply_physical_properties(node, &options, &mut timing)?) - })?; - let dag = compile_physical_asap_workload_with_node_ids(&timed)?.dag; + let timed = apply_physical_properties_to_roots(&split_roots, &options)?; + let dag = compile_physical_asap_workload_with_node_ids(&split_roots, &timed)?.dag; Ok(PhysicalWorkloadPlan { id: String::new(), from_logical: from_logical.to_string(), - roots: timed, + roots: split_roots, + physical_properties: timed, dag, }) } @@ -699,7 +697,7 @@ pub(crate) fn find_materialized_output_candidates( PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); } - apply_physical_properties(node, &trial, &mut NodeToTimedNodeCache::new()) + apply_physical_properties(node, &trial) .map(|_| ()) .map_err(|e| IngestionPruneReason::UpstreamAtQueryTime(e.to_string())) }); @@ -710,10 +708,9 @@ pub(crate) fn find_materialized_output_candidates( may_keep_across_evaluations: eligibility.may_keep_across_evaluations, } }); - // A node fixed at ingestion time cannot be built at query time. let shared_results = shared_results .into_iter() - .filter(|n| highest_shared(n) && n.timing != Some(ExecutionTiming::IngestionTime)) + .filter(|n| highest_shared(n)) .map(|node| MaterializedOutputCandidate { readers: readers[&Rc::as_ptr(&node)].clone(), node, @@ -1781,14 +1778,21 @@ mod tests { let [chain] = &node_of(b1, "SummaryTumblingWindowChain")[..] else { panic!("one chain") }; - assert_eq!(chain.lifetime, Lifetime::AcrossEvaluations(Medium::Memory)); - assert_eq!(chain.timing, Some(ExecutionTiming::IngestionTime)); + let properties = |node: &OperatorNode| b1.physical_properties.get(node).unwrap(); + assert_eq!( + properties(chain), + PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION + ); for below in OperatorNode::reachable(chain) { - assert_eq!(below.timing, Some(ExecutionTiming::IngestionTime)); + assert_eq!(properties(&below).timing(), ExecutionTiming::IngestionTime); } for kind in ["SummaryMerge", "SummaryEstimate"] { for node in node_of(b1, kind) { - assert_eq!(node.timing, Some(ExecutionTiming::QueryTime), "{kind}"); + assert_eq!( + properties(&node).timing(), + ExecutionTiming::QueryTime, + "{kind}" + ); } } } @@ -2260,7 +2264,14 @@ mod tests { let [build] = &node_of(&candidates[1], "SummaryAgg")[..] else { panic!("one shared build") }; - assert_eq!(build.lifetime, Lifetime::OneEvaluation(Medium::Memory)); + assert_eq!( + candidates[1] + .physical_properties + .get(build) + .unwrap() + .lifetime(), + Lifetime::OneEvaluation(Medium::Memory) + ); } /// Two queries share one scan: one reads it through tumbling windows maintained at diff --git a/crates/physical-asap-optimization/src/split_by_timing.rs b/crates/physical-asap-optimization/src/split_by_timing.rs index 2907e74de..134945b58 100644 --- a/crates/physical-asap-optimization/src/split_by_timing.rs +++ b/crates/physical-asap-optimization/src/split_by_timing.rs @@ -83,9 +83,7 @@ pub fn split_shared_by_timing( let out = if changed { Rc::new( OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) @@ -121,12 +119,12 @@ mod tests { use asap_types::ir::properties::{ExecutionDataState, ExecutionDataStateError}; use asap_types::ir::scalar::expr_ir::ColumnRef; use asap_types::ir::schema::sketch::{ - ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SketchStatistic, SummaryUpdate, }; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::{ - apply_physical_properties, ASAPOp, NodeToTimedNodeCache, NonASAPOp, Operator, + apply_physical_properties, apply_physical_properties_to_roots, ASAPOp, NonASAPOp, Operator, + QueryRoot, }; use asap_types::physical::{Lifetime, Medium, PhysicalProperties}; @@ -162,10 +160,6 @@ mod tests { ) } - fn exact_sum() -> FieldDataType { - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - } - fn agg(child: Rc, family: FieldDataType) -> Rc { Rc::new( OperatorNode::with_schema( @@ -215,11 +209,6 @@ mod tests { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Concat { children })).unwrap() } - fn finalize(child: Rc) -> Rc { - OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child })) - .unwrap() - } - fn child(node: &Rc) -> Rc { Rc::clone(node.children()[0]) } @@ -242,8 +231,7 @@ mod tests { &root, &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(&root) - ), - &mut NodeToTimedNodeCache::new() + ) ) .err(), Some(ExecutionDataStateError::ConflictingTiming { @@ -257,12 +245,7 @@ mod tests { std::slice::from_ref(&root), ), ); - assert!(apply_physical_properties( - &split[0], - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .is_ok()); + assert!(apply_physical_properties(&split[0], &assignment).is_ok()); } /// Splitting rebuilds the ancestors of a copied sub-DAG; an explicit @@ -281,16 +264,16 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); let (split, assignment) = split_shared_by_timing(std::slice::from_ref(&root), &assignment); - let timed = - apply_physical_properties(&split[0], &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let summary = child(&child(&child(&timed))); + let timed = apply_physical_properties(&split[0], &assignment).unwrap(); + let summary = child(&child(&child(&split[0]))); assert!(matches!( summary.operator, Operator::ASAP(ASAPOp::SummaryAgg { .. }) )); - assert_eq!(summary.lifetime, MATERIALIZED_ACROSS); - assert_eq!(summary.timing, Some(ExecutionTiming::IngestionTime)); + assert_eq!( + timed.get(&summary), + Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION) + ); } /// Two queries share a scan: one reads it through a summary built at @@ -314,80 +297,24 @@ mod tests { &summary, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed_unsplit = { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| apply_physical_properties(root, &assignment, &mut cache)) - .collect::, _>>() + let as_roots = |roots: &[Rc]| -> Vec { + roots.iter().cloned().map(QueryRoot::Operator).collect() }; assert!(matches!( - timed_unsplit, + apply_physical_properties_to_roots(&as_roots(&roots), &assignment), Err(ExecutionDataStateError::ConflictingTiming { .. }) )); let (split, assignment) = split_shared_by_timing(&roots, &assignment); assert_eq!(split.len(), 2); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = split - .iter() - .map(|root| apply_physical_properties(root, &assignment, &mut cache).unwrap()) - .collect(); - let summary = child(&child(&child(&timed[0]))); - assert_eq!(summary.lifetime, MATERIALIZED_ACROSS); - assert_eq!(child(&summary).timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!( - child(&child(&timed[1])).timing, - Some(ExecutionTiming::QueryTime) - ); - assert!(Rc::ptr_eq(timed[0].children()[1], timed[1].children()[1])); - } - - /// Splitting a workload that also shares a scan at two timings keeps a - /// conflicting assignment on a fixed-timing node: the split roots still - /// fail with the conflict instead of dropping the assignment. - #[test] - fn a_fixed_timing_conflict_survives_the_split() { - let shared = scan(); - let summary = agg(Rc::clone(&shared), kll()); - let pair = concat(vec![ - max(estimate(Rc::clone(&summary))), - max(Rc::clone(&shared)), - ]); - let state = agg(Rc::clone(&shared), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - let mut assignment = PhysicalPropertiesAssignment::default(); - assignment.set( - &summary, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - ); - assignment.set( - &state, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - ); - assignment.set( - &read, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - ); - let (split, assignment) = split_shared_by_timing(&[pair, read], &assignment); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Result, _> = split - .iter() - .map(|root| apply_physical_properties(root, &assignment, &mut cache)) - .collect(); + let timed = apply_physical_properties_to_roots(&as_roots(&split), &assignment).unwrap(); + let timing = |node: &OperatorNode| timed.get(node).unwrap().timing(); + let summary = child(&child(&child(&split[0]))); + assert_eq!(timed.get(&summary).unwrap().lifetime(), MATERIALIZED_ACROSS); + assert_eq!(timing(&child(&summary)), ExecutionTiming::IngestionTime); assert_eq!( - timed.err(), - Some( - ExecutionDataStateError::PhysicalPropertiesConflictWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - assigned: - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - } - ) + timing(&child(&child(&split[1]))), + ExecutionTiming::QueryTime ); + assert!(Rc::ptr_eq(split[0].children()[1], split[1].children()[1])); } } diff --git a/crates/plan-selection/src/groups/accuracy_positions.rs b/crates/plan-selection/src/groups/accuracy_positions.rs index 3c1688c1b..6dc5f3cbe 100644 --- a/crates/plan-selection/src/groups/accuracy_positions.rs +++ b/crates/plan-selection/src/groups/accuracy_positions.rs @@ -528,7 +528,7 @@ impl Checker<'_> { } /// `node` rebuilt over each combination of `per_input`'s outputs, with its -/// timing and lifetime and its guarantee composed again; outputs the +/// guarantee composed again; outputs the /// same for their readers kept once. fn rebuilt(node: &Rc, per_input: &[Vec>], keys: &mut Keys) -> Built { let combinations = combinations(per_input); @@ -539,12 +539,7 @@ fn rebuilt(node: &Rc, per_input: &[Vec>], keys: & .with_new_children(|_| next.next().expect("one output per input")) .map_err(|_| UnusableReason::NotChecked)?; let guarantee = composed_guarantee(&rebuilt); - outputs.push(Rc::new( - rebuilt - .with_guarantee(guarantee) - .with_timing(node.timing) - .with_lifetime(node.lifetime), - )); + outputs.push(Rc::new(rebuilt.with_guarantee(guarantee))); } Ok(distinct_for_readers(outputs, keys)) } diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 394fc8946..a4a9b4148 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -852,21 +852,22 @@ mod tests { continue; } // The proof the executor reads. - let proofs: Vec = with_confirmed_weights(&candidate.roots, &gauge) - .iter() - .flat_map(QueryRoot::operator_refs) - .flat_map(OperatorNode::reachable) - .filter_map(|n| match n.asap() { - Some(ASAPOp::SummaryAgg { - input, - family: FieldDataType::Sketch(kind, _), - .. - }) if *kind.algorithm() == SketchAlgorithm::CmsWithHeap => { - Some(input.weight_domain.clone()) - } - _ => None, - }) - .collect(); + let proofs: Vec = + with_confirmed_weights(&candidate.roots, &gauge, &mut Default::default()) + .iter() + .flat_map(QueryRoot::operator_refs) + .flat_map(OperatorNode::reachable) + .filter_map(|n| match n.asap() { + Some(ASAPOp::SummaryAgg { + input, + family: FieldDataType::Sketch(kind, _), + .. + }) if *kind.algorithm() == SketchAlgorithm::CmsWithHeap => { + Some(input.weight_domain.clone()) + } + _ => None, + }) + .collect(); assert!(!proofs.is_empty(), "{}", candidate.id); assert!( proofs.iter().all(|domain| *domain @@ -1040,7 +1041,7 @@ mod tests { [WeightDomain::RawDataSamplesOfMetric { metric: "m".into() }] ); let counter = with_metric_types(data(), &[("m", PrometheusMetricType::Counter)]); - let confirmed = with_confirmed_weights(&p2.roots, &counter); + let confirmed = with_confirmed_weights(&p2.roots, &counter, &mut Default::default()); assert_eq!( domains(&confirmed), [WeightDomain::NonNegative { @@ -1050,13 +1051,21 @@ mod tests { let mut gauge = with_metric_types(data(), &[("m", PrometheusMetricType::Gauge)]); gauge.sources.get_mut("m").unwrap().values_non_negative = true; assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &gauge)), + domains(&with_confirmed_weights( + &p2.roots, + &gauge, + &mut Default::default() + )), [WeightDomain::NonNegative { proof: NonNegativeWeightProof::DeclaredNonNegativeRawDataSamples }] ); assert_eq!( - domains(&with_confirmed_weights(&p2.roots, &data())), + domains(&with_confirmed_weights( + &p2.roots, + &data(), + &mut Default::default() + )), [WeightDomain::UnknownOrSigned] ); // Nothing open: the same roots. @@ -1064,7 +1073,9 @@ mod tests { let QueryRoot::Operator(before) = &exact.roots[0] else { panic!() }; - let QueryRoot::Operator(after) = &with_confirmed_weights(&exact.roots, &counter)[0] else { + let QueryRoot::Operator(after) = + &with_confirmed_weights(&exact.roots, &counter, &mut Default::default())[0] + else { panic!() }; assert!(Rc::ptr_eq(before, after)); @@ -3429,9 +3440,7 @@ mod tests { /// `quantile_over_time(0.99, m[1m])` with its KLL candidate chosen, its /// state materialized with `materialized` (`None`: not materialized). fn kll_dag(materialized: Option) -> PhysicalASAPDAG { - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, PhysicalPropertiesAssignment, - }; + use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment}; let root = lower_promql( "quantile_over_time(0.99, m[1m])", AccuracyTarget::max_error(0.01).failure_probability(0.01), @@ -3472,11 +3481,11 @@ mod tests { if let Some(materialized) = materialized { assignment.set(&state, materialized); } - let timed = - apply_physical_properties(root, &assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids(&[ - QueryRoot::Operator(timed), - ]) + let timed = apply_physical_properties(root, &assignment).unwrap(); + asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids( + &[QueryRoot::Operator(Rc::clone(root))], + &timed, + ) .unwrap() .dag } diff --git a/crates/plan-selection/src/report.rs b/crates/plan-selection/src/report.rs index e2617b580..4c26fc1f1 100644 --- a/crates/plan-selection/src/report.rs +++ b/crates/plan-selection/src/report.rs @@ -8,7 +8,7 @@ use asap_cost::{CostModel, PlanCost}; use asap_types::ir::physical_export::PhysicalASAPNodeId; use asap_types::ir::properties::AccuracyMetric; #[cfg(any(test, feature = "reference-pricer"))] -use asap_types::ir::QueryRoot; +use asap_types::ir::{PhysicalPropertiesAssignment, QueryRoot}; use asap_types::stages::{ DagOfSelectedCandidates, PhysicalCandidateId, PrunedCandidate, SelectedNodeId, }; @@ -198,6 +198,8 @@ pub struct WholePlanSelection { /// Stage 3 confirmed made a proof: the plan to run. The Stage 2 /// candidate keeps the open requirement. pub plan: Vec, + /// The physical properties of `plan`'s nodes. + pub physical_properties: PhysicalPropertiesAssignment, pub costs: BTreeMap, /// For each valid candidate, per query, the accuracy Stage 3 accepted. pub accepted: BTreeMap>, diff --git a/crates/plan-selection/src/selector.rs b/crates/plan-selection/src/selector.rs index 27a818d8f..c91c2917d 100644 --- a/crates/plan-selection/src/selector.rs +++ b/crates/plan-selection/src/selector.rs @@ -306,7 +306,8 @@ impl<'a> PlanSelector<'a> { } } let built = self.kept.swap_remove(best); - let plan = with_confirmed_weights(&built.roots, self.data); + let mut physical_properties = built.physical_properties.clone(); + let plan = with_confirmed_weights(&built.roots, self.data, &mut physical_properties); let tumbling_window_retention_ms = tumbling_window_retention_ms( &built.dag, &query_reach(&built.dag, &node_map(&built.dag)), @@ -315,6 +316,7 @@ impl<'a> PlanSelector<'a> { let selection = WholePlanSelection { selected, plan, + physical_properties, costs: self.costs, accepted: self.accepted, rejected: self.rejected, diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index 519e9066f..c9d4f2491 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -252,6 +252,7 @@ pub async fn plan_installable( let output = e2e_plan(input).await?; installable_plan_of( &output.roots(), + &output.physical_properties, output.selection.as_ref(), &output.tumbling_window_retention_ms, &entries, @@ -260,7 +261,9 @@ pub async fn plan_installable( } /// The installable plan of `roots`, the selected roots of a workload whose -/// entries are `entries` (in `QueryWorkload::entries()` order), with +/// entries are `entries` (in `QueryWorkload::entries()` order), each node +/// with the physical properties `physical_properties` lists for it (Stage +/// 3's selection; a node it does not list takes the timing pass's default), with /// Stage 3's report of its `selection` of them when there is one, and the /// tumbling-window retention of the selected plan's kept chains /// (`RealizedPlan::tumbling_window_retention_ms`, by node id in the DAG `roots` @@ -269,6 +272,7 @@ pub async fn plan_installable( /// what a runtime would install. pub fn installable_plan_of( roots: &[QueryRoot], + physical_properties: &PhysicalPropertiesAssignment, selection: Option<&Selection>, tumbling_window_retention_ms: &BTreeMap, entries: &[QueryWorkloadEntry], @@ -280,10 +284,10 @@ pub fn installable_plan_of( .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) .collect(); let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - // Every node keeps the timing Stage 3 gave it. - let timed = - apply_physical_properties_to_roots(roots, &PhysicalPropertiesAssignment::default())?; + // Every node keeps the physical properties Stage 3 gave it. + let timed = apply_physical_properties_to_roots(roots, physical_properties)?; let expanded = tumbling_window_retention_of_expanded_tumbling_windows( + roots, &timed, tumbling_window_retention_ms, )?; @@ -640,22 +644,24 @@ fn executor_refreshes( struct ExpandedWithRetention { dag: PhysicalASAPDAG, retention_ms_by_node: HashMap, - /// Each node id of the selected candidate's physical DAG (`timed` + /// Each node id of the selected candidate's physical DAG (its roots /// exported, Stage 3's ids) with the ids in `dag` that run it /// ([`PredictedNodeUsage::installed_node_ids`]). installed_node_ids: HashMap>, } -/// Expand the tumbling-window chains of `timed` and export it; each tumbling window takes the +/// Expand the tumbling-window chains of `roots`, whose nodes' physical +/// properties `timed` lists, and export it; each tumbling window takes the /// retention Stage 2 gives its chain (`RealizedPlan::tumbling_window_retention_ms`, by -/// node id in the selected candidate's physical DAG, which is `timed` +/// node id in the selected candidate's physical DAG, which is `roots` /// exported). fn tumbling_window_retention_of_expanded_tumbling_windows( - timed: &[QueryRoot], + roots: &[QueryRoot], + timed: &PhysicalPropertiesAssignment, tumbling_window_retention_ms: &BTreeMap, ) -> Result { use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let selected = compile_physical_asap_workload_with_node_ids(timed)?; + let selected = compile_physical_asap_workload_with_node_ids(roots, timed)?; // Reject unsupported raw-source shapes before expanding possibly thousands // of tumbling windows. The complete expanded-plan validation remains authoritative. selected.dag.check_raw_data_windows()?; @@ -668,9 +674,12 @@ fn tumbling_window_retention_of_expanded_tumbling_windows( .ok_or(InstallablePlanError::TumblingWindowRetentionOfUnknownChain { node_id })?; retention_by_chain.push((Rc::clone(chain), retention_ms)); } - let expansion = expand_tumbling_window_chains_in_workload(timed) + let expansion = expand_tumbling_window_chains_in_workload(roots, timed) .map_err(InstallablePlanError::TumblingWindowChainExpansion)?; - let compiled = compile_physical_asap_workload_with_node_ids(&expansion.roots)?; + let compiled = compile_physical_asap_workload_with_node_ids( + &expansion.roots, + &expansion.physical_properties, + )?; // Index physical identity once. Looking up every expanded tumbling window through // the identity map's linear search makes this phase quadratic in tumbling windows. let mut installed_ids_by_pointer = HashMap::with_capacity(compiled.dag.nodes.len()); @@ -869,9 +878,7 @@ mod tests { use asap_types::ir::operator::Source; use asap_types::ir::physical_export::compile_physical_asap_dag; use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; + use asap_types::ir::{apply_physical_properties, OperatorNode, TimeRangeKind}; use std::time::Duration; let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -903,13 +910,9 @@ mod tests { })) .unwrap(); let export = |root| { - let timed = apply_physical_properties( - &root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - compile_physical_asap_dag(&timed).unwrap() + let timed = + apply_physical_properties(&root, &PhysicalPropertiesAssignment::default()).unwrap(); + compile_physical_asap_dag(&root, &timed).unwrap() }; let install = |dag| { InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) @@ -947,9 +950,7 @@ mod tests { use asap_types::ir::schema::{ DataType, ExactKind, ExactParams, Field, FieldDataType, Schema, SummaryUpdate, }; - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, OperatorNode, TimeRangeKind, - }; + use asap_types::ir::{apply_physical_properties, OperatorNode, TimeRangeKind}; use std::time::Duration; let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::TimeSeries { @@ -1002,10 +1003,8 @@ mod tests { &merged, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = - apply_physical_properties(&merged, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = apply_physical_properties(&merged, &assignment).unwrap(); + let dag = compile_physical_asap_dag(&merged, &timed).unwrap(); match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { Err(InstallablePlanError::Uninstallable( InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { @@ -1043,10 +1042,8 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); } - let timed = - apply_physical_properties(&merge, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = apply_physical_properties(&merge, &assignment).unwrap(); + let dag = compile_physical_asap_dag(&merge, &timed).unwrap(); match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { Err(InstallablePlanError::Uninstallable( InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 91fcce57b..0923df532 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -222,6 +222,12 @@ pub struct PlanOutput { /// tumbling windows are kept ([`RealizedPlan::tumbling_window_retention_ms`](asap_types::stages::RealizedPlan::tumbling_window_retention_ms)); /// empty for a pass that does not set it. pub tumbling_window_retention_ms: BTreeMap, + /// The physical properties the pass selected for the plans' nodes, by + /// node: for [`Stage1ToStage3Pass`] every node, as Stage 3 selected it. + /// A node it does not list takes the timing pass's default (not + /// materialized, computed when its consumer runs); empty for a pass that + /// does not set it. + pub physical_properties: PhysicalPropertiesAssignment, } impl PlanOutput { @@ -232,6 +238,7 @@ impl PlanOutput { scalar_plans: Vec::new(), selection: None, tumbling_window_retention_ms: BTreeMap::new(), + physical_properties: PhysicalPropertiesAssignment::default(), } } @@ -299,12 +306,12 @@ impl PlanOutput { } /// The workload as one physical ASAP DAG, with shared sub-DAGs exported - /// once. A node keeps the timing and lifetime its pass gave it: - /// for [`Stage1ToStage3Pass`] that is Stage 3's selected plan as - /// selected (e.g. ingestion-time tumbling windows), the document a deployment - /// runtime executes. Only a node the pass left untimed takes the default - /// [`PhysicalPropertiesAssignment`] (not materialized, computed when its - /// consumer runs). Operator queries become its `roots` and standalone scalar + /// once. A node keeps the physical properties its pass gave it + /// (`physical_properties`): for [`Stage1ToStage3Pass`] that is Stage 3's + /// selected plan as selected (e.g. ingestion-time tumbling windows), the + /// document a deployment runtime executes. Only a node the pass did not + /// list takes the timing pass's default (not materialized, computed when + /// its consumer runs). Operator queries become its `roots` and standalone scalar /// queries its `scalar_roots`, keyed by position in the batch. That /// position is the entry index, because `check_contract` requires dense /// entry indices. @@ -317,11 +324,9 @@ impl PlanOutput { &self, ) -> Result { // A node several roots share by `Rc` is timed and exported once. - let timed = apply_physical_properties_to_roots( - &self.roots(), - &PhysicalPropertiesAssignment::default(), - )?; - compile_physical_asap_workload(&timed) + let roots = self.roots(); + let timed = apply_physical_properties_to_roots(&roots, &self.physical_properties)?; + compile_physical_asap_workload(&roots, &timed) } pub fn len(&self) -> usize { diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 5069ca1de..6291706d5 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -108,7 +108,8 @@ pub struct PlanningRun { /// The selected plan as Stage 2 built it, with its tumbling-window retention. pub selected_physical_plan: Option, /// The plan to run: the selected plan's roots with the weight proofs - /// Stage 3 made applied. + /// Stage 3 made applied. Its nodes' physical properties are in + /// `selected_physical_plan`'s `plan.physical_properties`. pub selected_plan: Option>, pub report: PlanningReport, } @@ -220,7 +221,7 @@ pub fn run_planning( let selection = selection.map_err(SelectionError::Selection)?; report.search_report = Some(selection.search_report.clone()); // The selected plan, as Stage 2 builds it. - let realized = realizer + let mut realized = realizer .realize(physical, &selection.selected_dag) .map_err(SelectionError::Realization)?; let proven: HashSet<*const OperatorNode> = selection @@ -233,6 +234,7 @@ pub fn run_planning( &realized.plan.roots, data, &|node| proven.contains(&Rc::as_ptr(node)), + &mut realized.plan.physical_properties, )); selected_physical_plan = Some(realized); Ok(selection) @@ -402,10 +404,10 @@ impl OptimizationPass for Stage1ToStage3Pass { } } } - output.tumbling_window_retention_ms = run - .selected_physical_plan - .map(|realized| realized.tumbling_window_retention_ms) - .unwrap_or_default(); + if let Some(realized) = run.selected_physical_plan { + output.tumbling_window_retention_ms = realized.tumbling_window_retention_ms; + output.physical_properties = realized.plan.physical_properties; + } output.selection = Some(selection); Ok(output) } diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index e14defe00..e0903ee9f 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -5,7 +5,7 @@ //! `cargo check -p asap-planner --tests --locked` on its own for that). use std::cell::Cell; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::sync::Arc; use crate::time_series; @@ -31,11 +31,10 @@ use asap_types::ir::{ apply_physical_properties_to_roots, ASAPOp, OperatorNode, PhysicalPropertiesAssignment, QueryRoot, }; -use asap_types::physical::{Lifetime, PhysicalProperties}; +use asap_types::physical::PhysicalProperties; use asap_types::stages::HowToComputeAggregateMeasure; use asap_types::stages::PruneReason; use futures::{executor::block_on, StreamExt}; -use std::collections::HashMap; use std::ops::Bound; use std::rc::Rc; use time_series::{series_labels, time_series_tables, RawDataSample}; @@ -1526,21 +1525,47 @@ impl OptimizationPass for StoredAsForced { fn optimize(&self, input: OptimizationInput<'_>) -> Result { let mut output = Stage1ToStage3Pass.optimize(input)?; - // Each node untimed once, so shared inputs stay shared, and its - // materialization in `assignment`. - let mut untimed: HashMap<*const OperatorNode, Rc> = HashMap::new(); - let mut assignment = PhysicalPropertiesAssignment::default(); + // Each node's materialized output: `store`'s, or the one Stage 3 + // selected. let roots: Vec> = output .plans .iter() - .map(|plan| untime(&plan.root, self.store, &mut untimed, &mut assignment)) + .map(|plan| Rc::clone(&plan.root)) .collect(); + let mut assignment = PhysicalPropertiesAssignment::default(); + let mut seen = HashSet::new(); + for root in &roots { + for node in OperatorNode::reachable(root) { + if !seen.insert(Rc::as_ptr(&node)) { + continue; + } + // A shared non-summary node Stage 2 built once per + // evaluation is not kept: a build forced to ingestion time + // above it cannot read a query-time result. + let selected = output + .physical_properties + .get(&node) + .filter(|properties| properties.lifetime().is_materialized()); + let own = match node.asap() { + Some( + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryTumblingWindowChain { .. } + | ASAPOp::SummaryMerge { .. }, + ) => selected, + _ => None, + }; + if let Some(kept) = (self.store)(&node).or(own) { + assignment.set(&node, kept); + } + } + } // A node read both at ingestion and at query time is copied, one // copy per timing, as Stage 2 does. let (roots, assignment) = split_shared_by_timing(&roots, &assignment); let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); - let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - for (plan, root) in output.plans.iter_mut().zip(timed) { + output.physical_properties = + apply_physical_properties_to_roots(&roots, &assignment).unwrap(); + for (plan, root) in output.plans.iter_mut().zip(roots) { let QueryRoot::Operator(root) = root else { unreachable!("operator roots in, operator roots out") }; @@ -1550,42 +1575,6 @@ impl OptimizationPass for StoredAsForced { } } -/// `node` untimed, with `store`'s materialized output (or its own) recorded in -/// `assignment` for every node of its sub-DAG. -fn untime( - node: &Rc, - store: fn(&OperatorNode) -> Option, - untimed: &mut HashMap<*const OperatorNode, Rc>, - assignment: &mut PhysicalPropertiesAssignment, -) -> Rc { - if let Some(done) = untimed.get(&Rc::as_ptr(node)) { - return Rc::clone(done); - } - let mut copy = (**node).clone(); - copy.operator = node - .operator - .map_children(|child| untime(child, store, untimed, assignment)); - copy.timing = None; - copy.lifetime = Lifetime::None; - let copy = Rc::new(copy); - // A shared non-summary node Stage 2 built once per evaluation is not - // kept: a build forced to ingestion time above it cannot read a - // query-time result. - let own = match node.asap() { - Some( - ASAPOp::SummaryAgg { .. } - | ASAPOp::SummaryTumblingWindowChain { .. } - | ASAPOp::SummaryMerge { .. }, - ) if node.lifetime.is_materialized() => node.physical_properties().map(Result::unwrap), - _ => None, - }; - if let Some(materialized) = store(node).or(own) { - assignment.set(©, materialized); - } - untimed.insert(Rc::as_ptr(node), Rc::clone(©)); - copy -} - /// Every `SummaryAgg` built at ingestion time. fn summaries_at_ingestion(node: &OperatorNode) -> Option { matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. })) diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index 9926e0632..f5c940069 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -15,10 +15,8 @@ use serde::{Deserialize, Serialize}; use crate::ir::operator::node::{Operator, OperatorNode, OperatorResultKind}; use crate::ir::operator::query::QueryRoot; -use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::schema::Schema; -use crate::physical::Lifetime; /// Index of a node in [`FlatDag::nodes`]. pub type NodeId = usize; @@ -30,8 +28,6 @@ pub struct FlatNode { pub(crate) result_kind: OperatorResultKind, pub(crate) schema: Schema, pub(crate) guarantee: Option, - pub(crate) timing: Option, - pub(crate) lifetime: Lifetime, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -78,8 +74,6 @@ fn visit( result_kind: node.result_kind, schema: node.schema.clone(), guarantee: node.guarantee.clone(), - timing: node.timing, - lifetime: node.lifetime, }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id); @@ -165,7 +159,6 @@ mod tests { #[test] fn json_round_trips() { let (dag, _) = flatten(&[QueryRoot::Operator(filter(values()))]); - assert!(dag.nodes.iter().all(|n| n.timing.is_none())); let json = serde_json::to_string(&dag).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), dag); } diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 99b12a821..d0d5f9dde 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -16,7 +16,7 @@ pub mod materialized_view; pub mod physical_export; pub use properties::timing::{ apply_physical_properties, apply_physical_properties_to_roots, data_state, is_filtered_build, - timing_under_consumer, NodeToTimedNodeCache, PhysicalPropertiesAssignment, + timing_under_consumer, PhysicalPropertiesAssignment, }; #[cfg(any(test, feature = "test-support"))] pub use properties::timing::{planned_data_state, validate_at_ingestion}; diff --git a/crates/types/src/ir/operator/node.rs b/crates/types/src/ir/operator/node.rs index 713a7fcba..009a69449 100644 --- a/crates/types/src/ir/operator/node.rs +++ b/crates/types/src/ir/operator/node.rs @@ -1,7 +1,9 @@ //! The unified operator node: one node type before and after ASAP //! optimization. A node is an operator plus the common planning properties -//! every traversal needs (output category and schema, accuracy guarantee, -//! execution timing). +//! every traversal needs (output category and schema, accuracy guarantee). +//! Physical properties (timing, lifetime) are not on the node: physical +//! stages hold them per node in a +//! [`crate::ir::PhysicalPropertiesAssignment`]. use std::collections::HashSet; use std::rc::Rc; @@ -10,12 +12,10 @@ use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; use super::non_asap::NonASAPOp; -use crate::ir::properties::execution_data_state::ExecutionTiming; use crate::ir::properties::guarantee::AccuracyGuarantee; use crate::ir::properties::summary_coverage::{NodeToSummaryCoverageCache, SummaryCoverage}; use crate::ir::schema::Schema; use crate::ir::SchemaDerivationError; -use crate::physical::{InvalidPhysicalProperties, Lifetime, PhysicalProperties}; /// The output category of an operator, derived from the operation and its /// inputs. Matching column schemas do not make categories interchangeable. @@ -91,30 +91,16 @@ impl Operator { /// /// `schema` and `result_kind` are derived from `operator` and its children /// at construction and retained. `guarantee` is `None` until accuracy -/// assessment establishes one (`None` never means exact). `timing` is `None` -/// until a physical properties assignment is applied; export rejects an executable -/// node without one. `lifetime` is how long that assignment materializes this -/// node's output (`None` until then, and for every node whose output is not -/// materialized); with `timing` it is the node's [`crate::physical::PhysicalProperties`]. +/// assessment establishes one (`None` never means exact). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OperatorNode { pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, pub guarantee: Option, - pub timing: Option, - pub lifetime: Lifetime, } impl OperatorNode { - /// Its timing and lifetime, once [`crate::ir::apply_physical_properties`] - /// has timed it (`None` before), or why they cannot go together. - pub fn physical_properties( - &self, - ) -> Option> { - Some(PhysicalProperties::new(self.timing?, self.lifetime)) - } - /// Build a node, deriving its schema and output category. Fails when the /// schema cannot be derived (a column reference out of range, a reserved /// ASAP operator, ...). @@ -135,8 +121,6 @@ impl OperatorNode { result_kind, schema, guarantee: None, - timing: None, - lifetime: Lifetime::None, } } @@ -152,16 +136,6 @@ impl OperatorNode { self } - pub fn with_timing(mut self, timing: Option) -> Self { - self.timing = timing; - self - } - - pub fn with_lifetime(mut self, lifetime: Lifetime) -> Self { - self.lifetime = lifetime; - self - } - /// What this summary state covers; `None` for a node that is not a /// `SummaryAgg` or a valid `SummaryMerge`. Derived on each call, since /// the public fields can be edited in place. @@ -245,8 +219,7 @@ impl OperatorNode { /// the old derived schema are retained, for either operator category. A /// change in output arity with such overrides needs an explicit new /// naming assignment. - /// `guarantee`, `timing` and `lifetime` depend on the inputs and - /// are cleared. + /// `guarantee` depends on the inputs and is cleared. pub fn with_new_children( &self, f: impl FnMut(&Rc) -> Rc, @@ -310,35 +283,11 @@ impl OperatorNode { out } - /// Validate assigned phases without imposing any particular runtime implementation. - #[cfg(any(test, feature = "test-support"))] - pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError> { - self.validate_structure()?; - for node in Self::reachable(self) { - let timing = node.timing.ok_or_else(|| { - SchemaDerivationError::InvalidScalarSignature( - "execution timing is unassigned".into(), - ) - })?; - if timing == crate::ir::properties::ExecutionTiming::IngestionTime - && node.children().iter().any(|child| { - child.timing != Some(crate::ir::properties::ExecutionTiming::IngestionTime) - }) - { - return Err(SchemaDerivationError::InvalidScalarSignature( - "ingestion-time operation depends on a query-time or unassigned input".into(), - )); - } - } - Ok(()) - } - /// Validate the whole DAG reachable from this node: every operator's /// input contract, scalar typing against the owning operator's input /// schema, and agreement between each retained schema and the one /// derived from the operator. Both operator categories may override field /// names and qualifiers; all structural metadata must match derivation. - /// `timing` may be `None`. pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError> { // Each summary state's coverage is derived once, not once per merge // above it; `reachable` keeps every node alive while it is used. diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 8a97e40cc..8a724e2d1 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -20,8 +20,10 @@ use super::node::{Operator, OperatorNode}; use super::non_asap::{NonASAPOp, TimeRangeKind}; use super::operator_properties::TimeShift; use super::query::QueryRoot; +use crate::ir::properties::timing::PhysicalPropertiesAssignment; use crate::ir::properties::ExecutionTiming; use crate::ir::SchemaDerivationError; +use crate::physical::PhysicalProperties; fn invalid(message: &str) -> SchemaDerivationError { SchemaDerivationError::InvalidScalarSignature(message.into()) @@ -228,12 +230,9 @@ fn build_over( /// reads each tumbling window of each chain it read, newest first, as plain /// `SummaryAgg` nodes the executor runs; a merge of the newest `n` tumbling windows /// reads the first `n`, the same nodes a merge of the whole chain reads. Every other node keeps its -/// guarantee, timing and lifetime. Each tumbling window takes its chain's -/// timing and lifetime, and the range and shift it reads take the -/// chain's timing: a chain materialized at ingestion time expands into -/// tumbling windows each built and stored at ingestion time. Expanding before timings -/// are applied leaves the tumbling windows untimed. A chain read by anything but a -/// `SummaryMerge` is an error. +/// guarantee. A chain read by anything but a `SummaryMerge` is an error. +/// [`expand_tumbling_window_chains_in_workload`] also gives the expanded +/// nodes their physical properties. #[cfg(any(test, feature = "test-support"))] pub fn expand_tumbling_window_chains( root: &Rc, @@ -255,6 +254,13 @@ pub struct TumblingWindowChainExpansion { /// so a caller can carry what it knows about a node to the expanded /// workload. pub expanded_nodes: Vec<(Rc, Rc)>, + /// The physical properties of the expanded workload: each node the + /// input listed keeps its properties, a node replacing one takes its + /// properties, each tumbling window takes its chain's, and the range + /// and shift it reads take the chain's timing, not materialized. A chain + /// materialized at ingestion time expands into tumbling windows each + /// built and stored at ingestion time. + pub physical_properties: PhysicalPropertiesAssignment, } /// One expanded chain: the `SummaryTumblingWindowChain` node of the input workload and the @@ -273,9 +279,11 @@ pub struct ExpandedTumblingWindowChain { /// read expands into one set of tumbling windows. A standalone scalar root has the /// operators it reads expanded. Also returns each chain's tumbling windows, so a /// caller can carry what it knows about a chain (e.g. its retention) to -/// them. +/// them, and the expanded workload's physical properties from `timed`, the +/// input's (empty for an input not yet timed). pub fn expand_tumbling_window_chains_in_workload( roots: &[QueryRoot], + timed: &PhysicalPropertiesAssignment, ) -> Result { let mut cache = NodeToExpandedNodeCache::default(); let roots = roots @@ -321,28 +329,48 @@ pub fn expand_tumbling_window_chains_in_workload( tumbling_windows, } }) - .collect(); + .collect::>(); + let mut physical_properties = timed.clone(); + for (original, replacement) in &cache.expanded { + if let Some(properties) = timed.get(original) { + physical_properties.set(replacement, properties); + } + } + for expanded in &chains { + let Some(properties) = timed.get(&expanded.chain) else { + continue; + }; + for tumbling_window in &expanded.tumbling_windows { + physical_properties.set(tumbling_window, properties); + not_materialized_below( + tumbling_window, + properties.timing(), + &mut physical_properties, + ); + } + } Ok(TumblingWindowChainExpansion { roots, chains, expanded_nodes: cache.expanded, + physical_properties, }) } -/// The untimed `node` with `timing` written into it and into every untimed -/// node below it: the nodes [`tumbling_window`] created, over the chain's input, which -/// the timing pass already timed. -fn timed_like(node: &Rc, timing: ExecutionTiming) -> OperatorNode { - OperatorNode::with_schema( - node.operator.map_children(|child| match child.timing { - Some(_) => Rc::clone(child), - None => Rc::new(timed_like(child, timing)), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()) - .with_lifetime(node.lifetime) - .with_timing(Some(timing)) +/// List every node below `node` that `properties` does not list yet at +/// `timing`, not materialized: the nodes [`tumbling_window`] created, over +/// the chain's input, which the timing pass already timed. +fn not_materialized_below( + node: &OperatorNode, + timing: ExecutionTiming, + properties: &mut PhysicalPropertiesAssignment, +) { + for child in node.children() { + if properties.get(child).is_none() { + properties.set(child, PhysicalProperties::not_materialized_at(timing)); + not_materialized_below(child, timing, properties); + } + } } /// What one expansion has built: each expanded node by its input's @@ -390,13 +418,6 @@ fn expand( continue; } let tumbling_window = Rc::new(tumbling_window(child, index)?); - let tumbling_window = match child.timing { - Some(timing) => Rc::new( - timed_like(&tumbling_window, timing) - .with_lifetime(child.lifetime), - ), - None => tumbling_window, - }; if index == 0 { cache.chains.push(Rc::clone(child)); } @@ -416,9 +437,7 @@ fn expand( }))?; Rc::new( OperatorNode::with_schema(merge.operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime), + .with_guarantee(node.guarantee.clone()), ) } _ => { @@ -436,9 +455,7 @@ fn expand( .map_children(|child| Rc::clone(&children[&Rc::as_ptr(child)])), node.schema.clone(), ) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime), + .with_guarantee(node.guarantee.clone()), ) } else { Rc::clone(node) diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 5d0d7acd5..77b51e38f 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -25,10 +25,10 @@ use crate::ir::properties::execution_data_state::{ }; use crate::ir::properties::guarantee::AccuracyGuarantee; #[cfg(any(test, feature = "test-support"))] +use crate::ir::properties::timing::apply_physical_properties_to_roots; use crate::ir::properties::timing::{ - apply_physical_properties_to_roots, PhysicalPropertiesAssignment, + data_state, is_materializable_state_output, PhysicalPropertiesAssignment, }; -use crate::ir::properties::timing::{data_state, is_materializable_state_output}; use crate::ir::schema::{FieldDataType, Schema}; use crate::physical::{InvalidPhysicalProperties, Lifetime, PhysicalProperties}; @@ -376,14 +376,14 @@ impl PhysicalASAPDAG { /// installed plan may not keep. #[cfg(any(test, feature = "test-support"))] pub fn with_every_node_at_query_time(&self) -> Result { - let roots = self.untimed_query_roots()?; + let roots = self.query_roots()?; let timed = apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default())?; - Ok(compile_physical_asap_workload(&timed)?) + Ok(compile_physical_asap_workload(&roots, &timed)?) } - /// Every node of this DAG as IR, by node id, untimed and - /// not materialized with its exported guarantee: what a caller derives + /// Every node of this DAG as IR, by node id, with its exported + /// guarantee (an operator node has no physical properties): what a caller derives /// IR properties of a plan file's node from, such as its /// [`SummaryCoverage`](crate::ir::properties::summary_coverage::SummaryCoverage). /// Shared inputs stay shared. Validates the DAG first. @@ -581,11 +581,11 @@ impl PhysicalASAPDAG { .collect() } - /// The IR this DAG exports, one root per query in batch order, every - /// node untimed and not materialized: the inverse of - /// [`compile_physical_asap_workload`] up to `Rc` identity and timing. + /// The IR this DAG exports, one root per query in batch order: the + /// inverse of [`compile_physical_asap_workload`] up to `Rc` identity and + /// the nodes' physical properties. #[cfg(any(test, feature = "test-support"))] - fn untimed_query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { + fn query_roots(&self) -> Result, PhysicalASAPDAGValidationError> { let rebuilt = self.operator_nodes()?; let mut operator_roots = self.roots.iter(); let mut scalar_roots = self.scalar_roots.iter().peekable(); @@ -850,31 +850,37 @@ pub struct PhysicalASAPDAGCompilation { #[cfg(any(test, feature = "test-support"))] pub fn compile_physical_asap_dag( root: &Rc, + timed: &PhysicalPropertiesAssignment, ) -> Result { - Ok(compile_physical_asap_dag_with_node_ids(root)?.dag) + Ok(compile_physical_asap_dag_with_node_ids(root, timed)?.dag) } -/// Export the timed DAG below `root`. Every reachable node must carry a -/// timing (see [`crate::ir::properties::timing::apply_physical_properties`]); the data-state +/// Export the DAG below `root` with each node's physical properties from +/// `timed`, which must list every reachable node (the completed assignment +/// [`crate::ir::properties::timing::apply_physical_properties`] returns); the data-state /// rules were checked by that pass and are not re-run here. #[cfg(any(test, feature = "test-support"))] pub(crate) fn compile_physical_asap_dag_with_node_ids( root: &Rc, + timed: &PhysicalPropertiesAssignment, ) -> Result { - compile_physical_asap_workload_with_node_ids(&[QueryRoot::Operator(Rc::clone(root))]) + compile_physical_asap_workload_with_node_ids(&[QueryRoot::Operator(Rc::clone(root))], timed) } -/// Export a timed batch as one DAG: operator queries become `roots` and +/// Export a batch as one DAG, each node with its physical properties from +/// `timed` (the completed assignment): operator queries become `roots` and /// standalone scalar queries `scalar_roots`, keyed by their position in the /// batch. pub fn compile_physical_asap_workload( roots: &[QueryRoot], + timed: &PhysicalPropertiesAssignment, ) -> Result { - Ok(compile_physical_asap_workload_with_node_ids(roots)?.dag) + Ok(compile_physical_asap_workload_with_node_ids(roots, timed)?.dag) } pub fn compile_physical_asap_workload_with_node_ids( roots: &[QueryRoot], + timed: &PhysicalPropertiesAssignment, ) -> Result { let (flat, nodes_by_id) = flatten(roots); let mut scalar_roots = Vec::new(); @@ -894,9 +900,12 @@ pub fn compile_physical_asap_workload_with_node_ids( let mut edges = Vec::new(); for (id, flat_node) in flat.nodes.into_iter().enumerate() { let node = &nodes_by_id[id]; - let output_state = data_state(node).ok_or(ExecutionDataStateError::UntimedNode { - operator: node.operator.kind_name(), - })?; + let properties = timed + .get(node) + .ok_or(ExecutionDataStateError::UntimedNode { + operator: node.operator.kind_name(), + })?; + let output_state = data_state(node, timed).expect("listed node has a data state"); // Children come before their parents, so every producer is already in `nodes`. for (producer, role) in edge_roles(&flat_node.operator) { let producer_state = nodes[producer].output_state; @@ -920,7 +929,7 @@ pub fn compile_physical_asap_workload_with_node_ids( id, payload: flat_node.operator, output_state, - lifetime: flat_node.lifetime, + lifetime: properties.lifetime(), output_schema: flat_node.schema, guarantee: flat_node.guarantee, retention_ms: None, diff --git a/crates/types/src/ir/properties/execution_data_state.rs b/crates/types/src/ir/properties/execution_data_state.rs index ac336479c..36459e4c9 100644 --- a/crates/types/src/ir/properties/execution_data_state.rs +++ b/crates/types/src/ir/properties/execution_data_state.rs @@ -3,15 +3,13 @@ //! [`ExecutionTiming`] says when a node's value is produced (ingestion vs. //! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] //! the edge carries (raw values vs. summary state). A materialized output's -//! [`PhysicalProperties`] are the per-sub-DAG choice a node's timing is +//! [`crate::physical::PhysicalProperties`] are the per-sub-DAG choice a node's timing is //! derived from. The rules that assign //! and check them over a DAG live in [`crate::ir::properties::timing`], which reports //! violations as [`ExecutionDataStateError`]. use thiserror::Error; -use crate::physical::{InvalidPhysicalProperties, PhysicalProperties}; - /// When a post-ASAP value is produced. #[derive( Default, Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize, @@ -134,7 +132,8 @@ pub enum ExecutionDataStateError { second: ExecutionDataState, }, /// A plan root that runs at ingestion time without being stored there - /// (a timing fixed by a candidate builder): no query reads its rows. A + /// (an assignment entry that fixes it at ingestion time, not + /// materialized): no query reads its rows. A /// root assigned materialized at ingestion time is valid; the query reads the stored /// output. #[error( @@ -142,27 +141,6 @@ pub enum ExecutionDataStateError { output" )] UnstoredIngestionRoot { operator: &'static str }, - /// An explicit materialization on a node whose timing a candidate builder - /// fixed (e.g. a query-time `FinalizeExactAccumulator` read boundary) - /// that needs the other timing. - #[error( - "{operator} is fixed at {} by its candidate; its assigned materialized output {assigned:?} needs \ - the other timing", - fixed.as_str() - )] - PhysicalPropertiesConflictWithFixedTiming { - operator: &'static str, - fixed: ExecutionTiming, - assigned: PhysicalProperties, - }, - /// A timed node whose timing and lifetime cannot go together: a bug in - /// whatever built the node. - #[error("{operator} node: {reason}")] - InvalidPhysicalProperties { - operator: &'static str, - #[source] - reason: InvalidPhysicalProperties, - }, /// A summary build with a filter placed at ingestion time. An /// ingestion-time build reads raw data samples, which carry none of the /// query's columns the filter reads, so only a query-time build can @@ -190,8 +168,8 @@ pub enum ExecutionDataStateError { /// query time (#509 Algorithm 3, line 13). #[error("{operator}: upstream of ingestion-time node not at ingestion time")] UpstreamNotAtIngestion { operator: &'static str }, - /// A node reached by export without a timing: the physical properties pass - /// was not applied to the DAG first. + /// A node reached by export that the physical properties assignment + /// does not list: the timing pass was not applied to the DAG first. #[error("{operator} node has no execution timing; apply physical properties before export")] UntimedNode { operator: &'static str }, /// The exported physical DAG failed its own validation, e.g. an empty diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs index f934ea019..661a252e8 100644 --- a/crates/types/src/ir/properties/timing.rs +++ b/crates/types/src/ir/properties/timing.rs @@ -1,38 +1,36 @@ -//! Execution timing: written into every node from a materialization -//! assignment, then validated against each operator's kind and its consuming -//! edges. +//! Execution timing: the physical properties of every node, derived from a +//! physical properties assignment, then validated against each operator's +//! kind and its consuming edges. //! -//! The logical DAG carries no timing. Materialization (#509 -//! §Materialization) decides, for each sub-DAG, whether its output is materialized, -//! and if so with which [`PhysicalProperties`]: its timing and its -//! [`Lifetime`] (#1200). [`PhysicalPropertiesAssignment`] records that per -//! sub-DAG, named by the node whose output would be materialized (any node: a -//! summary state, a population, or e.g. the tumbling windows of #509 -//! Example 4), and [`apply_physical_properties`] expands it into a -//! timing and a lifetime on every node: +//! The logical DAG carries no physical properties (#1200: a logical +//! candidate has only its logical properties). Materialization (#509 +//! §Materialization) decides, for each sub-DAG, whether its output is +//! materialized, and if so with which [`PhysicalProperties`]: its timing and +//! its [`Lifetime`] (#1200). [`PhysicalPropertiesAssignment`] records that +//! per sub-DAG, named by the node whose output would be materialized (any +//! node: a summary state, a population, or e.g. the tumbling windows of #509 +//! Example 4), and [`apply_physical_properties`] completes it into the +//! physical properties of every node: //! //! - `SummaryEstimate` and `EvaluatePopulation` run at query time; -//! - a node assigned a materialized output runs at that output's timing. A -//! `SummaryAgg` or `MaintainPopulation` without an assignment is not materialized -//! and runs at query time. Every node upstream of an ingestion-time node -//! also runs at ingestion time, so a materialized ingestion-time output (or an -//! ingestion timing fixed by a candidate builder) over something that can -//! only exist at query time is an error -//! ([`ExecutionDataStateError::UpstreamNotAtIngestion`]); +//! - a node the assignment lists runs at its entry's timing with its entry's +//! lifetime. A `SummaryAgg` or `MaintainPopulation` the assignment does +//! not list is not materialized and runs at query time. Every node +//! upstream of an ingestion-time node also runs at ingestion time, so an +//! ingestion-time entry over something that can only exist at query time +//! is an error ([`ExecutionDataStateError::UpstreamNotAtIngestion`]); //! - a PromQL `BinaryOp` over per-series inputs that are not the same series //! rows runs at query time, since maintenance cannot align them; //! - every other node is not materialized (lifetime `None`) and runs when its //! consumer runs: everything that feeds an ingestion-time node runs at //! ingestion time, everything above an evaluation at query time. //! -//! A node whose timing a candidate builder fixed (e.g. the query-time -//! `FinalizeExactAccumulator` of `finalize_query_candidate`) keeps that -//! timing. An explicit assignment on it that agrees is used; one that needs -//! the other timing is an error -//! ([`ExecutionDataStateError::PhysicalPropertiesConflictWithFixedTiming`]). -//! A root may be materialized at ingestion time across evaluations: the query reads -//! its stored output. A root of rows that runs at ingestion time without -//! being materialized there is an error +//! An entry whose lifetime is `None` fixes only the node's timing: e.g. the +//! completed assignment of a plan Stage 3 selected, applied again for +//! export, gives every node the properties Stage 3 selected. A root may be +//! materialized at ingestion time across evaluations: the query reads its +//! stored output. A root of rows that runs at ingestion time without being +//! materialized there is an error //! ([`ExecutionDataStateError::UnstoredIngestionRoot`]). //! //! A node reached from two consumers that need different timings cannot be @@ -41,7 +39,7 @@ //! is applied, and the pass itself //! rejects a conflict it still finds. //! -//! ## Edge rules (checked after the write) +//! ## Edge rules (checked after the assignment is completed) //! //! | Consumer | Accepts from an input | //! |---|---| @@ -49,7 +47,7 @@ //! | `SummaryEstimate.summary_input` | Summary state at either timing | //! | `SummaryMerge.children` | Summary state at either timing; at ingestion time only for a merge at ingestion time | //! | `FinalizeExactAccumulator.child` | Exact-accumulator state | -//! | `EvaluatePopulation.child` | A timed `MaintainPopulation` | +//! | `EvaluatePopulation.child` | A `MaintainPopulation` | //! | `MaintainPopulation.child` | Rows at its own timing matching the population's input; unlike a `NonASAP` consumer, a query-time population does not read a materialized at ingestion time input's stored output | //! | any `NonASAP` consumer | Rows (or exact-accumulator state for a projection-like operator) at the consumer's own timing, or an output materialized at ingestion time read at query time; ingestion work never reads a query-time value | @@ -65,63 +63,82 @@ use crate::ir::properties::execution_data_state::{ DataPrimitive, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; use crate::ir::schema::{DataType, FieldDataType, Schema}; +use crate::physical::PhysicalProperties; #[cfg(test)] -use crate::physical::Medium; -use crate::physical::{Lifetime, PhysicalProperties}; - -/// Which sub-DAG outputs are materialized, and with which timing and lifetime -/// (#509 §Materialization: "Materialization decides, for each sub-DAG, -/// whether its output is stored"). A sub-DAG is named by the node whose -/// output is materialized (by identity); any node may be assigned. A node without -/// an assignment is not materialized: a summary build or `MaintainPopulation` then -/// runs at query time, every other node when its consumer runs. `Default` -/// materializes nothing. +use crate::physical::{Lifetime, Medium}; + +/// The physical properties of operator nodes, by node identity (#509 +/// §Materialization: "Materialization decides, for each sub-DAG, whether its +/// output is stored"). Stage 2 lists each materialized sub-DAG output, named +/// by the node whose output is materialized; an entry whose lifetime is +/// `None` fixes only the node's timing. A node the assignment does not list +/// is placed by [`apply_physical_properties`]' rules: not materialized, a +/// summary build or `MaintainPopulation` at query time, every other node +/// when its consumer runs. [`apply_physical_properties`] returns the +/// completed assignment, which lists every node it reached; physical stages +/// read a node's timing and lifetime from it. `Default` lists nothing. /// -/// The assignment keeps every assigned node alive, so a later allocation +/// The assignment keeps every listed node alive, so a later allocation /// cannot reuse its address and match a stale entry. #[derive(Clone, Default)] pub struct PhysicalPropertiesAssignment { - materialized: HashMap<*const OperatorNode, (Rc, PhysicalProperties)>, + properties_of: HashMap<*const OperatorNode, (Rc, PhysicalProperties)>, } -/// Each materialized node as its operator kind and address, not its whole sub-DAG. +/// Each listed node as its operator kind and address, not its whole sub-DAG. impl std::fmt::Debug for PhysicalPropertiesAssignment { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let materialized: Vec<_> = self - .materialized + let properties: Vec<_> = self + .properties_of .iter() .map(|(ptr, (node, properties))| { (format!("{}@{ptr:p}", node.operator.kind_name()), properties) }) .collect(); f.debug_struct("PhysicalPropertiesAssignment") - .field("materialized", &materialized) + .field("properties", &properties) .finish() } } +/// Equal when they list the same nodes (by identity) with the same +/// properties. +impl PartialEq for PhysicalPropertiesAssignment { + fn eq(&self, other: &Self) -> bool { + self.properties_of.len() == other.properties_of.len() + && self.properties_of.iter().all(|(node, (_, properties))| { + other + .properties_of + .get(node) + .is_some_and(|(_, other)| other == properties) + }) + } +} + impl PhysicalPropertiesAssignment { - /// Materialize the output of the sub-DAG `node` produces with `materialized`'s - /// timing and lifetime. A `materialized` whose lifetime is `None` does not - /// materialize the output: it removes any assignment of `node`, so `node` - /// is not materialized, in debug and release builds alike. - pub fn set(&mut self, node: &Rc, materialized: PhysicalProperties) { - if materialized.lifetime().is_materialized() { - self.materialized - .insert(Rc::as_ptr(node), (Rc::clone(node), materialized)); - } else { - self.materialized.remove(&Rc::as_ptr(node)); - } + /// Give the node `node` the physical properties `properties`: a + /// materialized lifetime materializes the output of the sub-DAG it produces, + /// lifetime `None` fixes only its timing. + pub fn set(&mut self, node: &Rc, properties: PhysicalProperties) { + self.properties_of + .insert(Rc::as_ptr(node), (Rc::clone(node), properties)); } - /// Give `copy` the assignment of `original`, if it has one: for a node + /// Give `copy` the entry of `original`, if it has one: for a node /// rebuilt as a copy of another. pub fn copy_assignment(&mut self, original: &OperatorNode, copy: &Rc) { - if let Some(materialized) = self.get(original) { - self.set(copy, materialized); + if let Some(properties) = self.get(original) { + self.set(copy, properties); } } + /// The physical properties of `node`, if the assignment lists it. + pub fn get(&self, node: &OperatorNode) -> Option { + self.properties_of + .get(&(node as *const OperatorNode)) + .map(|(_, properties)| *properties) + } + /// Every summary build and `MaintainPopulation` reachable from `roots` /// materialized in memory at ingestion time where the DAG allows it: not under a /// filter, and with nothing below it that can only run at query time. @@ -138,8 +155,7 @@ impl PhysicalPropertiesAssignment { for child in node.children() { visit(child, assignment, visited); } - if node.timing.is_none() - && is_materializable_state_output(&node.operator) + if is_materializable_state_output(&node.operator) && !is_filtered_build(node) && !node .children() @@ -159,55 +175,6 @@ impl PhysicalPropertiesAssignment { } assignment } - - /// How `node` is materialized, if it is. - fn get(&self, node: &OperatorNode) -> Option { - self.materialized - .get(&(node as *const OperatorNode)) - .map(|(_, materialized)| *materialized) - } - - /// How `node` is materialized, checked against what is below it: - /// `upstream_at_query_time` says whether something below it can only - /// run at query time. - fn resolve( - &self, - node: &OperatorNode, - upstream_at_query_time: bool, - ) -> Result, ExecutionDataStateError> { - match self.get(node) { - Some(materialized) - if materialized.timing() == ExecutionTiming::IngestionTime - && is_filtered_build(node) => - { - Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }) - } - Some(materialized) - if materialized.timing() == ExecutionTiming::IngestionTime - && upstream_at_query_time => - { - Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }) - } - materialized => Ok(materialized), - } - } -} - -/// `node`'s physical properties once timed (`None` before), or its error -/// when its timing and lifetime cannot go together. -fn timed_properties( - node: &OperatorNode, -) -> Result, ExecutionDataStateError> { - node.physical_properties().transpose().map_err(|reason| { - ExecutionDataStateError::InvalidPhysicalProperties { - operator: node.operator.kind_name(), - reason, - } - }) } /// Whether `operator`'s output is a sub-DAG output that materialization may @@ -223,6 +190,15 @@ pub(crate) fn is_materializable_state_output(operator: &Operator) -> bool ) } +/// Whether `node` only ever runs at query time: a `SummaryEstimate` or an +/// `EvaluatePopulation`. +fn runs_at_query_time_only(node: &OperatorNode) -> bool { + matches!( + node.asap(), + Some(ASAPOp::SummaryEstimate { .. } | ASAPOp::EvaluatePopulation { .. }) + ) +} + /// Whether `node` is a summary build (`SummaryAgg` or `SummaryTumblingWindowChain`) with a /// filter, which never runs at ingestion time /// ([`ExecutionDataStateError::FilteredSummaryAtIngestion`]). @@ -241,25 +217,6 @@ pub fn is_filtered_build(node: &OperatorNode) -> bool { ) } -/// Cache of one [`apply_physical_properties`] pass: `input node → timed node`, -/// shared by every root of a workload so a node shared by two roots stays -/// one `Rc`. Re-reaching a node with a different timing is a conflict. -#[derive(Default)] -pub struct NodeToTimedNodeCache { - timed_node_of: HashMap<*const OperatorNode, Rc>, -} - -impl NodeToTimedNodeCache { - pub fn new() -> Self { - Self::default() - } - - /// The timed node produced for `input`, if the pass has reached it. - pub(crate) fn timed(&self, input: &Rc) -> Option<&Rc> { - self.timed_node_of.get(&Rc::as_ptr(input)) - } -} - /// What `node`'s output carries: summary state from a build, a merge or a /// node that keeps whole states (a key filter, a projection of the state /// column), raw rows otherwise. @@ -274,14 +231,32 @@ fn primitive(node: &OperatorNode) -> DataPrimitive { } } -/// The data state a timed node's output carries. -pub fn data_state(node: &OperatorNode) -> Option { +/// The data state `node`'s output carries, if `timed` lists it. +pub fn data_state( + node: &OperatorNode, + timed: &PhysicalPropertiesAssignment, +) -> Option { Some(ExecutionDataState { - timing: node.timing?, + timing: timed.get(node)?.timing(), primitive: primitive(node), }) } +/// Whether `node` is a PromQL `BinaryOp` over per-series inputs that are not +/// the same series rows, which maintenance cannot align. +fn is_unaligned_series_binary(node: &OperatorNode) -> bool { + matches!( + &node.operator, + Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) + if node + .schema + .fields + .iter() + .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) + && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) + ) +} + /// Whether the sub-DAG below `node` contains a node that can only run at /// query time (an evaluation, or a sub-DAG `assignment` does not put at /// ingestion time), which forces every consumer above it to query time as well. @@ -301,95 +276,68 @@ fn forces_query_time( .iter() .any(|child| forces_query_time(child, assignment, seen)) }; - let forced = match &node.operator { - _ if node.timing == Some(ExecutionTiming::QueryTime) => true, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => true, - _ if node.timing.is_none() - && (is_materializable_state_output(&node.operator) - || assignment.get(node).is_some()) => - { - let upstream = upstream(seen); - match assignment.resolve(node, upstream) { - Ok(None) if !is_materializable_state_output(&node.operator) => upstream, - Ok(None) => true, - Ok(Some(materialized)) => materialized.timing() == ExecutionTiming::QueryTime, - Err(_) => true, - } - } - Operator::NonASAP(NonASAPOp::BinaryOp { lhs, rhs, .. }) - if node - .schema - .fields - .iter() - .any(|f| f.name == crate::ir::schema::PROMQL_SERIES_IDENTITY) - && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) => - { - true + let forced = if runs_at_query_time_only(node) { + true + } else { + match assignment.get(node) { + Some(listed) if listed.timing() == ExecutionTiming::QueryTime => true, + // An ingestion-time entry the DAG cannot honor runs at query time. + Some(_) => is_filtered_build(node) || upstream(seen), + // A summary build that is not materialized runs at query time. + None if is_materializable_state_output(&node.operator) => true, + None if is_unaligned_series_binary(node) => true, + None => upstream(seen), } - _ => upstream(seen), }; seen.insert(key, forced); forced } -/// Write the timings of `assignment` into every node reachable from `root`, -/// top-down, then validate every edge. Returns the timed copy of `root`; -/// `cache` carries the sharing across the roots of one workload. +/// Complete `assignment` over every node reachable from `root`, top-down, +/// then validate every edge: the physical properties of every node, as the +/// completed assignment. pub fn apply_physical_properties( root: &Rc, assignment: &PhysicalPropertiesAssignment, - cache: &mut NodeToTimedNodeCache, -) -> Result, ExecutionDataStateError> { - let mut forced = HashMap::new(); - let timed = write( - root, - ExecutionTiming::QueryTime, - assignment, - cache, - &mut forced, - )?; - // A root stored at ingestion time is read by the query; one that runs at - // ingestion time without being stored there produces rows nothing reads. - if timed.timing == Some(ExecutionTiming::IngestionTime) - && !timed_properties(&timed)?.is_some_and(|p| p.readable_at(ExecutionTiming::QueryTime)) - && data_state(&timed).map(|s| s.primitive) == Some(DataPrimitive::Raw) - { - return Err(ExecutionDataStateError::UnstoredIngestionRoot { - operator: timed.operator.kind_name(), - }); - } - validate(&timed, &mut HashMap::new())?; - Ok(timed) +) -> Result { + apply_physical_properties_to_roots(&[QueryRoot::Operator(Rc::clone(root))], assignment) } -/// [`apply_physical_properties`] over every root of one workload with -/// one cache, so a node several roots share is timed once and stays one -/// node. A standalone scalar root has each operator it reads timed, and is -/// rebuilt over the timed nodes. +/// [`apply_physical_properties`] over every root of one workload, so a node +/// several roots share is timed once. A standalone scalar root has each +/// operator it reads timed. pub fn apply_physical_properties_to_roots( roots: &[QueryRoot], assignment: &PhysicalPropertiesAssignment, -) -> Result, ExecutionDataStateError> { - let mut cache = NodeToTimedNodeCache::new(); - roots - .iter() - .map(|root| { - Ok(match root { - QueryRoot::Operator(node) => { - QueryRoot::Operator(apply_physical_properties(node, assignment, &mut cache)?) - } - QueryRoot::Scalar(expr) => { - for node in expr.operator_refs() { - apply_physical_properties(node, assignment, &mut cache)?; - } - QueryRoot::Scalar(expr.map_operator_refs(&mut |node| { - Rc::clone(cache.timed(node).expect("scalar reference was timed")) - })) - } - }) - }) - .collect() +) -> Result { + let mut timed = PhysicalPropertiesAssignment::default(); + let mut forced = HashMap::new(); + let mut validated = HashMap::new(); + for root in roots { + for node in root.operator_refs() { + write( + node, + ExecutionTiming::QueryTime, + assignment, + &mut timed, + &mut forced, + )?; + // A root stored at ingestion time is read by the query; one that + // runs at ingestion time without being stored there produces rows + // nothing reads. + let properties = timed.get(node).expect("written node is timed"); + if properties.timing() == ExecutionTiming::IngestionTime + && !properties.readable_at(ExecutionTiming::QueryTime) + && primitive(node) == DataPrimitive::Raw + { + return Err(ExecutionDataStateError::UnstoredIngestionRoot { + operator: node.operator.kind_name(), + }); + } + validate(node, &timed, &mut validated)?; + } + } + Ok(timed) } /// Validate the sub-DAG below `root` with every sub-DAG materialized at ingestion time @@ -398,7 +346,6 @@ pub fn apply_physical_properties_to_roots( /// `root` consumed at `root_timing`. For planning-time legality checks of a /// candidate before it is assembled into a workload DAG: a candidate must stay /// executable if materialization later puts its states at ingestion time. -/// Nothing is materialized. #[cfg(any(test, feature = "test-support"))] pub fn validate_at_ingestion( root: &Rc, @@ -407,10 +354,10 @@ pub fn validate_at_ingestion( let assignment = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(root), ); - let mut cache = NodeToTimedNodeCache::new(); + let mut timed = PhysicalPropertiesAssignment::default(); let mut forced = HashMap::new(); - let timed = write(root, root_timing, &assignment, &mut cache, &mut forced)?; - validate(&timed, &mut HashMap::new()) + write(root, root_timing, &assignment, &mut timed, &mut forced)?; + validate(root, &timed, &mut HashMap::new()) } /// The data state `node` produces with every sub-DAG materialized at ingestion time where @@ -426,80 +373,50 @@ pub fn planned_data_state( let assignment = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(node), ); - let (timing, _) = placement(node, consumer, &assignment, &mut forced) + let properties = placement(node, consumer, &assignment, &mut forced) .expect("an assignment materialized where allowed never fails"); ExecutionDataState { - timing, + timing: properties.timing(), primitive: primitive(node), } } -/// The timing and lifetime `node` takes when its consumer runs at +/// The physical properties `node` takes when its consumer runs at /// `consumer`. fn placement( node: &Rc, consumer: ExecutionTiming, assignment: &PhysicalPropertiesAssignment, forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result<(ExecutionTiming, Lifetime), ExecutionDataStateError> { - // A placement fixed when the candidate was built (an exact-state read - // boundary that must run at query time, or one that feeds maintenance) - // is honored; a conflicting consumer is rejected by validation. - let upstream_at_query_time = |forced: &mut HashMap<*const OperatorNode, bool>| { - node.children() - .iter() - .any(|child| forces_query_time(child, assignment, forced)) - }; - if let Some(placed) = node.timing { - if placed == ExecutionTiming::IngestionTime && is_filtered_build(node) { - return Err(ExecutionDataStateError::FilteredSummaryAtIngestion { - operator: node.operator.kind_name(), - }); - } - if placed == ExecutionTiming::IngestionTime && upstream_at_query_time(forced) { - return Err(ExecutionDataStateError::UpstreamNotAtIngestion { - operator: node.operator.kind_name(), - }); - } - // An explicit assignment is never silently dropped: one that agrees - // with the fixed timing is used, one that needs the other timing is - // an error. - return match assignment.get(node) { - Some(materialized) if materialized.timing() != placed => Err( - ExecutionDataStateError::PhysicalPropertiesConflictWithFixedTiming { +) -> Result { + if let Some(listed) = assignment.get(node) { + if listed.timing() == ExecutionTiming::IngestionTime { + if is_filtered_build(node) { + return Err(ExecutionDataStateError::FilteredSummaryAtIngestion { operator: node.operator.kind_name(), - fixed: placed, - assigned: materialized, - }, - ), - Some(materialized) => Ok((placed, materialized.lifetime())), - None => Ok((placed, node.lifetime)), - }; - } - Ok(match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - | Operator::ASAP(ASAPOp::EvaluatePopulation { .. }) => match assignment.get(node) { - Some(materialized) if materialized.timing() == ExecutionTiming::IngestionTime => { + }); + } + if runs_at_query_time_only(node) + || node + .children() + .iter() + .any(|child| forces_query_time(child, assignment, forced)) + { return Err(ExecutionDataStateError::UpstreamNotAtIngestion { operator: node.operator.kind_name(), - }) - } - Some(materialized) => (ExecutionTiming::QueryTime, materialized.lifetime()), - None => (ExecutionTiming::QueryTime, Lifetime::None), - }, - _ if is_materializable_state_output(&node.operator) || assignment.get(node).is_some() => { - let upstream = upstream_at_query_time(forced); - match assignment.resolve(node, upstream)? { - None if !is_materializable_state_output(&node.operator) => { - (consumer, Lifetime::None) - } - // A summary build that is not materialized runs at query time. - None => (ExecutionTiming::QueryTime, Lifetime::None), - Some(materialized) => (materialized.timing(), materialized.lifetime()), + }); } } - _ => (consumer, Lifetime::None), - }) + return Ok(listed); + } + // A summary build that is not materialized runs at query time. + Ok( + if runs_at_query_time_only(node) || is_materializable_state_output(&node.operator) { + PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME + } else { + PhysicalProperties::not_materialized_at(consumer) + }, + ) } /// The timing `node` takes when its consumer runs at `consumer`, by the @@ -516,78 +433,61 @@ pub fn timing_under_consumer( query_time_forced: &mut HashMap<*const OperatorNode, bool>, ) -> ExecutionTiming { placement(node, consumer, assignment, query_time_forced) - .map_or(ExecutionTiming::QueryTime, |(timing, _)| timing) + .map_or(ExecutionTiming::QueryTime, |properties| properties.timing()) } +/// List `node` and every node below it in `timed`, `node` consumed at +/// `consumer`. fn write( node: &Rc, consumer: ExecutionTiming, assignment: &PhysicalPropertiesAssignment, - cache: &mut NodeToTimedNodeCache, + timed: &mut PhysicalPropertiesAssignment, forced: &mut HashMap<*const OperatorNode, bool>, -) -> Result, ExecutionDataStateError> { - let (timing, lifetime) = placement(node, consumer, assignment, forced)?; - if let Some(done) = cache.timed_node_of.get(&Rc::as_ptr(node)) { - let previous = done.timing.expect("memoized node is timed"); - if previous != timing { +) -> Result<(), ExecutionDataStateError> { + let properties = placement(node, consumer, assignment, forced)?; + if let Some(previous) = timed.get(node) { + if previous.timing() != properties.timing() { return Err(ExecutionDataStateError::ConflictingTiming { first: ExecutionDataState { - timing: previous, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), + timing: previous.timing(), + primitive: primitive(node), }, second: ExecutionDataState { - timing, - primitive: data_state(done).map_or(DataPrimitive::Raw, |s| s.primitive), + timing: properties.timing(), + primitive: primitive(node), }, }); } - return Ok(Rc::clone(done)); + return Ok(()); } - let mut error = None; - let operator = - node.operator.map_children( - |child| match write(child, timing, assignment, cache, forced) { - Ok(timed) => timed, - Err(e) => { - error.get_or_insert(e); - Rc::clone(child) - } - }, - ); - if let Some(e) = error { - return Err(e); + for child in node.children() { + write(child, properties.timing(), assignment, timed, forced)?; } - let timed = Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(Some(timing)) - .with_lifetime(lifetime), - ); - cache - .timed_node_of - .insert(Rc::as_ptr(node), Rc::clone(&timed)); - Ok(timed) + timed.set(node, properties); + Ok(()) } -fn state_of(node: &OperatorNode) -> ExecutionDataState { - data_state(node).expect("timed node") +fn state_of(node: &OperatorNode, timed: &PhysicalPropertiesAssignment) -> ExecutionDataState { + data_state(node, timed).expect("timed node") } /// Check every edge below `node` against the module-level rules. fn validate( node: &Rc, + timed: &PhysicalPropertiesAssignment, seen: &mut HashMap<*const OperatorNode, ()>, ) -> Result<(), ExecutionDataStateError> { if seen.insert(Rc::as_ptr(node), ()).is_some() { return Ok(()); } - let timing = node.timing.expect("timed node"); + let timing = timed.get(node).expect("timed node").timing(); match &node.operator { - Operator::ASAP(op) => validate_asap(node, op, timing)?, - Operator::NonASAP(op) => validate_non_asap(node, op, timing)?, + Operator::ASAP(op) => validate_asap(node, op, timing, timed)?, + Operator::NonASAP(op) => validate_non_asap(node, op, timing, timed)?, } for child in node.children() { - validate(child, seen)?; + validate(child, timed, seen)?; } Ok(()) } @@ -610,6 +510,7 @@ fn validate_asap( node: &OperatorNode, op: &ASAPOp, timing: ExecutionTiming, + timed: &PhysicalPropertiesAssignment, ) -> Result<(), ExecutionDataStateError> { match op { ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. } => { @@ -617,7 +518,7 @@ fn validate_asap( ASAPOp::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain.child", _ => "SummaryAgg.child", }; - let avail = state_of(child); + let avail = state_of(child, timed); match avail { ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => {} s if s.primitive == DataPrimitive::SummaryState => { @@ -637,7 +538,7 @@ fn validate_asap( // `NonASAP` consumer does. if timing == ExecutionTiming::QueryTime && avail.timing == ExecutionTiming::IngestionTime - && !timed_properties(child)?.is_some_and(|p| p.readable_at(timing)) + && !timed.get(child).is_some_and(|p| p.readable_at(timing)) { return Err( ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { @@ -659,7 +560,7 @@ fn validate_asap( Ok(()) } ASAPOp::SummaryEstimate { summary_input, .. } => { - let s = state_of(summary_input); + let s = state_of(summary_input, timed); if s.primitive != DataPrimitive::SummaryState { return Err(ExecutionDataStateError::IllegalChildDataState { edge: "SummaryEstimate.summary_input", @@ -669,13 +570,13 @@ fn validate_asap( if timing != ExecutionTiming::QueryTime { return Err(ExecutionDataStateError::IllegalChildDataState { edge: "SummaryEstimate", - child: state_of(node), + child: state_of(node, timed), }); } Ok(()) } ASAPOp::FinalizeExactAccumulator { child } => { - let s = state_of(child); + let s = state_of(child, timed); if s.primitive != DataPrimitive::SummaryState || is_exact_accumulator_state(&child.schema).is_err() || (timing == ExecutionTiming::IngestionTime && s.timing != timing) @@ -689,7 +590,7 @@ fn validate_asap( } ASAPOp::MaintainPopulation { child, population } => { let valid = population.matches_node(child) - && state_of(child) + && state_of(child, timed) == ExecutionDataState { timing, primitive: DataPrimitive::Raw, @@ -705,7 +606,6 @@ fn validate_asap( &child.operator, Operator::ASAP(ASAPOp::MaintainPopulation { population, .. }) if population.supports(evaluation) - && child.timing.is_some() ); if !valid { return Err(ExecutionDataStateError::InvalidMaintainedPopulation); @@ -714,7 +614,7 @@ fn validate_asap( } ASAPOp::SummaryMerge { children, .. } => { for child in children { - let s = state_of(child); + let s = state_of(child, timed); if s.primitive != DataPrimitive::SummaryState || (timing == ExecutionTiming::IngestionTime && s.timing != timing) { @@ -772,12 +672,13 @@ fn validate_non_asap( node: &OperatorNode, op: &NonASAPOp, timing: ExecutionTiming, + timed: &PhysicalPropertiesAssignment, ) -> Result<(), ExecutionDataStateError> { // Every input is rows at this node's own timing. Ingestion work never // reads a query-time value; exact-accumulator state may pass through // the projection-like operators unchanged. for child in op.children() { - let s = state_of(child); + let s = state_of(child, timed); let passes_state = matches!( op, NonASAPOp::Project { .. } @@ -788,7 +689,7 @@ fn validate_non_asap( && (is_exact_accumulator_state(&child.schema).is_ok() || selects_states(node)); // A query-time consumer may read the stored output of a sub-DAG // materialized at ingestion time. - let reads_stored = timed_properties(child)?.is_some_and(|p| p.readable_at(timing)); + let reads_stored = timed.get(child).is_some_and(|p| p.readable_at(timing)); if (s.timing != timing && !reads_stored) || (s.primitive != DataPrimitive::Raw && !passes_state) { @@ -1015,26 +916,44 @@ mod tests { aggregate(AggIntent::Max { col: None }, child) } - /// `node` with its timing fixed in advance, as a candidate builder does - /// for an operator that must feed maintenance. - fn placed_at_ingestion(node: Rc) -> Rc { - Rc::new( - (*node) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ) + /// An assignment that fixes each of `nodes` at ingestion time, not + /// materialized, as for an operator that must feed maintenance. + fn fixed_at_ingestion(nodes: &[&Rc]) -> PhysicalPropertiesAssignment { + let mut assignment = PhysicalPropertiesAssignment::default(); + for node in nodes { + assignment.set(node, PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION); + } + assignment } - /// Apply with every sub-DAG materialized at ingestion time where the DAG allows it, the - /// placement whose edge rules these tests exercise. - fn apply(root: &Rc) -> Result, ExecutionDataStateError> { - apply_physical_properties( - root, - &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( - std::slice::from_ref(root), - ), - &mut NodeToTimedNodeCache::new(), - ) + /// Apply with every sub-DAG materialized at ingestion time where the DAG + /// allows it, the placement whose edge rules these tests exercise, and + /// each of `fixed` fixed at ingestion time, not materialized. + fn apply_fixing( + root: &Rc, + fixed: &[&Rc], + ) -> Result { + let mut assignment = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( + std::slice::from_ref(root), + ); + for node in fixed { + assignment.set(node, PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION); + } + apply_physical_properties(root, &assignment) + } + + fn apply( + root: &Rc, + ) -> Result { + apply_fixing(root, &[]) + } + + fn timing_of(timed: &PhysicalPropertiesAssignment, node: &OperatorNode) -> ExecutionTiming { + timed.get(node).expect("timed node").timing() + } + + fn lifetime_of(timed: &PhysicalPropertiesAssignment, node: &OperatorNode) -> Lifetime { + timed.get(node).expect("timed node").lifetime() } const MATERIALIZED_ACROSS: Lifetime = Lifetime::AcrossEvaluations(Medium::Memory); @@ -1049,26 +968,22 @@ mod tests { /// Without a materialization decision, a summary is not materialized and /// it and its input run at query time; an explicit choice overrides the - /// default and is recorded on the timed node. + /// default and is recorded in the completed assignment. #[test] fn default_assignment_materializes_nothing() { let summary = agg(scan(), kll()); - let root = apply_physical_properties( - &summary, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); + let timed = + apply_physical_properties(&summary, &PhysicalPropertiesAssignment::default()).unwrap(); assert_eq!( - data_state(&root), + data_state(&summary, &timed), Some(ExecutionDataState { timing: ExecutionTiming::QueryTime, primitive: DataPrimitive::SummaryState, }) ); - assert_eq!(root.lifetime, Lifetime::None); + assert_eq!(lifetime_of(&timed, &summary), Lifetime::None); assert_eq!( - data_state(&child(&root)), + data_state(&child(&summary), &timed), Some(ExecutionDataState::QUERY_ROWS) ); for (lifetime, timing) in [ @@ -1080,13 +995,11 @@ mod tests { ] { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set(&summary, properties(timing, lifetime)); - let root = - apply_physical_properties(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(root.timing, Some(timing)); - assert_eq!(root.lifetime, lifetime); - assert_eq!(child(&root).timing, Some(timing)); - assert_eq!(child(&root).lifetime, Lifetime::None); + let timed = apply_physical_properties(&summary, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &summary), timing); + assert_eq!(lifetime_of(&timed, &summary), lifetime); + assert_eq!(timing_of(&timed, &child(&summary)), timing); + assert_eq!(lifetime_of(&timed, &child(&summary)), Lifetime::None); } } @@ -1103,12 +1016,7 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); assert_eq!( - apply_physical_properties( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), + apply_physical_properties(&estimate(outer), &assignment).err(), Some(ExecutionDataStateError::UpstreamNotAtIngestion { operator: "SummaryAgg" }) @@ -1122,25 +1030,21 @@ mod tests { &outer, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let root = apply_physical_properties( - &estimate(outer), - &assignment, - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - assert_eq!(child(&root).lifetime, MATERIALIZED_ACROSS); - assert_eq!(child(&child(&root)).lifetime, MATERIALIZED_ACROSS); + let timed = apply_physical_properties(&estimate(Rc::clone(&outer)), &assignment).unwrap(); + assert_eq!(lifetime_of(&timed, &outer), MATERIALIZED_ACROSS); + assert_eq!(lifetime_of(&timed, &inner), MATERIALIZED_ACROSS); } #[test] fn summary_agg_input_runs_at_ingestion_time() { - let root = apply(&agg(scan(), kll())).unwrap(); + let root = agg(scan(), kll()); + let timed = apply(&root).unwrap(); assert_eq!( - data_state(&root), + data_state(&root, &timed), Some(ExecutionDataState::INGESTION_SUMMARY) ); assert_eq!( - data_state(&child(&root)), + data_state(&child(&root), &timed), Some(ExecutionDataState::INGESTION_ROWS) ); } @@ -1154,8 +1058,9 @@ mod tests { #[test] fn evaluation_can_feed_summary_construction_at_query_time() { let inner = estimate(agg(scan(), kll())); - let root = apply(&estimate(agg(inner, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); + let root = estimate(agg(inner, kll())); + let timed = apply(&root).unwrap(); + assert_eq!(timing_of(&timed, &child(&root)), ExecutionTiming::QueryTime); } /// Any non-ASAP operator over an evaluation runs at query time. @@ -1169,40 +1074,44 @@ mod tests { })) .unwrap(); for root in [max(evaluation()), sorted] { - let root = apply(&root).unwrap(); - assert_eq!(data_state(&root), Some(ExecutionDataState::QUERY_ROWS)); + let timed = apply(&root).unwrap(); + assert_eq!( + data_state(&root, &timed), + Some(ExecutionDataState::QUERY_ROWS) + ); } } #[test] fn query_time_values_can_feed_query_time_summary_construction() { let post = max(estimate(agg(scan(), kll()))); - let root = apply(&estimate(agg(post, kll()))).unwrap(); - assert_eq!(child(&root).timing, Some(ExecutionTiming::QueryTime)); + let root = estimate(agg(post, kll())); + let timed = apply(&root).unwrap(); + assert_eq!(timing_of(&timed, &child(&root)), ExecutionTiming::QueryTime); } #[test] fn function_under_summary_agg_is_legal_but_not_at_root() { - let operation = placed_at_ingestion(max(scan())); + let operation = max(scan()); assert_eq!( - apply(&operation).err(), + apply_fixing(&operation, &[&operation]).err(), Some(ExecutionDataStateError::UnstoredIngestionRoot { operator: "Aggregate" }) ); - let root = apply(&estimate(agg(operation, kll()))).unwrap(); - let timed_operation = child(&child(&root)); + let root = estimate(agg(Rc::clone(&operation), kll())); + let timed = apply_fixing(&root, &[&operation]).unwrap(); assert_eq!( - data_state(&timed_operation), + data_state(&operation, &timed), Some(ExecutionDataState::INGESTION_ROWS) ); } #[test] fn function_over_evaluation_is_rejected() { - let operation = placed_at_ingestion(max(estimate(agg(scan(), kll())))); + let operation = max(estimate(agg(scan(), kll()))); assert_eq!( - apply(&estimate(agg(operation, kll()))).err(), + apply_fixing(&estimate(agg(Rc::clone(&operation), kll())), &[&operation]).err(), Some(ExecutionDataStateError::UpstreamNotAtIngestion { operator: "Aggregate" }) @@ -1231,24 +1140,21 @@ mod tests { ); } - /// A finalize pinned to ingestion time (as fixed-window rate candidates - /// build it) over a state that is not materialized at ingestion time names the missing - /// upstream, not an illegal edge. + /// A finalize fixed at ingestion time over a state that is not + /// materialized at ingestion time names the missing upstream, not an + /// illegal edge. #[test] - fn ingestion_pinned_finalize_over_query_time_state_is_upstream_not_at_ingestion() { - let finalize = placed_at_ingestion( - OperatorNode::new_shared(crate::ir::Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { - child: agg(scan(), exact_sum()), - }, - )) - .unwrap(), - ); + fn ingestion_fixed_finalize_over_query_time_state_is_upstream_not_at_ingestion() { + let finalize = OperatorNode::new_shared(crate::ir::Operator::ASAP( + ASAPOp::FinalizeExactAccumulator { + child: agg(scan(), exact_sum()), + }, + )) + .unwrap(); assert_eq!( apply_physical_properties( - &estimate(agg(finalize, kll())), - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() + &estimate(agg(Rc::clone(&finalize), kll())), + &fixed_at_ingestion(&[&finalize]), ) .err(), Some(ExecutionDataStateError::UpstreamNotAtIngestion { @@ -1257,10 +1163,10 @@ mod tests { ); } - /// Setting a lifetime `None` does not materialize the node, in release - /// builds too: it removes the node's assignment. + /// An entry whose lifetime is `None` replaces a materialized one and + /// records only the node's timing. #[test] - fn setting_a_lifetime_none_removes_the_assignment() { + fn setting_a_lifetime_none_records_only_the_timing() { let node = max(scan()); let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( @@ -1268,9 +1174,10 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); assignment.set(&node, PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME); - assert_eq!(assignment.get(&node), None); - assignment.set(&node, PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION); - assert_eq!(assignment.get(&node), None); + assert_eq!( + assignment.get(&node), + Some(PhysicalProperties::NOT_MATERIALIZED_AT_QUERY_TIME) + ); } /// Any node may be materialized, not only a summary state: stored rows @@ -1285,14 +1192,16 @@ mod tests { &tumbling_windows, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).lifetime, MATERIALIZED_ACROSS); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::IngestionTime)); + let timed = apply_physical_properties(&root, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &root), ExecutionTiming::QueryTime); + assert_eq!(lifetime_of(&timed, &tumbling_windows), MATERIALIZED_ACROSS); + assert_eq!( + timing_of(&timed, &tumbling_windows), + ExecutionTiming::IngestionTime + ); assert_eq!( - child(&child(&timed)).timing, - Some(ExecutionTiming::IngestionTime) + timing_of(&timed, &child(&tumbling_windows)), + ExecutionTiming::IngestionTime ); let over_evaluation = max(estimate(agg(scan(), kll()))); @@ -1302,12 +1211,7 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); assert_eq!( - apply_physical_properties( - &over_evaluation, - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), + apply_physical_properties(&over_evaluation, &assignment).err(), Some(ExecutionDataStateError::UpstreamNotAtIngestion { operator: "Aggregate" }) @@ -1354,20 +1258,15 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); assert_eq!( - apply_physical_properties( - &estimate(Rc::clone(&filtered)), - &assignment, - &mut NodeToTimedNodeCache::new() - ) - .err(), + apply_physical_properties(&estimate(Rc::clone(&filtered)), &assignment).err(), Some(ExecutionDataStateError::FilteredSummaryAtIngestion { operator: "SummaryAgg" }) ); - let timed = apply(&estimate(filtered)).unwrap(); - assert_eq!(child(&timed).timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).lifetime, Lifetime::None); - assert!(validate_at_ingestion(&child(&timed), ExecutionTiming::QueryTime).is_ok()); + let timed = apply(&estimate(Rc::clone(&filtered))).unwrap(); + assert_eq!(timing_of(&timed, &filtered), ExecutionTiming::QueryTime); + assert_eq!(lifetime_of(&timed, &filtered), Lifetime::None); + assert!(validate_at_ingestion(&filtered, ExecutionTiming::QueryTime).is_ok()); } fn finalize(child: Rc) -> Rc { @@ -1377,9 +1276,14 @@ mod tests { .unwrap() } - /// The lifetime the exported DAG records for `node`'s operator kind. - fn exported(root: &Rc, kind: &str) -> Lifetime { - let dag = crate::ir::physical_export::compile_physical_asap_dag(root).unwrap(); + /// The lifetime the exported DAG of `root`, timed as `timed`, records + /// for `node`'s operator kind. + fn exported( + root: &Rc, + timed: &PhysicalPropertiesAssignment, + kind: &str, + ) -> Lifetime { + let dag = crate::ir::physical_export::compile_physical_asap_dag(root, timed).unwrap(); dag.validate().unwrap(); dag.nodes .iter() @@ -1388,46 +1292,18 @@ mod tests { .lifetime } - /// A node whose timing its candidate fixed (the query-time read boundary - /// of `finalize_query_candidate`) keeps an explicit assignment that agrees - /// with that timing and rejects one that needs the other timing. + /// An assignment entry gives a node its timing and lifetime, and the + /// exported DAG records them. #[test] - fn fixed_timing_keeps_an_agreeing_assignment_and_rejects_a_conflict() { - let state = agg(scan(), exact_sum()); - let read = Rc::new( - (*finalize(Rc::clone(&state))) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); + fn an_entry_gives_a_node_its_timing_and_lifetime() { + let read = finalize(agg(scan(), exact_sum())); let stored = Lifetime::OneEvaluation(Medium::Disk); let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set(&read, properties(ExecutionTiming::QueryTime, stored)); - let timed = apply_physical_properties(&read, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(timed.lifetime, stored); - assert_eq!(exported(&timed, "FinalizeExactAccumulator"), stored); - - let mut assignment = PhysicalPropertiesAssignment::default(); - assignment.set( - &state, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - ); - assignment.set( - &read, - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - ); - assert_eq!( - apply_physical_properties(&read, &assignment, &mut NodeToTimedNodeCache::new()).err(), - Some( - ExecutionDataStateError::PhysicalPropertiesConflictWithFixedTiming { - operator: "FinalizeExactAccumulator", - fixed: ExecutionTiming::QueryTime, - assigned: - PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, - } - ) - ); + let timed = apply_physical_properties(&read, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &read), ExecutionTiming::QueryTime); + assert_eq!(lifetime_of(&timed, &read), stored); + assert_eq!(exported(&read, &timed, "FinalizeExactAccumulator"), stored); } /// The Debug output names each assigned node by its operator kind, not @@ -1444,44 +1320,15 @@ mod tests { assert!(!text.contains("fields"), "{text}"); } - /// A node fixed at ingestion time by its candidate rejects a `OneEvaluation` - /// assignment, which needs the other timing. - #[test] - fn ingestion_fixed_node_rejects_mat_at_query() { - let fixed = placed_at_ingestion(max(scan())); - let root = max(Rc::clone(&fixed)); - let at_query = properties( - ExecutionTiming::QueryTime, - Lifetime::OneEvaluation(Medium::Memory), - ); - let mut assignment = PhysicalPropertiesAssignment::default(); - assignment.set(&fixed, at_query); - assert_eq!( - apply_physical_properties(&root, &assignment, &mut NodeToTimedNodeCache::new()).err(), - Some( - ExecutionDataStateError::PhysicalPropertiesConflictWithFixedTiming { - operator: "Aggregate", - fixed: ExecutionTiming::IngestionTime, - assigned: at_query, - } - ) - ); - } - /// A query-time summary build cannot read ingestion-time rows that are /// not stored: at query time they no longer exist. Storing the rows node /// at ingestion time makes the same build valid (#738). #[test] fn query_time_summary_over_unstored_ingestion_rows_is_rejected() { - let rows = placed_at_ingestion(max(scan())); + let rows = max(scan()); let summary = agg(Rc::clone(&rows), kll()); assert_eq!( - apply_physical_properties( - &summary, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), + apply_physical_properties(&summary, &fixed_at_ingestion(&[&rows])).err(), Some( ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { edge: "SummaryAgg.child", @@ -1495,26 +1342,19 @@ mod tests { &rows, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = - apply_physical_properties(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).lifetime, MATERIALIZED_ACROSS); + let timed = apply_physical_properties(&summary, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &summary), ExecutionTiming::QueryTime); + assert_eq!(lifetime_of(&timed, &child(&summary)), MATERIALIZED_ACROSS); } /// The same holds for exact-accumulator state: a query-time build cannot /// read an ingestion-time exact state that is not stored. #[test] fn query_time_summary_over_unstored_ingestion_state_is_rejected() { - let state = placed_at_ingestion(agg(scan(), exact_sum())); + let state = agg(scan(), exact_sum()); let summary = agg(Rc::clone(&state), kll()); assert_eq!( - apply_physical_properties( - &summary, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), + apply_physical_properties(&summary, &fixed_at_ingestion(&[&state])).err(), Some( ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { edge: "SummaryAgg.child", @@ -1528,82 +1368,9 @@ mod tests { &state, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = - apply_physical_properties(&summary, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::QueryTime)); - assert_eq!(child(&timed).lifetime, MATERIALIZED_ACROSS); - } - - /// A node placed at ingestion time with a lifetime that cannot go with - /// it reports that, not "not stored". - #[test] - fn an_invalid_timed_node_reports_its_error() { - let root = Rc::new( - (*placed_at_ingestion(max(scan()))) - .clone() - .with_lifetime(Lifetime::OneEvaluation(Medium::Memory)), - ); - assert_eq!( - apply_physical_properties( - &root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::InvalidPhysicalProperties { - operator: "Aggregate", - reason: crate::physical::InvalidPhysicalProperties::OneEvaluationAtIngestion, - }) - ); - } - - /// `node` fixed at ingestion time with a lifetime that cannot go with it. - fn invalid_at_ingestion(node: Rc) -> Rc { - Rc::new( - (*placed_at_ingestion(node)) - .clone() - .with_lifetime(Lifetime::OneEvaluation(Medium::Memory)), - ) - } - - /// A query-time summary build whose ingestion-time input has an invalid - /// timing and lifetime reports that (`validate_asap`), not "not stored". - #[test] - fn a_summary_build_over_an_invalid_input_reports_its_error() { - let root = agg(invalid_at_ingestion(scan()), exact_sum()); - assert_eq!( - apply_physical_properties( - &root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::InvalidPhysicalProperties { - operator: "Scan", - reason: crate::physical::InvalidPhysicalProperties::OneEvaluationAtIngestion, - }) - ); - } - - /// A query-time `NonASAP` node whose ingestion-time input has an invalid - /// timing and lifetime reports that (`validate_non_asap`), not an illegal - /// child data state. - #[test] - fn a_non_asap_node_over_an_invalid_input_reports_its_error() { - let root = max(invalid_at_ingestion(max(scan()))); - assert_eq!( - apply_physical_properties( - &root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), - Some(ExecutionDataStateError::InvalidPhysicalProperties { - operator: "Aggregate", - reason: crate::physical::InvalidPhysicalProperties::OneEvaluationAtIngestion, - }) - ); + let timed = apply_physical_properties(&summary, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &summary), ExecutionTiming::QueryTime); + assert_eq!(lifetime_of(&timed, &child(&summary)), MATERIALIZED_ACROSS); } /// A root stored at ingestion time is valid: the query reads the stored @@ -1617,10 +1384,9 @@ mod tests { &root, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); - assert_eq!(exported(&timed, "Aggregate"), MATERIALIZED_ACROSS); + let timed = apply_physical_properties(&root, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &root), ExecutionTiming::IngestionTime); + assert_eq!(exported(&root, &timed, "Aggregate"), MATERIALIZED_ACROSS); let state = agg(scan(), exact_sum()); let root = finalize(Rc::clone(&state)); @@ -1633,22 +1399,16 @@ mod tests { &root, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&root, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - assert_eq!(timed.timing, Some(ExecutionTiming::IngestionTime)); + let timed = apply_physical_properties(&root, &assignment).unwrap(); + assert_eq!(timing_of(&timed, &root), ExecutionTiming::IngestionTime); assert_eq!( - exported(&timed, "FinalizeExactAccumulator"), + exported(&root, &timed, "FinalizeExactAccumulator"), MATERIALIZED_ACROSS ); - let root = placed_at_ingestion(max(scan())); + let root = max(scan()); assert_eq!( - apply_physical_properties( - &root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new() - ) - .err(), + apply_physical_properties(&root, &fixed_at_ingestion(&[&root])).err(), Some(ExecutionDataStateError::UnstoredIngestionRoot { operator: "Aggregate" }) @@ -1738,7 +1498,7 @@ mod tests { }), schema, )); - validate_at_ingestion(&placed_at_ingestion(binary), ExecutionTiming::IngestionTime) + validate_at_ingestion(&binary, ExecutionTiming::IngestionTime) }; let mut schema = Schema::lifted( vec![ diff --git a/crates/types/src/ir/schema/schema_support.rs b/crates/types/src/ir/schema/schema_support.rs index 820dd29bf..d63742553 100644 --- a/crates/types/src/ir/schema/schema_support.rs +++ b/crates/types/src/ir/schema/schema_support.rs @@ -92,8 +92,6 @@ pub fn with_promql_series_identity( } let mut rebuilt = OperatorNode::new(operator).map_err(|e| e.to_string())?; rebuilt.guarantee = node.guarantee.clone(); - rebuilt.timing = node.timing; - rebuilt.lifetime = node.lifetime; let rebuilt = Rc::new(rebuilt); cache.insert(Rc::as_ptr(node), Rc::clone(&rebuilt)); Ok(rebuilt) diff --git a/crates/types/src/physical/properties.rs b/crates/types/src/physical/properties.rs index 622ccfe5d..46fedf68d 100644 --- a/crates/types/src/physical/properties.rs +++ b/crates/types/src/physical/properties.rs @@ -103,6 +103,14 @@ impl PhysicalProperties { } } + /// `(timing, None)`: not materialized, at `timing`. + pub fn not_materialized_at(timing: ExecutionTiming) -> Self { + Self { + timing, + lifetime: Lifetime::None, + } + } + pub fn timing(self) -> ExecutionTiming { self.timing } diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs index c27a5439c..5fb8e5306 100644 --- a/crates/types/src/physical/workload_plan.rs +++ b/crates/types/src/physical/workload_plan.rs @@ -1,6 +1,6 @@ //! One evaluated physical properties assignment rebuilt as a physical DAG. use crate::ir::physical_export::PhysicalASAPDAG; -use crate::ir::QueryRoot; +use crate::ir::{PhysicalPropertiesAssignment, QueryRoot}; /// One CandidatePhysicalASAPDAG (Stage 2 output), derived from exactly one /// CandidateLogicalASAPDAG (Stage 1 output). It holds planning data only; @@ -11,9 +11,14 @@ pub struct PhysicalWorkloadPlan { pub id: String, /// The id of the logical candidate it implements. pub from_logical: String, - /// The timed query roots, one per query in workload order, that `dag` + /// The query roots, one per query in workload order, that `dag` /// exports. pub roots: Vec, + /// The physical properties of every node of `roots`: the completed + /// assignment `dag` exports them with. The facade adds the nodes it + /// rebuilds from them (`with_confirmed_weights_at`), so it also covers the + /// plan it runs. + pub physical_properties: PhysicalPropertiesAssignment, pub dag: PhysicalASAPDAG, } diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 6ce4619a3..1fbdbbeab 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -9,7 +9,7 @@ use crate::ir::operator::maintained_population::CurrentSeriesInput; use crate::ir::operator::node::Operator; use crate::ir::physical_export::PhysicalASAPDAGCompilation; use crate::ir::properties::summary_coverage::SummaryCoverage; -use crate::ir::properties::timing::NodeToTimedNodeCache; +use crate::ir::properties::timing::PhysicalPropertiesAssignment; use crate::ir::properties::ExecutionDataStateError; use crate::ir::schema::Schema; use crate::ir::{ASAPOp, NonASAPOp}; @@ -36,8 +36,9 @@ pub fn tumbling_window( /// `physical_export::compile_physical_asap_dag_with_node_ids`. pub fn compile_physical_asap_dag_with_node_ids( root: &Rc, + timed: &PhysicalPropertiesAssignment, ) -> Result { - crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root) + crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root, timed) } /// `Operator::output_schema`. @@ -60,14 +61,6 @@ pub fn matches_node(input: &CurrentSeriesInput, node: &OperatorNode) -> bool { input.matches_node(node) } -/// `NodeToTimedNodeCache::timed`. -pub fn timed<'a>( - cache: &'a NodeToTimedNodeCache, - input: &Rc, -) -> Option<&'a Rc> { - cache.timed(input) -} - /// `RawDataWindow::lookback_ms`. pub fn lookback_ms(window: &RawDataWindow) -> u64 { window.lookback_ms() diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index 4b7a2713b..e5d052af8 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -9,6 +9,7 @@ use crate::ir::operator::operator_properties::{ }; use crate::ir::operator::{AggIntent, GroupKeys, Reduction, Source}; use crate::ir::physical_export::PhysicalASAPOperatorPayload as Payload; +use crate::ir::properties::timing::PhysicalPropertiesAssignment; use crate::ir::schema::{Schema, SketchStatistic, WeightDomain}; use crate::ir::{ ASAPOp, NonASAPOp, Operator, OperatorNode, OperatorResultKind, QueryRoot, ScalarExpr, @@ -435,9 +436,14 @@ fn declared_counter(data: &DataWorkload, metric: &str) -> bool { /// `roots` with every open [`WeightDomain::RawDataSamplesOfMetric`] requirement /// replaced by its answer from `data` ([`confirmed_weights`]): the form of a /// selected plan the executor runs, which reads no data workload. The same -/// `Rc` where nothing changes. -pub fn with_confirmed_weights(roots: &[QueryRoot], data: &DataWorkload) -> Vec { - with_confirmed_weights_at(roots, data, &|_| true) +/// `Rc` where nothing changes; a rebuilt node takes the physical properties +/// `physical_properties` lists for the node it replaces. +pub fn with_confirmed_weights( + roots: &[QueryRoot], + data: &DataWorkload, + physical_properties: &mut PhysicalPropertiesAssignment, +) -> Vec { + with_confirmed_weights_at(roots, data, &|_| true, physical_properties) } /// [`with_confirmed_weights`] at the summaries `confirm_at` names only: the @@ -447,19 +453,21 @@ pub fn with_confirmed_weights_at( roots: &[QueryRoot], data: &DataWorkload, confirm_at: &dyn Fn(&Rc) -> bool, + physical_properties: &mut PhysicalPropertiesAssignment, ) -> Vec { fn confirm( node: &Rc, data: &DataWorkload, confirm_at: &dyn Fn(&Rc) -> bool, cache: &mut HashMap<*const OperatorNode, Rc>, + physical_properties: &mut PhysicalPropertiesAssignment, ) -> Rc { if let Some(done) = cache.get(&Rc::as_ptr(node)) { return Rc::clone(done); } let mut changed = false; let mut operator = node.operator.map_children(|child| { - let rebuilt = confirm(child, data, confirm_at, cache); + let rebuilt = confirm(child, data, confirm_at, cache, physical_properties); changed |= !Rc::ptr_eq(&rebuilt, child); rebuilt }); @@ -474,12 +482,14 @@ pub fn with_confirmed_weights_at( } } let rebuilt = if changed { - Rc::new( + let rebuilt = Rc::new( OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()) - .with_timing(node.timing) - .with_lifetime(node.lifetime), - ) + .with_guarantee(node.guarantee.clone()), + ); + if let Some(properties) = physical_properties.get(node) { + physical_properties.set(&rebuilt, properties); + } + rebuilt } else { Rc::clone(node) }; @@ -490,10 +500,16 @@ pub fn with_confirmed_weights_at( roots .iter() .map(|root| match root { - QueryRoot::Operator(n) => QueryRoot::Operator(confirm(n, data, confirm_at, &mut cache)), - QueryRoot::Scalar(expr) => QueryRoot::Scalar( - expr.map_operator_refs(&mut |n| confirm(n, data, confirm_at, &mut cache)), - ), + QueryRoot::Operator(n) => QueryRoot::Operator(confirm( + n, + data, + confirm_at, + &mut cache, + physical_properties, + )), + QueryRoot::Scalar(expr) => QueryRoot::Scalar(expr.map_operator_refs(&mut |n| { + confirm(n, data, confirm_at, &mut cache, physical_properties) + })), }) .collect() } diff --git a/crates/types/tests/materialized_view.rs b/crates/types/tests/materialized_view.rs index 952f92da5..63dbeac49 100644 --- a/crates/types/tests/materialized_view.rs +++ b/crates/types/tests/materialized_view.rs @@ -21,8 +21,8 @@ use asap_types::ir::schema::{ SketchParams, SummaryUpdate, }; use asap_types::ir::{ - apply_physical_properties, apply_physical_properties_to_roots, ASAPOp, NodeToTimedNodeCache, - NonASAPOp, Operator, OperatorNode, PhysicalPropertiesAssignment, QueryRoot, TimeRangeKind, + apply_physical_properties, apply_physical_properties_to_roots, ASAPOp, NonASAPOp, Operator, + OperatorNode, PhysicalPropertiesAssignment, QueryRoot, TimeRangeKind, }; use asap_types::physical::PhysicalProperties; @@ -102,9 +102,8 @@ fn tumbling_windows_and_kept_merge() -> (PhysicalASAPDAG, usize) { &merge, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, ); - let timed = - apply_physical_properties(&merge, &assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = apply_physical_properties(&merge, &assignment).unwrap(); + let dag = compile_physical_asap_dag(&merge, &timed).unwrap(); let merge_id = dag.roots[0]; (dag, merge_id) } @@ -246,9 +245,10 @@ fn three_chains_over_one_scan() -> (PhysicalASAPDAG, PhysicalASAPDAG) { }) .collect(); let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let unexpanded = compile_physical_asap_workload(&timed).unwrap(); - let expanded = expand_tumbling_window_chains_in_workload(&timed).unwrap(); - let mut installed = compile_physical_asap_workload(&expanded.roots).unwrap(); + let unexpanded = compile_physical_asap_workload(&roots, &timed).unwrap(); + let expanded = expand_tumbling_window_chains_in_workload(&roots, &timed).unwrap(); + let mut installed = + compile_physical_asap_workload(&expanded.roots, &expanded.physical_properties).unwrap(); installed .set_retention(&HashMap::new(), &HashMap::new()) .unwrap(); @@ -379,7 +379,7 @@ fn a_shifted_tumbling_window_has_its_own_phase() { .map(|tumbling_window| QueryRoot::Operator(Rc::clone(tumbling_window))) .collect(); let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); + let mut dag = compile_physical_asap_workload(&roots, &timed).unwrap(); dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); let mut windows: Vec<(u64, u64)> = dag .ingestion_refresh_groups() @@ -424,7 +424,7 @@ fn views_of_unbounded_windows_are_groups_of_their_own() { }) .collect(); let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); + let mut dag = compile_physical_asap_workload(&roots, &timed).unwrap(); dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); let groups = dag.ingestion_refresh_groups().unwrap(); assert_eq!(groups.len(), 2); @@ -454,10 +454,9 @@ fn a_view_reading_a_view_of_its_group_follows_it() { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); } - let timed = - apply_physical_properties_to_roots(&[QueryRoot::Operator(Rc::clone(&rollup))], &assignment) - .unwrap(); - let mut dag = compile_physical_asap_workload(&timed).unwrap(); + let roots = [QueryRoot::Operator(Rc::clone(&rollup))]; + let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); + let mut dag = compile_physical_asap_workload(&roots, &timed).unwrap(); dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); let groups = dag.ingestion_refresh_groups().unwrap(); assert_eq!(groups.len(), 1); @@ -506,9 +505,8 @@ fn a_view_has_its_refresh_window_on_a_dag_without_edges() { &view, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = - apply_physical_properties(&view, &assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = apply_physical_properties(&view, &assignment).unwrap(); + let mut dag = compile_physical_asap_dag(&view, &timed).unwrap(); let view_id = dag.roots[0]; let window = Some(RefreshWindow { width_ms: 60_000, diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index b29392575..81dccf616 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -1,4 +1,5 @@ -//! A timed plan exports as a PhysicalASAPDAG that keeps timing. +//! A plan with its nodes' physical properties exports as a PhysicalASAPDAG +//! that keeps them. use asap_types::ir::operator::{Reduction, Source}; use asap_types::ir::physical_export::{ compile_physical_asap_dag, compile_physical_asap_workload, PhysicalASAPDAG, @@ -9,8 +10,8 @@ use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, SummaryUpdate}; use asap_types::ir::{ - apply_physical_properties, ASAPOp, NodeToTimedNodeCache, NonASAPOp, Operator, OperatorNode, - PhysicalPropertiesAssignment, QueryRoot, + apply_physical_properties, apply_physical_properties_to_roots, ASAPOp, NonASAPOp, Operator, + OperatorNode, PhysicalPropertiesAssignment, QueryRoot, }; use asap_types::physical::{InvalidPhysicalProperties, Lifetime, Medium, PhysicalProperties}; use std::rc::Rc; @@ -45,23 +46,31 @@ fn plan() -> Rc { const ON_DISK: Lifetime = Lifetime::AcrossEvaluations(Medium::Disk); -/// `plan()` timed with its summary materialized at ingestion time on disk. -fn timed_at_ingestion() -> Rc { +/// `plan()` with its nodes' physical properties, its summary materialized +/// at ingestion time on disk. +fn timed_at_ingestion() -> (Rc, PhysicalPropertiesAssignment) { let root = plan(); let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set( root.children()[0], PhysicalProperties::new(ExecutionTiming::IngestionTime, ON_DISK).unwrap(), ); - apply_physical_properties(&root, &assignment, &mut NodeToTimedNodeCache::new()).unwrap() + let timed = apply_physical_properties(&root, &assignment).unwrap(); + (root, timed) +} + +/// `timed_at_ingestion()` exported. +fn exported_at_ingestion() -> PhysicalASAPDAG { + let (root, timed) = timed_at_ingestion(); + compile_physical_asap_dag(&root, &timed).unwrap() } /// A summary materialized at ingestion time is read at query time, and the /// exported DAG says where to store it. #[test] fn timed_plan_exports_with_timing() { - let timed = timed_at_ingestion(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let (root, timed) = timed_at_ingestion(); + let dag = compile_physical_asap_dag(&root, &timed).unwrap(); dag.validate().unwrap(); let timings: Vec<_> = dag.nodes.iter().map(|n| n.output_state.timing).collect(); @@ -76,7 +85,7 @@ fn timed_plan_exports_with_timing() { let lifetimes: Vec<_> = dag.nodes.iter().map(|n| n.lifetime).collect(); assert_eq!(lifetimes, vec![Lifetime::None, ON_DISK, Lifetime::None]); // Timing keeps what the summary covers. - assert!(timed.children()[0].coverage().is_some()); + assert!(root.children()[0].coverage().is_some()); let decoded: PhysicalASAPDAG = serde_json::from_str(&serde_json::to_string(&dag).unwrap()).unwrap(); @@ -86,7 +95,7 @@ fn timed_plan_exports_with_timing() { /// A query-time producer cannot feed an ingestion-time consumer. #[test] fn query_time_input_to_ingestion_is_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); + let mut dag = exported_at_ingestion(); dag.nodes[0].output_state.timing = ExecutionTiming::QueryTime; *dag.edges[0].data_state_mut() = dag.nodes[0].output_state; assert!(matches!( @@ -100,7 +109,7 @@ fn query_time_input_to_ingestion_is_rejected() { /// runs at query time. #[test] fn invalid_physical_properties_are_rejected() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); + let mut dag = exported_at_ingestion(); dag.nodes[1].lifetime = Lifetime::OneEvaluation(Medium::Disk); assert_eq!( dag.validate(), @@ -118,10 +127,10 @@ fn invalid_physical_properties_are_rejected() { dag.validate().unwrap(); } -/// Untimed plans cannot be exported as physical plans. +/// A plan whose nodes have no physical properties cannot be exported. #[test] fn untimed_plan_is_rejected() { - assert!(compile_physical_asap_dag(&plan()).is_err()); + assert!(compile_physical_asap_dag(&plan(), &PhysicalPropertiesAssignment::default()).is_err()); } /// Two queries reading one summary state export once, with one root per query. @@ -134,14 +143,14 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { child: state, })) .unwrap(); - let assignment = PhysicalPropertiesAssignment::default(); - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = [first, second] - .iter() - .map(|root| apply_physical_properties(root, &assignment, &mut cache).unwrap()) + let roots: Vec = [first, second] + .into_iter() + .map(QueryRoot::Operator) .collect(); - let roots: Vec = timed.into_iter().map(QueryRoot::Operator).collect(); - let dag = compile_physical_asap_workload(&roots).unwrap(); + let timed = + apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) + .unwrap(); + let dag = compile_physical_asap_workload(&roots, &timed).unwrap(); dag.validate().unwrap(); assert_eq!(dag.roots.len(), 2); assert_eq!(dag.nodes.len(), 4, "scan and summary are exported once"); @@ -153,22 +162,21 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { fn scalar_roots_export_in_batch_order_and_reference_nodes() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - let timed = apply_physical_properties( - &plan(), - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); + let read = plan(); + let timed = apply_physical_properties(&read, &PhysicalPropertiesAssignment::default()).unwrap(); let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, - left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&timed))), + left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&read))), right: Box::new(ScalarExpr::literal_f64(1.0)), semantics: ExprSemantics::Promql, }; - let dag = compile_physical_asap_workload(&[ - QueryRoot::Scalar(scalar), - QueryRoot::Operator(Rc::clone(&timed)), - ]) + let dag = compile_physical_asap_workload( + &[ + QueryRoot::Scalar(scalar), + QueryRoot::Operator(Rc::clone(&read)), + ], + &timed, + ) .unwrap(); assert_eq!(dag.nodes.len(), 3, "the read plan is exported once"); assert_eq!(dag.roots.len(), 1); @@ -272,13 +280,9 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { child: series("a"), })) .unwrap(); - let timed = apply_physical_properties( - &project, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = + apply_physical_properties(&project, &PhysicalPropertiesAssignment::default()).unwrap(); + let dag = compile_physical_asap_dag(&project, &timed).unwrap(); dag.validate().unwrap(); let roles: Vec<_> = dag .edges @@ -314,13 +318,8 @@ fn swapped_edge_roles_are_rejected() { right: scan("b"), })) .unwrap(); - let timed = apply_physical_properties( - &join, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); - let mut dag = compile_physical_asap_dag(&timed).unwrap(); + let timed = apply_physical_properties(&join, &PhysicalPropertiesAssignment::default()).unwrap(); + let mut dag = compile_physical_asap_dag(&join, &timed).unwrap(); dag.validate().unwrap(); for edge in &mut dag.edges { edge.role = match edge.role { @@ -341,7 +340,7 @@ fn swapped_edge_roles_are_rejected() { #[test] fn an_empty_batch_is_an_error() { assert!(matches!( - compile_physical_asap_workload(&[]), + compile_physical_asap_workload(&[], &PhysicalPropertiesAssignment::default()), Err(ExecutionDataStateError::InvalidPhysicalDAG( PhysicalASAPDAGValidationError::NoRoots )) @@ -385,19 +384,13 @@ fn exports_summary_over_summary_as_typed_precompute_edges() { child: outer, })) .unwrap(); - let mut cache = NodeToTimedNodeCache::new(); let at_ingestion = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(&root), ); - let timed = apply_physical_properties(&root, &at_ingestion, &mut cache).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); - assert_eq!(compiled.node_ids.node_id(&timed), Some(3)); - assert_eq!( - compiled - .node_ids - .node_id(asap_types::test_support::timed(&cache, &inner).unwrap()), - Some(1) - ); + let timed = apply_physical_properties(&root, &at_ingestion).unwrap(); + let compiled = compile_physical_asap_dag_with_node_ids(&root, &timed).unwrap(); + assert_eq!(compiled.node_ids.node_id(&root), Some(3)); + assert_eq!(compiled.node_ids.node_id(&inner), Some(1)); let dag = compiled.dag; assert_eq!(dag.roots, vec![3]); assert_eq!( @@ -449,7 +442,7 @@ fn exports_summary_over_summary_as_typed_precompute_edges() { /// neither a view nor a query-time scan (here an ingestion-time node). #[test] fn retention_is_only_on_materialized_views_and_query_time_scans() { - let mut dag = compile_physical_asap_dag(&timed_at_ingestion()).unwrap(); + let mut dag = exported_at_ingestion(); let view = dag .nodes .iter() @@ -498,30 +491,31 @@ fn physical_node_ids_serialize_in_deterministic_binding_order() { fn every_node_at_query_time_keeps_the_operators_and_stores_nothing() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - let workload = |root: Rc| { + let workload = |root: Rc, timed: &PhysicalPropertiesAssignment| { let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&root))), right: Box::new(ScalarExpr::literal_f64(1.0)), semantics: ExprSemantics::Promql, }; - compile_physical_asap_workload(&[ - QueryRoot::Operator(Rc::clone(&root)), - QueryRoot::Scalar(scalar), - QueryRoot::Operator(root), - ]) + compile_physical_asap_workload( + &[ + QueryRoot::Operator(Rc::clone(&root)), + QueryRoot::Scalar(scalar), + QueryRoot::Operator(root), + ], + timed, + ) .unwrap() }; - let materialized = workload(timed_at_ingestion()); - let at_query_time = apply_physical_properties( - &plan(), - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .unwrap(); + let (root, timed) = timed_at_ingestion(); + let materialized = workload(root, &timed); + let at_query_time = plan(); + let timed = apply_physical_properties(&at_query_time, &PhysicalPropertiesAssignment::default()) + .unwrap(); assert_eq!( materialized.with_every_node_at_query_time().unwrap(), - workload(at_query_time) + workload(at_query_time, &timed) ); } @@ -569,17 +563,11 @@ fn exported_with_scan( asap_types::ir::physical_export::PhysicalASAPDAG, asap_types::ir::physical_export::PhysicalASAPNodeId, ) { - // One timing cache, so the roots keep sharing their `Scan`. - let mut cache = NodeToTimedNodeCache::new(); - let timed: Vec<_> = roots - .into_iter() - .map(|root| { - apply_physical_properties(&root, &PhysicalPropertiesAssignment::default(), &mut cache) - .unwrap() - }) - .map(QueryRoot::Operator) - .collect(); - let dag = compile_physical_asap_workload(&timed).unwrap(); + let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); + let timed = + apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) + .unwrap(); + let dag = compile_physical_asap_workload(&roots, &timed).unwrap(); let scans: Vec<_> = dag .nodes .iter() @@ -767,10 +755,13 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { #[test] fn workload_export_matches_the_single_root_export() { use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let timed = timed_at_ingestion(); - let roots = [QueryRoot::Operator(Rc::clone(&timed))]; - let exported = compile_physical_asap_workload_with_node_ids(&roots).unwrap(); - assert_eq!(exported.dag, compile_physical_asap_dag(&timed).unwrap()); + let (root, timed) = timed_at_ingestion(); + let roots = [QueryRoot::Operator(Rc::clone(&root))]; + let exported = compile_physical_asap_workload_with_node_ids(&roots, &timed).unwrap(); + assert_eq!( + exported.dag, + compile_physical_asap_dag(&root, &timed).unwrap() + ); let untimed = [QueryRoot::Operator(plan())]; - assert!(compile_physical_asap_workload_with_node_ids(&untimed).is_err()); + assert!(compile_physical_asap_workload_with_node_ids(&untimed, &timed).is_err()); } diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index f9bc36032..b48dd0217 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,5 +1,5 @@ use asap_types::ir::operator::{AggIntent, Reduction, Source}; -use asap_types::ir::properties::{AccuracyGuarantee, ExecutionTiming}; +use asap_types::ir::properties::AccuracyGuarantee; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; use asap_types::ir::schema::{ExactKind, ExactParams, GroupingStrategy, SummaryUpdate}; @@ -67,8 +67,7 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { schema.fields[0].name = "alias".into(); schema.fields[0].table = Some("result".into()); let renamed = OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))) - .with_timing(Some(ExecutionTiming::QueryTime)); + .with_guarantee(Some(AccuracyGuarantee::exact("fixture"))); let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); @@ -80,7 +79,6 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { Some(&DataType::Utf8) ); assert!(rebuilt.guarantee.is_none()); - assert!(rebuilt.timing.is_none()); rebuilt.validate_structure().unwrap(); } } @@ -258,17 +256,11 @@ fn current_series_topk() -> Rc { /// Time `root` under the default physical properties assignment, which runs the /// data-state and population-contract validation, and export it. fn compile_physical_asap_dag(root: &Rc) -> Result<(), String> { - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, PhysicalPropertiesAssignment, - }; + use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment}; root.validate_structure().map_err(|e| e.to_string())?; - let timed = apply_physical_properties( - root, - &PhysicalPropertiesAssignment::default(), - &mut NodeToTimedNodeCache::new(), - ) - .map_err(|e| format!("{e:?}"))?; - asap_types::ir::physical_export::compile_physical_asap_dag(&timed) + let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) + .map_err(|e| format!("{e:?}"))?; + asap_types::ir::physical_export::compile_physical_asap_dag(root, &timed) .map_err(|e| format!("{e:?}"))?; Ok(()) } @@ -339,9 +331,7 @@ fn a_current_series_population_matches_only_its_selectors_lookback() { fn population_timing_follows_its_physical_properties() { use asap_types::ir::physical_export::compile_physical_asap_dag as export_timed; use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, PhysicalPropertiesAssignment, - }; + use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment}; use asap_types::physical::{Lifetime, Medium, PhysicalProperties}; let candidate = current_series_topk(); let Some(ASAPOp::EvaluatePopulation { @@ -364,10 +354,8 @@ fn population_timing_follows_its_physical_properties() { ), (at_ingestion, IngestionTime, on_disk), ] { - let timed = - apply_physical_properties(&candidate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let dag = export_timed(&timed).unwrap(); + let timed = apply_physical_properties(&candidate, &assignment).unwrap(); + let dag = export_timed(&candidate, &timed).unwrap(); let [population, evaluation] = &dag.nodes[dag.nodes.len() - 2..] else { unreachable!() }; @@ -375,9 +363,12 @@ fn population_timing_follows_its_physical_properties() { assert_eq!(population.lifetime, lifetime); assert_eq!(evaluation.output_state.timing, QueryTime); } - let mut evaluation_at_ingestion = (*candidate).clone(); - evaluation_at_ingestion.timing = Some(IngestionTime); - assert!(compile_physical_asap_dag(&Rc::new(evaluation_at_ingestion)).is_err()); + let mut evaluation_at_ingestion = PhysicalPropertiesAssignment::default(); + evaluation_at_ingestion.set( + &candidate, + PhysicalProperties::NOT_MATERIALIZED_AT_INGESTION, + ); + assert!(apply_physical_properties(&candidate, &evaluation_at_ingestion).is_err()); } /// An evaluation cannot exceed its population's contract (`max_k`), and a diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index 0c16614cf..e7a93c099 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -141,36 +141,6 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { assert!(validate_asap_inputs(&ASAPOp::FinalizeExactAccumulator { child: state }).is_err()); } -/// Phase validation checks dependencies, without declaring a computation query-only. -#[test] -fn execution_timing_checks_edges_not_function_names() { - use asap_types::ir::properties::ExecutionTiming::{IngestionTime, QueryTime}; - use asap_types::ir::ProjectItem; - let input = Rc::new((*scan()).clone().with_timing(Some(IngestionTime))); - let mut project = OperatorNode::new(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - child: input, - cols: vec![ProjectItem { - expr: ScalarExpr::FunctionCall { - name: "promql_abs".into(), - args: vec![ScalarExpr::Column(0)], - }, - alias: None, - }], - qualifier: None, - })) - .unwrap() - .with_timing(Some(IngestionTime)); - Rc::new(project.clone()) - .validate_execution_timing() - .unwrap(); - if let asap_types::ir::Operator::NonASAP(NonASAPOp::Project { child, .. }) = - &mut project.operator - { - *child = Rc::new(child.as_ref().clone().with_timing(Some(QueryTime))); - } - assert!(Rc::new(project).validate_execution_timing().is_err()); -} - /// Both operator categories use the same fallible schema-deriving constructor. #[test] fn shared_construction_derives_both_operator_categories() { @@ -195,7 +165,6 @@ fn shared_construction_derives_both_operator_categories() { assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); assert!(state.guarantee.is_none()); - assert!(state.timing.is_none()); state.validate_structure().unwrap(); assert!( diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs index 07357264b..cde9c88b7 100644 --- a/crates/types/tests/summary_coverage.rs +++ b/crates/types/tests/summary_coverage.rs @@ -10,7 +10,6 @@ use asap_types::ir::operator::TimeShift; use asap_types::ir::properties::summary_coverage::{ ColumnIdentity, Constraint, CoverageError, SelectionBox, }; -use asap_types::ir::properties::ExecutionTiming; use asap_types::ir::scalar::expr_ir::{ArithmeticOpKind, CompareOpKind, ScalarValue}; use asap_types::ir::scalar::ColumnRef; use asap_types::ir::schema::{DataType, Field, FieldDataType, Schema}; @@ -554,22 +553,6 @@ fn merges_nest() { ); } -/// When a state is computed is not what it computes. -#[test] -fn timing_does_not_block_merges() { - let ingested = Rc::new( - (*tumbling_window(60_000, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)), - ); - let queried = Rc::new( - (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::QueryTime)), - ); - merge(vec![ingested, queried]).unwrap(); -} - /// A merge built around `new` is still checked by `validate_structure`. #[test] fn forged_merges_fail_validation() { @@ -805,11 +788,9 @@ fn equal_definitions_get_equal_ids() { let us = latency_where(eq(REGION, "us")); let eu = latency_where(eq(REGION, "eu")); assert_eq!(id(&us), id(&eu)); - let ingested = (*tumbling_window(0, vec![])) - .clone() - .with_timing(Some(ExecutionTiming::IngestionTime)); - assert_eq!(id(&ingested), id(&tumbling_window(60_000, vec![]))); - assert_eq!(id(&ingested).len(), 32); + let newest = tumbling_window(0, vec![]); + assert_eq!(id(&newest), id(&tumbling_window(60_000, vec![]))); + assert_eq!(id(&newest).len(), 32); let size = kll(filter(table(), eq(REGION, "eu")), "size"); assert_ne!(id(&size), id(&eu)); assert_ne!( diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index 931414247..02f72cafe 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -10,8 +10,8 @@ use asap_types::ir::schema::{ StatisticKind, SummaryUpdate, WaveletKind, WaveletParams, }; use asap_types::ir::{ - apply_physical_properties, ASAPOp, ExprSemantics, NodeToTimedNodeCache, NonASAPOp, Operator, - OperatorNode, PhysicalPropertiesAssignment, Predicate, ScalarExpr, + apply_physical_properties, ASAPOp, ExprSemantics, NonASAPOp, Operator, OperatorNode, + PhysicalPropertiesAssignment, Predicate, ScalarExpr, }; use asap_types::physical::PhysicalProperties; use std::rc::Rc; @@ -703,14 +703,13 @@ fn summary_merge_timing() { let tumbling_windows = [state(200, "us"), state(200, "eu")]; let root = merge(tumbling_windows.to_vec()).unwrap(); let timings = |assignment: &PhysicalPropertiesAssignment| { - let timed = - apply_physical_properties(&root, assignment, &mut NodeToTimedNodeCache::new()).unwrap(); - let children = timed.children(); + let timed = apply_physical_properties(&root, assignment).unwrap(); + let timing = |node: &OperatorNode| timed.get(node).unwrap().timing(); ( - timed.timing.unwrap(), - children + timing(&root), + root.children() .iter() - .map(|c| c.timing.unwrap()) + .map(|c| timing(c)) .collect::>(), ) }; @@ -732,9 +731,7 @@ fn summary_merge_timing() { // stored at ingestion time needs its tumbling windows there too. let mut only_merge = PhysicalPropertiesAssignment::default(); only_merge.set(&root, at_ingestion); - assert!( - apply_physical_properties(&root, &only_merge, &mut NodeToTimedNodeCache::new()).is_err() - ); + assert!(apply_physical_properties(&root, &only_merge).is_err()); let mut stored_merge = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(&root), ); diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 8585d7022..14ec6ad5b 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -181,10 +181,9 @@ fn expansion_merges_the_tumbling_windows() { /// An not_materialized chain expands into query-time tumbling windows. #[test] fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { + use asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload; use asap_types::ir::properties::ExecutionTiming; - use asap_types::ir::{ - apply_physical_properties, NodeToTimedNodeCache, PhysicalPropertiesAssignment, - }; + use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment, QueryRoot}; use asap_types::physical::{Lifetime, Medium, PhysicalProperties}; let materialized_across = Lifetime::AcrossEvaluations(Medium::Memory); for (lifetime, timing) in [ @@ -201,19 +200,30 @@ fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { if lifetime.is_materialized() { assignment.set(&chain, PhysicalProperties::new(timing, lifetime).unwrap()); } - let timed = - apply_physical_properties(&estimate, &assignment, &mut NodeToTimedNodeCache::new()) - .unwrap(); - let expanded = expand_tumbling_window_chains(&timed).unwrap(); + let timed = apply_physical_properties(&estimate, &assignment).unwrap(); + let expansion = expand_tumbling_window_chains_in_workload( + &[QueryRoot::Operator(Rc::clone(&estimate))], + &timed, + ) + .unwrap(); + let properties = |node: &OperatorNode| expansion.physical_properties.get(node).unwrap(); + let QueryRoot::Operator(expanded) = &expansion.roots[0] else { + unreachable!() + }; let merged = expanded.children()[0]; - assert_eq!(merged.timing, Some(ExecutionTiming::QueryTime)); + assert_eq!(properties(merged).timing(), ExecutionTiming::QueryTime); assert_eq!(merged.children().len(), 5); for tumbling_window in merged.children() { - assert_eq!(tumbling_window.timing, Some(timing), "{lifetime:?}"); - assert_eq!(tumbling_window.lifetime, lifetime); + assert_eq!(properties(tumbling_window).timing(), timing, "{lifetime:?}"); + assert_eq!(properties(tumbling_window).lifetime(), lifetime); for below in OperatorNode::reachable(tumbling_window).iter().skip(1) { - assert_eq!(below.timing, Some(timing), "{}", below.operator.kind_name()); - assert_eq!(below.lifetime, Lifetime::None); + assert_eq!( + properties(below).timing(), + timing, + "{}", + below.operator.kind_name() + ); + assert_eq!(properties(below).lifetime(), Lifetime::None); } } } @@ -258,7 +268,7 @@ fn merges_over_one_chain_share_its_tumbling_windows() { QueryRoot::Operator(merge(vec![Rc::clone(&shared)])), QueryRoot::Operator(merge(vec![shared])), ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); + let expansion = expand_tumbling_window_chains_in_workload(&roots, &Default::default()).unwrap(); assert_eq!(expansion.chains.len(), 1); assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); let expanded = expansion.roots; @@ -338,7 +348,7 @@ fn a_merge_of_the_newest_tumbling_windows_reads_the_first_tumbling_windows_of_th QueryRoot::Operator(newest(Rc::clone(&shared), 2)), QueryRoot::Operator(merge(vec![shared])), ]; - let expansion = expand_tumbling_window_chains_in_workload(&roots).unwrap(); + let expansion = expand_tumbling_window_chains_in_workload(&roots, &Default::default()).unwrap(); assert_eq!(expansion.chains.len(), 1); assert_eq!(expansion.chains[0].tumbling_windows.len(), 5); let tumbling_windows = |root: &QueryRoot| match root { @@ -456,7 +466,7 @@ fn expansion_preserves_every_tumbling_window_and_shared_reader() { QueryRoot::Operator(merge(vec![shared.clone()])), QueryRoot::Operator(merge(vec![shared])), ]; - let result = expand_tumbling_window_chains_in_workload(&roots).unwrap(); + let result = expand_tumbling_window_chains_in_workload(&roots, &Default::default()).unwrap(); assert_eq!(result.chains.len(), 1); assert_eq!(result.chains[0].tumbling_windows.len(), 5); let QueryRoot::Operator(first) = &result.roots[0] else { diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index fa9beae81..c8bb69139 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -122,13 +122,15 @@ for `OneEvaluation`. A per sub-DAG, the properties of its materialized output, named by the node whose output would be materialized. Any node may be named: a summary state, a population, or, as in #509 Example 4, the tumbling windows a query reads. `set(node, properties)` -materializes one; every other sub-DAG is not materialized. -Stage 2 chooses the assignment. -`apply_physical_properties(root, &assignment, &mut NodeToTimedNodeCache)` then writes a -timing and a lifetime into every node, top-down: +lists one; an entry whose lifetime is `None` fixes only the node's timing, and every +sub-DAG the assignment does not list is not materialized. Stage 2 chooses the +assignment. Operator nodes carry no timing or lifetime (#1200): the assignment is +where physical stages keep them. +`apply_physical_properties(root, &assignment)` then completes it into a timing and a +lifetime for every node, top-down, and returns the completed assignment: - `SummaryEstimate` and `EvaluatePopulation` run at query time; -- a node assigned a materialized output runs at that output's timing; a `SummaryAgg` +- a node the assignment lists runs at its entry's timing; a `SummaryAgg` or `MaintainPopulation` that is not materialized runs at query time. Every node upstream of an ingestion-time node also runs at ingestion time, so a materialized ingestion-time output over something that can only exist at query time (an @@ -140,18 +142,15 @@ timing and a lifetime into every node, top-down: an evaluation at query time. A query-time consumer reads the stored output of a node materialized at ingestion time (`AcrossEvaluations`). -A node whose timing a candidate builder fixed (the query-time -`FinalizeExactAccumulator` read boundary of `finalize_query_candidate`) keeps -it: an explicit assignment that agrees with that timing is kept, and one that -needs the other timing fails with -`ExecutionDataStateError::PhysicalPropertiesConflictWithFixedTiming`; an -explicit materialized output is never silently moved or dropped. A root may be materialized +Applying a completed assignment again gives every node the same properties: the +facade re-applies Stage 3's selection this way for export +(`PlanOutput::physical_properties`). A root may be materialized at ingestion time, and the query reads its stored output; a root of rows that runs at ingestion time without being stored there fails with `ExecutionDataStateError::UnstoredIngestionRoot`. The pass then validates every edge (rows or exact-accumulator state into a -`SummaryAgg`, state into an evaluation, a timed `MaintainPopulation` under +`SummaryAgg`, state into an evaluation, a `MaintainPopulation` under an `EvaluatePopulation`, no ingestion work reading a query-time value) and rejects a node reached from two consumers that need different timings; `split_shared_by_timing` copies such a sub-DAG for one side before the @@ -192,8 +191,9 @@ children as edges and no embedded sub-DAGs: expressions (`scalar(v)`). Every edge records the intermediate schema, the producer's data state and grouping/window compatibility. - Each node records `output_state` (timing plus `Raw` or `SummaryState`), - `output_schema` and `guarantee`. Export reads the timing written by - `apply_physical_properties` and rejects an untimed node + `output_schema` and `guarantee`. Export reads each node's timing and lifetime + from the assignment `apply_physical_properties` completed and rejects a node + it does not list (`ExecutionDataStateError::UntimedNode`); it does not re-run data-state validation. diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 020d2961b..122eae3b8 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -360,8 +360,8 @@ optional also does not guarantee every planning operation can succeed without it | Function/type | Purpose | | --- | --- | -| `asap_types::ir::apply_physical_properties(&root, &assignment, &mut NodeToTimedNodeCache::new())` | Write execution timing into every node from a `PhysicalPropertiesAssignment` (default: all query time) and validate the data-state edges; `PlanOutput::physical_asap_dag()` applies the default to a planned workload | -| `asap_types::ir::physical_export::compile_physical_asap_dag(&timed_root)` | Export a timed DAG as a `PhysicalASAPDAG`; rejects an untimed node; `compile_physical_asap_workload(&roots)` exports a batch with one root per query; `.validate()` checks it explicitly | +| `asap_types::ir::apply_physical_properties(&root, &assignment)` | Complete a `PhysicalPropertiesAssignment` (default: all query time) into the physical properties of every node and validate the data-state edges; `apply_physical_properties_to_roots` does it for a workload; `PlanOutput::physical_asap_dag()` applies the pass's `physical_properties` to a planned workload | +| `asap_types::ir::physical_export::compile_physical_asap_dag(&root, &timed)` | Export a DAG as a `PhysicalASAPDAG`, each node with its properties from the completed assignment `timed`; rejects a node it does not list; `compile_physical_asap_workload(&roots, &timed)` exports a batch with one root per query; `.validate()` checks it explicitly | | `PhysicalASAPDAG::materialized_views()` and `.raw_data_retention()` (`asap_types::ir::materialized_view`) | What a runtime keeps for an installed DAG, derived from the DAG and the `retention_ms` installation sets: each materialized view (refresh mode, coverage, inputs, the node's `retention_ms`) and how far back each raw-data source is kept for query-time scans (each query-time `Scan`'s `retention_ms`) | Choose the export matching your intended handoff: an untimed semantic DAG is not diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 5f2f5b10e..9a2d23dc5 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -33,8 +33,6 @@ pub struct OperatorNode { pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar pub schema: Schema, // output schema, derived at construction pub guarantee: Option, // None until accuracy assessment establishes one - pub timing: Option, // None until a physical properties assignment is applied - pub lifetime: Lifetime, // None until Stage 2 materializes the node's output } ``` @@ -47,12 +45,11 @@ pub struct OperatorNode { derived (a column reference out of range, a reserved ASAP operator). ASAP planning may retain a more specific schema through `OperatorNode::with_schema`. - `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. -- `timing` is `None` in every front-end DAG and every candidate. It is written by - `ir::timing::apply_physical_properties` (see the Post-ASAP IR document); export rejects an - untimed node. -- `lifetime` is `Lifetime::None` in every front-end DAG. Stage 2 writes it, with - `timing`, through `ir::timing::apply_physical_properties` from its materialization - assignment; together they are the node's `PhysicalProperties`. +- A node has no physical properties (timing, lifetime): a logical candidate has only its + logical properties (#1200). Physical stages keep each node's `PhysicalProperties` in a + `PhysicalPropertiesAssignment` keyed by node; `ir::timing::apply_physical_properties` + completes Stage 2's assignment over every node (see the Post-ASAP IR document), and + export reads it. `OperatorNode::children()` returns the operator's inputs in field order followed by the operator nodes its scalar expressions read (see "Scalar expressions"). Every DAG traversal — diff --git a/tools/dag-viewer/stages.js b/tools/dag-viewer/stages.js index fa9e12a15..d30b2c307 100644 --- a/tools/dag-viewer/stages.js +++ b/tools/dag-viewer/stages.js @@ -241,8 +241,7 @@ function stageDag(dag, physical) { const [kind, body] = stageOperatorParts(node.operator); stageLogicalChildren(body).forEach((producer) => edges.push({ producer, consumer: id, raw: null })); return { - id, kind, body, schema: node.schema, guarantee: node.guarantee, - lifetime: node.lifetime, timing: node.timing === null ? undefined : node.timing, raw: node, + id, kind, body, schema: node.schema, guarantee: node.guarantee, raw: node, }; }); const queries = dag.roots.map((root) => ('Scalar' in root ? { scalar: root.Scalar } : { node: root.Operator })); diff --git a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json index c14b9d9ce..a5bcc3476 100644 --- a/tools/dag-viewer/test-fixtures/memo-enumeration-small.json +++ b/tools/dag-viewer/test-fixtures/memo-enumeration-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"TimeRange":{"child":6,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":9,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":12}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Aggregate":{"child":13,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":15}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":16,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":7,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":19,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[0],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[1],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[2],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"rate per series","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[3],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum by (job)","group_of_node":{"2":2},"node":3}],"node":3},{"id":4,"label":"Scan http_requests_total","children":[],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[4],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan http_requests_total","group_of_node":{},"node":0}],"node":0},{"id":5,"label":"TimeRange 1m","children":[4],"parents":[6],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[5],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":4},"node":1}],"node":1},{"id":6,"label":"sum_over_time per series","children":[5],"parents":[7],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[6],"children":[5],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"sum_over_time per series","group_of_node":{"1":5},"node":4}],"node":4},{"id":7,"label":"topk by (job) 10","children":[6],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":4,"candidate_count_by_origin":{"pass1":3,"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact accumulator","group_of_node":{"4":6},"node":5},{"index":1,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap","group_of_node":{"4":6},"node":5},{"index":2,"implements":[7],"children":[6],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"4":6},"node":5},{"index":3,"implements":[7],"children":[6],"origin":"shared_build","accuracy_budget_parts":0.5,"added":true,"label":"CMS+heap shared build","group_of_node":{"4":6},"node":5}],"node":5}]}],"selected_choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"enumeration":{"step":"pass1","per_step":{"stage0":{"count":1,"complete":true},"pass1":{"count":4,"complete":true}},"limit":1000,"choices":[{"id":"E1","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":3.0,"cheaper_than_selected":false,"roots":[11,14]},{"id":"E2","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":1},"accuracy_budget_parts_1":true,"selected":true,"valid":true,"rejection":null,"cost_per_second":2.0,"cheaper_than_selected":false,"roots":[11,17]},{"id":"E3","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":2},"accuracy_budget_parts_1":true,"selected":false,"valid":true,"rejection":null,"cost_per_second":1.5,"cheaper_than_selected":true,"roots":[11,20]},{"id":"E4","choice":{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":3},"accuracy_budget_parts_1":false,"selected":false,"valid":false,"rejection":"accuracy_target_missed","cost_per_second":null,"cheaper_than_selected":false,"roots":[11,17]}],"priced":{"count":4,"cheaper_than_selected":1,"selected_cost_per_second":2.0}}}} diff --git a/tools/dag-viewer/test-fixtures/memo-small.json b/tools/dag-viewer/test-fixtures/memo-small.json index 9c32559d4..2b8af0eb1 100644 --- a/tools/dag-viewer/test-fixtures/memo-small.json +++ b/tools/dag-viewer/test-fixtures/memo-small.json @@ -1 +1 @@ -{"memo":{"nodes":[{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"lifetime":"None","operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null},{"guarantee":null,"lifetime":"None","operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"timing":null}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} +{"memo":{"nodes":[{"guarantee":null,"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"TimeRange":{"child":0,"kind":"range","range":{"nanos":0,"secs":60}}}},"result_kind":"range_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"kind":"rate"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":2,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":1,"filters":[],"having":null,"measures":[{"col":null,"kind":"sum"}],"output_names":[""],"reduction":"PerEntity"}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"NonASAP":{"Aggregate":{"child":4,"filters":[],"having":null,"measures":[{"accuracy":{"accuracy_scope":"whole_result","failure_probability":0.001,"max_error":0.01},"k":10,"kind":"top_k"}],"output_names":[],"reduction":{"Reduce":[2]}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"topk_10","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":6}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":2,"family":{"ExactAggregate":["Sum","Sum"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"ExactAggregate":["Sum","Sum"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":8}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"sum","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CmsWithHeap","category":"TopK","params":{"CmsWithHeap":{"depth":7,"heap_size":100,"width":272}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":10}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":11,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"ASAP":{"SummaryAgg":{"child":1,"family":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":{"Column":{"Named":"$promql_series_identity"}},"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"samples_of_metric","metric":"http_requests_total"}},"reduction":{"Reduce":[2]}}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Sketch":[{"algorithm":"CountSketchWithHeap","category":"TopK","params":{"CountSketchWithHeap":{"depth":81,"heap_size":100,"width":30000}}},"PerSubpopulationInstance"]},"name":"state","nullable":false,"table":null}],"time_index":null,"unique_keys":[[0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryEstimate":{"query":{"TopK":{"k":10}},"summary_input":13}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"topk","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":null,"operator":{"NonASAP":{"Project":{"child":14,"cols":[{"alias":"job","expr":{"Column":0}},{"alias":"topk_10","expr":{"Column":2}},{"alias":"$promql_series_identity","expr":{"Column":1}}],"qualifier":null}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"float64"},"name":"topk_10","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":null,"unique_keys":[]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact over exact inputs"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"NonASAP":{"Scan":{"predicates":[],"schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]},"source":{"TimeSeries":{"metric":"http_requests_total"}}}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":16,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryMerge":{"children":[17]}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":{"provenance":[{"kind":"exact","reason":"exact accumulator over exact input"}],"spec":{"accuracy_metric":"absolute","accuracy_scope":"per_value","error":{"op":"zero"},"failure_probability":{"op":"zero"}}},"operator":{"ASAP":{"FinalizeExactAccumulator":{"child":18}}},"result_kind":"instant_vector","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"Plain":"float64"},"name":"value","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}},{"guarantee":null,"operator":{"ASAP":{"SummaryTumblingWindowChain":{"child":0,"family":{"ExactAggregate":["Rate","Rate"]},"filter":null,"grouping":"PerSubpopulationInstance","input":{"item":null,"weight":{"Column":"SampleValue"},"weight_domain":{"kind":"unknown_or_signed"}},"tumbling_window_count":6,"tumbling_window_width_ms":10000,"reduction":"PerEntity"}}},"result_kind":"state","schema":{"closed":true,"fields":[{"dtype":{"Plain":"timestamp"},"name":"ts","nullable":false,"table":null},{"dtype":{"ExactAggregate":["Rate","Rate"]},"name":"state","nullable":false,"table":null},{"dtype":{"Plain":"utf8"},"name":"job","nullable":true,"table":null},{"dtype":{"Plain":"utf8"},"name":"$promql_series_identity","nullable":false,"table":null}],"time_index":0,"unique_keys":[[3,0]]}}],"steps":[{"step":"stage0","title":"Stage 0","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":0,"candidate_count_by_origin":{},"candidates_capped":false,"candidates":[],"node":5}]},{"step":"pass1","title":"Stage 1, Pass 1","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":2,"candidate_count_by_origin":{"pass1":2},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":true,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5}]},{"step":"pass2","title":"Stage 1, Pass 2","groups":[{"id":0,"label":"Scan http_requests_total","children":[],"parents":[1,6],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"Scan","group_of_node":{},"node":0}],"node":0},{"id":1,"label":"TimeRange 1m","children":[0],"parents":[2,4],"queries":[0,1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"TimeRange 1m","group_of_node":{"0":0},"node":1}],"node":1},{"id":2,"label":"rate per series","children":[1],"parents":[3],"queries":[0],"roots_of":[],"candidate_count":20,"candidate_count_by_origin":{"pass1":2,"tumbling_windows":9,"shared_build":9},"candidates_capped":true,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact rate accumulator","group_of_node":{"1":1},"node":7},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate rate","group_of_node":{"1":1},"node":2},{"index":2,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 1","group_of_node":{"17":6},"node":19},{"index":3,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 2","group_of_node":{"17":6},"node":19},{"index":4,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 3","group_of_node":{"17":6},"node":19},{"index":5,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 4","group_of_node":{"17":6},"node":19},{"index":6,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 5","group_of_node":{"17":6},"node":19},{"index":7,"implements":[],"children":[6],"origin":"tumbling_windows","accuracy_budget_parts":1,"added":true,"label":"rate accumulator over G6's panes, variant 6","group_of_node":{"17":6},"node":19},{"index":8,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 1","group_of_node":{"17":6},"node":19},{"index":9,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 2","group_of_node":{"17":6},"node":19},{"index":10,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 3","group_of_node":{"17":6},"node":19},{"index":11,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 4","group_of_node":{"17":6},"node":19},{"index":12,"implements":[],"children":[6],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"rate accumulator read from shared build, variant 5","group_of_node":{"17":6},"node":19}],"node":2},{"id":3,"label":"sum by (job)","children":[2],"parents":[],"queries":[0],"roots_of":[0],"candidate_count":3,"candidate_count_by_origin":{"pass1":2,"rollup":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum","group_of_node":{"2":2},"node":3},{"index":1,"implements":[],"children":[2],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact sum accumulator","group_of_node":{"2":2},"node":9},{"index":2,"implements":[],"children":[2],"origin":"rollup","accuracy_budget_parts":1,"added":true,"label":"sum by (job) rolled up from G2's accumulator","group_of_node":{"2":2},"node":9}],"node":3},{"id":4,"label":"sum_over_time per series","children":[1],"parents":[5],"queries":[1],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"pass1":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate sum_over_time","group_of_node":{"1":1},"node":4}],"node":4},{"id":5,"label":"topk 10 by (job)","children":[4],"parents":[],"queries":[1],"roots_of":[1],"candidate_count":3,"candidate_count_by_origin":{"pass1":3},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[4],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"exact Aggregate topk","group_of_node":{"4":4},"node":5},{"index":1,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CMS+heap depth 7 width 272","group_of_node":{"1":1},"node":12},{"index":2,"implements":[],"children":[1],"origin":"pass1","accuracy_budget_parts":1,"added":false,"label":"CountSketch+heap","group_of_node":{"1":1},"node":15}],"node":5},{"id":6,"label":"SummaryTumblingWindowChain 6 × 10 s","children":[0],"parents":[],"queries":[0],"roots_of":[],"candidate_count":1,"candidate_count_by_origin":{"shared_build":1},"candidates_capped":false,"candidates":[{"index":0,"implements":[],"children":[0],"origin":"shared_build","accuracy_budget_parts":1,"added":true,"label":"SummaryTumblingWindowChain 6 × 10 s, rate accumulator","group_of_node":{"0":0},"node":20}],"node":17}]}],"selected_choice":{"5":1,"3":0,"2":2,"6":0,"1":0,"0":0,"4":0}}} From 5ef723d9c48b1fa3f1575e596859da8957dfec32 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:02:20 +0000 Subject: [PATCH 2/2] refactor: review fixes for physical properties off the logical IR Docs: the Pass 1 module doc, post-asap-ir.md and asap-primitive-schema.md no longer describe OperatorNode::timing; operator-sharing.md gets a status note. The facade carries the plan to run as one SelectedPlan (roots + physical properties) and leaves the RealizedPlan as Stage 2 built it; a debug check asserts the assignment lists every node of the roots. bounds.rs and tumbling_window_chain.rs use copy_assignment. Assignment variables are named physical_properties, not timed; the integration-tests helpers are default_physical_properties and physical_properties_at_ingestion, and the logical crate's unused copies are deleted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K1M6br9PL4SDeVS1XpezKJ --- crates/accuracy/src/propagation.rs | 8 +- crates/cost/tests/ingestion_refresh_runs.rs | 4 +- crates/devtools/src/bin/asap_planner.rs | 15 +- crates/devtools/src/planner_fuzz/mod.rs | 6 +- crates/devtools/src/planning_stages.rs | 31 +- crates/executor/src/physical_planner/mod.rs | 4 +- crates/executor/tests/common/mod.rs | 8 +- crates/executor/tests/time_series_window.rs | 4 +- .../executor/tests/weighted_topk_binding.rs | 5 +- .../frontend-promql/tests/planner_support.rs | 7 +- crates/frontend-promql/tests/support.rs | 7 +- .../tests/univmon_candidates.rs | 3 +- crates/frontend-sql/tests/pearson_corr.rs | 12 +- crates/integration-tests/src/lib.rs | 24 +- .../tests/exact_composition.rs | 14 +- .../tests/frequency_execution.rs | 4 +- .../tests/kll_tumbling_window_execution.rs | 5 +- .../tests/per_series_exact_execution.rs | 5 +- .../tests/physical_common/mod.rs | 5 +- .../tests/physical_properties_e2e.rs | 4 +- .../tests/promql_to_post_asap.rs | 8 +- crates/integration-tests/tests/reuse_rules.rs | 5 +- .../tests/tumbling_window_merge.rs | 8 +- .../src/pass1/memo_population.rs | 2 +- .../src/pass1/per_computation_candidates.rs | 4 +- .../src/test_support.rs | 21 -- crates/physical-asap-optimization/src/lib.rs | 6 +- .../src/split_by_timing.rs | 14 +- crates/plan-selection/src/lib.rs | 4 +- crates/planner/src/installable.rs | 24 +- crates/planner/src/lib.rs | 2 +- crates/planner/src/pass.rs | 10 +- crates/planner/src/run.rs | 66 +++-- crates/planner/tests/e2e_plan.rs | 2 +- crates/planner/tests/runtime_install.rs | 10 +- .../src/ir/operator/tumbling_window_chain.rs | 10 +- crates/types/src/ir/physical_export.rs | 38 ++- crates/types/src/ir/properties/timing.rs | 277 ++++++++++++------ crates/types/src/physical/workload_plan.rs | 5 +- crates/types/src/test_support.rs | 4 +- crates/types/src/workload/bounds.rs | 4 +- crates/types/tests/materialized_view.rs | 26 +- crates/types/tests/physical_export.rs | 62 ++-- crates/types/tests/schema_rebuilding.rs | 11 +- crates/types/tests/summary_merge_structure.rs | 4 +- crates/types/tests/tumbling_window_chain.rs | 4 +- docs/design_docs/concepts/post-asap-ir.md | 8 +- .../proposals/asap-primitive-schema.md | 5 +- .../design_docs/proposals/operator-sharing.md | 3 + 49 files changed, 484 insertions(+), 338 deletions(-) diff --git a/crates/accuracy/src/propagation.rs b/crates/accuracy/src/propagation.rs index 2c15ba4b8..b969b641e 100644 --- a/crates/accuracy/src/propagation.rs +++ b/crates/accuracy/src/propagation.rs @@ -647,10 +647,10 @@ fn merged_tumbling_windows_guarantee(node: &OperatorNode) -> Option, entries: &[QueryWorkloadEntry], run: &PlanningS fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value { let mut stage3 = selection_json(&run.run.selection); match (&run.run.selection, &run.run.selected_plan) { - (Ok(selection), Some(roots)) => { - let (tumbling_window_retention_ms, physical_properties) = run + (Ok(selection), Some(plan)) => { + let tumbling_window_retention_ms = run .run .selected_physical_plan .as_ref() - .map(|realized| { - ( - realized.tumbling_window_retention_ms.clone(), - realized.plan.physical_properties.clone(), - ) - }) + .map(|realized| realized.tumbling_window_retention_ms.clone()) .unwrap_or_default(); stage3["tumbling_window_retention_ms"] = tumbling_window_retention_ms .iter() @@ -412,8 +407,8 @@ fn stage3_json(run: &PlanningStagesRun, entries: &[QueryWorkloadEntry]) -> Value .collect::>() .into(); match materialized_views_json( - roots, - &physical_properties, + &plan.roots, + &plan.physical_properties, selection, &tumbling_window_retention_ms, entries, diff --git a/crates/devtools/src/planner_fuzz/mod.rs b/crates/devtools/src/planner_fuzz/mod.rs index 1668399b5..51871ed31 100644 --- a/crates/devtools/src/planner_fuzz/mod.rs +++ b/crates/devtools/src/planner_fuzz/mod.rs @@ -503,7 +503,11 @@ impl Planner { selected: "selected".into(), cost_per_second, stage0_roots: roots, - logical_roots: run.run.selected_plan.clone(), + logical_roots: run + .run + .selected_plan + .as_ref() + .map(|plan| plan.roots.clone()), }) } } diff --git a/crates/devtools/src/planning_stages.rs b/crates/devtools/src/planning_stages.rs index c401d5d22..6b247923c 100644 --- a/crates/devtools/src/planning_stages.rs +++ b/crates/devtools/src/planning_stages.rs @@ -13,18 +13,16 @@ use asap_physical_asap_optimization::{ }; use asap_plan_selection::select_plan; use asap_planner::{ - run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectionError, + run_planning, PlanSearchSettings, PlanningModels, PlanningReport, PlanningRun, SelectedPlan, + SelectionError, }; use asap_types::ir::schema::schema_support::with_promql_series_identity; use asap_types::ir::schema::{DataType, Field, Schema}; use asap_types::ir::QueryRoot; use asap_types::physical::PhysicalWorkloadPlan; use asap_types::stages::{AssignmentRealizer, DagOfLogicalMemoGroups, Memo}; -use asap_types::workload::bounds::with_confirmed_weights_at; use asap_types::workload::{DataWorkload, PlanningWorkload, QueryWorkloadEntry}; use serde_json::Value; -use std::collections::HashSet; -use std::rc::Rc; use crate::memo_pricing::{reference_enumeration, ReferenceEnumeration}; @@ -109,9 +107,9 @@ impl PlanningStagesRun { .run .selected_plan .iter() - .map(|roots| ShownLogicalCandidate { + .map(|plan| ShownLogicalCandidate { id: "L1".into(), - roots: Ok(roots.clone()), + roots: Ok(plan.roots.clone()), }); let enumerated = self.reference.iter().flat_map(|reference| { reference @@ -290,8 +288,8 @@ pub fn run_search_over( validate(&roots)?; let run = run_planning(roots.clone(), entries, data, models, settings) .map_err(|e| PlanningStagesError::Pass1(e.to_string()))?; - if let Some(roots) = &run.selected_plan { - validate(roots)?; + if let Some(plan) = &run.selected_plan { + validate(&plan.roots)?; } let reference = reference_choices .and_then(|max| reference_enumeration(&run, &roots, entries, data, models, max)); @@ -431,22 +429,11 @@ pub fn run_stages_until( ) { Ok(selection) => { match realizer.realize(&physical, &selection.selected_dag) { - Ok(mut realized) => { + Ok(realized) => { // As the facade does: the plan to run carries the weight // proofs Stage 3 made. - let proven: HashSet<*const asap_types::ir::OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - let plan = with_confirmed_weights_at( - &realized.plan.roots, - data, - &|node| proven.contains(&Rc::as_ptr(node)), - &mut realized.plan.physical_properties, - ); - validate(&plan)?; + let plan = SelectedPlan::of(&realized, &selection, data); + validate(&plan.roots)?; run.selected_plan = Some(plan); run.selected_physical_plan = Some(realized); run.selection = Ok(selection); diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs index 150099f87..70d9f0428 100644 --- a/crates/executor/src/physical_planner/mod.rs +++ b/crates/executor/src/physical_planner/mod.rs @@ -1644,10 +1644,10 @@ mod compilation_tests { }) .collect(); let roots: Vec<_> = windows.into_iter().map(QueryRoot::Operator).collect(); - let timed = + let physical_properties = apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) .unwrap(); - let dag = compile_physical_asap_workload(&roots, &timed).unwrap(); + let dag = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); let roots = dag.roots.iter().map(|&id| id as NodeId).collect(); (dag, roots) } diff --git a/crates/executor/tests/common/mod.rs b/crates/executor/tests/common/mod.rs index ada7a1ef4..05ac51cdf 100644 --- a/crates/executor/tests/common/mod.rs +++ b/crates/executor/tests/common/mod.rs @@ -8,8 +8,9 @@ use std::rc::Rc; pub fn compile_physical_asap_dag( root: &Rc, ) -> Result> { - let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default())?; - Ok(planner_types::ir::physical_export::compile_physical_asap_dag(root, &timed)?) + let physical_properties = + apply_physical_properties(root, &PhysicalPropertiesAssignment::default())?; + Ok(planner_types::ir::physical_export::compile_physical_asap_dag(root, &physical_properties)?) } /// The logical DAG the planner pipeline (`asap_planner::run_planning`) selects for the @@ -22,7 +23,8 @@ pub fn selected_dag( let run = run_all_stages(root, accuracy); let [QueryRoot::Operator(selected)] = run .selected_plan - .as_deref() + .as_ref() + .map(|plan| plan.roots.as_slice()) .unwrap_or_else(|| panic!("a plan is selected: {:?}", run.selection)) else { panic!("one operator root"); diff --git a/crates/executor/tests/time_series_window.rs b/crates/executor/tests/time_series_window.rs index 6f91223ef..9f303287e 100644 --- a/crates/executor/tests/time_series_window.rs +++ b/crates/executor/tests/time_series_window.rs @@ -84,9 +84,9 @@ fn window_over_rows( child, })) .unwrap(); - let timed = + let physical_properties = apply_physical_properties(&range, &PhysicalPropertiesAssignment::default()).unwrap(); - compile_physical_asap_dag(&range, &timed).unwrap() + compile_physical_asap_dag(&range, &physical_properties).unwrap() } /// `(instance, ts, value)` samples of `latency_ms` as a store's raw data. diff --git a/crates/executor/tests/weighted_topk_binding.rs b/crates/executor/tests/weighted_topk_binding.rs index 0dc3bc6d1..0264d7b6d 100644 --- a/crates/executor/tests/weighted_topk_binding.rs +++ b/crates/executor/tests/weighted_topk_binding.rs @@ -288,6 +288,7 @@ fn continuously_maintained_dag(candidate: &Rc) ); } } - let timed = apply_physical_properties(candidate, &assignment).unwrap(); - planner_types::ir::physical_export::compile_physical_asap_dag(candidate, &timed).unwrap() + let physical_properties = apply_physical_properties(candidate, &assignment).unwrap(); + planner_types::ir::physical_export::compile_physical_asap_dag(candidate, &physical_properties) + .unwrap() } diff --git a/crates/frontend-promql/tests/planner_support.rs b/crates/frontend-promql/tests/planner_support.rs index 39f00e0ca..778c99641 100644 --- a/crates/frontend-promql/tests/planner_support.rs +++ b/crates/frontend-promql/tests/planner_support.rs @@ -37,7 +37,12 @@ pub fn run_all_stages( /// at `accuracy`. pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { let run = run_all_stages(root, accuracy); - match run.selected_plan.as_deref().expect("a plan is selected") { + match run + .selected_plan + .as_ref() + .map(|plan| plan.roots.as_slice()) + .expect("a plan is selected") + { [asap_types::ir::QueryRoot::Operator(node)] => Rc::clone(node), other => panic!("one operator root: {other:?}"), } diff --git a/crates/frontend-promql/tests/support.rs b/crates/frontend-promql/tests/support.rs index 53a5d8d8c..b0c85e767 100644 --- a/crates/frontend-promql/tests/support.rs +++ b/crates/frontend-promql/tests/support.rs @@ -84,9 +84,10 @@ pub fn physical_asap_dag( root: &Rc, ) -> asap_types::ir::physical_export::PhysicalASAPDAG { use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment}; - let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) - .expect("default physical properties"); - asap_types::ir::physical_export::compile_physical_asap_dag(root, &timed) + let physical_properties = + apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) + .expect("default physical properties"); + asap_types::ir::physical_export::compile_physical_asap_dag(root, &physical_properties) .expect("post-ASAP DAG export") } diff --git a/crates/frontend-promql/tests/univmon_candidates.rs b/crates/frontend-promql/tests/univmon_candidates.rs index de94f5e2c..0d4305f04 100644 --- a/crates/frontend-promql/tests/univmon_candidates.rs +++ b/crates/frontend-promql/tests/univmon_candidates.rs @@ -141,7 +141,8 @@ fn uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets() { assert!(!unknown.is_empty(), "{query}"); let [asap_types::ir::QueryRoot::Operator(selected)] = run .selected_plan - .as_deref() + .as_ref() + .map(|plan| plan.roots.as_slice()) .expect("a certified plan is selected") else { panic!("one operator result"); diff --git a/crates/frontend-sql/tests/pearson_corr.rs b/crates/frontend-sql/tests/pearson_corr.rs index 8e6648bdd..3fb630c58 100644 --- a/crates/frontend-sql/tests/pearson_corr.rs +++ b/crates/frontend-sql/tests/pearson_corr.rs @@ -171,7 +171,12 @@ async fn corr_survives_exact_plan_compilation() { asap_planner::PlanSearchSettings::default(), ) .unwrap(); - let [asap_types::ir::QueryRoot::Operator(plan)] = run.selected_plan.as_deref().unwrap() else { + let [asap_types::ir::QueryRoot::Operator(plan)] = run + .selected_plan + .as_ref() + .map(|plan| plan.roots.as_slice()) + .unwrap() + else { panic!("one operator root"); }; assert!(plan.guarantee.as_ref().unwrap().is_exact()); @@ -179,6 +184,7 @@ async fn corr_survives_exact_plan_compilation() { // operator DAG, no ASAP node added. assert!(!plan.contains_asap(), "expected the exact plan"); assert_eq!(aggregate(plan).0, aggregate(&query).0); - let timed = apply_physical_properties(plan, &PhysicalPropertiesAssignment::default()).unwrap(); - compile_physical_asap_dag(plan, &timed).unwrap(); + let physical_properties = + apply_physical_properties(plan, &PhysicalPropertiesAssignment::default()).unwrap(); + compile_physical_asap_dag(plan, &physical_properties).unwrap(); } diff --git a/crates/integration-tests/src/lib.rs b/crates/integration-tests/src/lib.rs index 3c735bb8b..e8079bc70 100644 --- a/crates/integration-tests/src/lib.rs +++ b/crates/integration-tests/src/lib.rs @@ -104,14 +104,16 @@ pub mod physical_export { /// The physical properties of every node of `root` under the default /// assignment (no sub-DAG materialized, so every summary computed at /// query time). - pub fn timed(root: &Rc) -> PhysicalPropertiesAssignment { - timed_with(root, &PhysicalPropertiesAssignment::default()) + pub fn default_physical_properties(root: &Rc) -> PhysicalPropertiesAssignment { + completed_physical_properties(root, &PhysicalPropertiesAssignment::default()) } /// The physical properties of every node of `root` with every sub-DAG /// materialized at ingestion time where the DAG allows it. - pub fn at_ingestion(root: &Rc) -> PhysicalPropertiesAssignment { - timed_with( + pub fn physical_properties_at_ingestion( + root: &Rc, + ) -> PhysicalPropertiesAssignment { + completed_physical_properties( root, &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(root), @@ -119,7 +121,7 @@ pub mod physical_export { ) } - fn timed_with( + fn completed_physical_properties( root: &Rc, assignment: &PhysicalPropertiesAssignment, ) -> PhysicalPropertiesAssignment { @@ -128,13 +130,14 @@ pub mod physical_export { /// Time `root` (default assignment), then export the physical DAG. pub fn physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(root, &timed(root)).expect("physical ASAP DAG export failed") + compile_physical_asap_dag(root, &default_physical_properties(root)) + .expect("physical ASAP DAG export failed") } /// Time `root` with every sub-DAG materialized at ingestion time where the DAG allows /// it, then export the physical DAG. pub fn at_ingestion_physical_asap_dag(root: &Rc) -> PhysicalASAPDAG { - compile_physical_asap_dag(root, &at_ingestion(root)) + compile_physical_asap_dag(root, &physical_properties_at_ingestion(root)) .expect("physical ASAP DAG export failed") } } @@ -220,7 +223,12 @@ pub mod planning { /// The logical DAG the planner pipeline selects for the single query /// `root` at `accuracy`. pub fn selected_dag(root: Rc, accuracy: AccuracyTarget) -> Rc { - match run_all_stages(root, accuracy).run.selected_plan.as_deref() { + match run_all_stages(root, accuracy) + .run + .selected_plan + .as_ref() + .map(|plan| plan.roots.as_slice()) + { Some([QueryRoot::Operator(node)]) => Rc::clone(node), other => panic!("one selected operator root: {other:?}"), } diff --git a/crates/integration-tests/tests/exact_composition.rs b/crates/integration-tests/tests/exact_composition.rs index 738fc115f..3ce280a00 100644 --- a/crates/integration-tests/tests/exact_composition.rs +++ b/crates/integration-tests/tests/exact_composition.rs @@ -9,7 +9,9 @@ use std::rc::Rc; -use asap_integration_tests::physical_export::{at_ingestion, physical_asap_dag, timed}; +use asap_integration_tests::physical_export::{ + default_physical_properties, physical_asap_dag, physical_properties_at_ingestion, +}; use asap_integration_tests::planning::explicit_local_candidates; use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; use asap_types::ir::operator::operator_properties::{Reduction, Source}; @@ -198,7 +200,7 @@ fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { .unwrap_or_else(|| panic!("{kind:?}: a KLL over the exact accumulator")); // Timing is not stored on the plan: time it with the outer summary // maintained (which also validates every edge) and inspect it. - let timed = at_ingestion(&root); + let physical_properties = physical_properties_at_ingestion(&root); let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { panic!("expected KLL evaluation, got {:?}", root.operator); }; @@ -210,7 +212,9 @@ fn every_exact_accumulator_is_finalized_before_an_outer_sketch() { panic!("{kind:?}: missing maintenance finalization"); }; assert_eq!( - timed.get(child).map(|properties| properties.timing()), + physical_properties + .get(child) + .map(|properties| properties.timing()), Some(ExecutionTiming::IngestionTime), "{kind:?}: finalization runs at maintenance time" ); @@ -254,7 +258,7 @@ fn max_and_avg_over_quantile_compose_at_query_time() { "the composed plan's schema is the pre-ASAP target's own" ); assert_eq!( - timed(&composed) + default_physical_properties(&composed) .get(&composed) .map(|properties| properties.timing()), Some(ExecutionTiming::QueryTime), @@ -276,7 +280,7 @@ fn identity_and_genuine_multi_row_folds_both_compose() { let root = agg(vec![0], AggIntent::Max { col: None }, inner); let composed = composed(&root); assert_eq!( - timed(&composed) + default_physical_properties(&composed) .get(&composed) .map(|properties| properties.timing()), Some(ExecutionTiming::QueryTime), diff --git a/crates/integration-tests/tests/frequency_execution.rs b/crates/integration-tests/tests/frequency_execution.rs index a4fe275a3..af611df34 100644 --- a/crates/integration-tests/tests/frequency_execution.rs +++ b/crates/integration-tests/tests/frequency_execution.rs @@ -35,9 +35,9 @@ fn built_family(candidate: &Rc) -> Option { /// Execute `candidate` at query time over `SAMPLES`, all inside the window. fn execute(candidate: &Rc) -> Vec { - let timed = + let physical_properties = apply_physical_properties(candidate, &PhysicalPropertiesAssignment::default()).unwrap(); - let dag = compile_physical_asap_dag(candidate, &timed).unwrap(); + let dag = compile_physical_asap_dag(candidate, &physical_properties).unwrap(); let ranges = dag .nodes .iter() diff --git a/crates/integration-tests/tests/kll_tumbling_window_execution.rs b/crates/integration-tests/tests/kll_tumbling_window_execution.rs index 852fc2c95..ae65dd9e9 100644 --- a/crates/integration-tests/tests/kll_tumbling_window_execution.rs +++ b/crates/integration-tests/tests/kll_tumbling_window_execution.rs @@ -124,10 +124,11 @@ fn fixture() -> Fixture { QueryRoot::Operator(estimate(0.5)), QueryRoot::Operator(estimate(0.99)), ]; - let timed = + let physical_properties = apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) .unwrap(); - let compiled = compile_physical_asap_workload_with_node_ids(&roots, &timed).unwrap(); + let compiled = + compile_physical_asap_workload_with_node_ids(&roots, &physical_properties).unwrap(); let mut tumbling_window_ids = vec![0; 5]; let (mut merge, mut p50, mut p99) = (0, 0, 0); for node in &compiled.dag.nodes { diff --git a/crates/integration-tests/tests/per_series_exact_execution.rs b/crates/integration-tests/tests/per_series_exact_execution.rs index 1d1e54e7f..06fb33006 100644 --- a/crates/integration-tests/tests/per_series_exact_execution.rs +++ b/crates/integration-tests/tests/per_series_exact_execution.rs @@ -96,8 +96,9 @@ fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap { /// at its `Scan` of `x`: each answered series' `instance` and value. fn execute(query: &str, samples: &[RawDataSample]) -> Vec<(String, f64)> { let plan = per_entity_plan(query); - let timed = apply_physical_properties(&plan, &PhysicalPropertiesAssignment::default()).unwrap(); - let dag = compile_physical_asap_dag(&plan, &timed).unwrap(); + let physical_properties = + apply_physical_properties(&plan, &PhysicalPropertiesAssignment::default()).unwrap(); + let dag = compile_physical_asap_dag(&plan, &physical_properties).unwrap(); let node = |matches: fn(&PhysicalASAPOperatorPayload) -> bool| { let found: Vec<_> = dag .nodes diff --git a/crates/integration-tests/tests/physical_common/mod.rs b/crates/integration-tests/tests/physical_common/mod.rs index 4eaab0083..e2ea6f007 100644 --- a/crates/integration-tests/tests/physical_common/mod.rs +++ b/crates/integration-tests/tests/physical_common/mod.rs @@ -61,9 +61,10 @@ fn compile_with( root: &std::rc::Rc, assignment: &asap_types::ir::PhysicalPropertiesAssignment, ) -> Result> { - let timed = asap_types::ir::apply_physical_properties(root, assignment)?; + let physical_properties = asap_types::ir::apply_physical_properties(root, assignment)?; Ok(asap_types::ir::physical_export::compile_physical_asap_dag( - root, &timed, + root, + &physical_properties, )?) } diff --git a/crates/integration-tests/tests/physical_properties_e2e.rs b/crates/integration-tests/tests/physical_properties_e2e.rs index 0057d8455..636b8e6eb 100644 --- a/crates/integration-tests/tests/physical_properties_e2e.rs +++ b/crates/integration-tests/tests/physical_properties_e2e.rs @@ -92,8 +92,8 @@ fn timed_dag( assignment: &PhysicalPropertiesAssignment, ) -> PhysicalASAPDAG { let (split, assignment) = split_shared_by_timing(std::slice::from_ref(root), assignment); - let timed = apply_physical_properties(&split[0], &assignment).unwrap(); - compile_physical_asap_dag(&split[0], &timed).unwrap() + let physical_properties = apply_physical_properties(&split[0], &assignment).unwrap(); + compile_physical_asap_dag(&split[0], &physical_properties).unwrap() } fn states(dag: &PhysicalASAPDAG) -> Vec { diff --git a/crates/integration-tests/tests/promql_to_post_asap.rs b/crates/integration-tests/tests/promql_to_post_asap.rs index 334a9b785..b15a95974 100644 --- a/crates/integration-tests/tests/promql_to_post_asap.rs +++ b/crates/integration-tests/tests/promql_to_post_asap.rs @@ -11,7 +11,7 @@ use std::rc::Rc; use asap_integration_tests::fixtures::lower_promql; use asap_integration_tests::physical_export::{ - at_ingestion, at_ingestion_physical_asap_dag, physical_asap_dag, + at_ingestion_physical_asap_dag, physical_asap_dag, physical_properties_at_ingestion, }; use asap_integration_tests::planning::{ logical_candidates, logical_candidates_ignoring_guarantees, @@ -642,7 +642,7 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { .expect("a KLL over the finalized Sum accumulator"); // Timings are not stored on the plan: time it with its outer summary // maintained and read each node's timing. - let timed = at_ingestion(&plan); + let physical_properties = physical_properties_at_ingestion(&plan); let Some(ASAPOp::SummaryEstimate { summary_input, .. }) = plan.asap() else { panic!("expected selected quantile summary"); }; @@ -656,7 +656,9 @@ fn nested_summary_explicitly_finalizes_exact_child_at_ingestion_time() { ); }; assert_eq!( - timed.get(child).map(|properties| properties.timing()), + physical_properties + .get(child) + .map(|properties| properties.timing()), Some(asap_types::ir::properties::ExecutionTiming::IngestionTime) ); assert!(matches!( diff --git a/crates/integration-tests/tests/reuse_rules.rs b/crates/integration-tests/tests/reuse_rules.rs index 1e3cbc531..874bf6cd7 100644 --- a/crates/integration-tests/tests/reuse_rules.rs +++ b/crates/integration-tests/tests/reuse_rules.rs @@ -111,12 +111,13 @@ fn samples() -> Vec<(&'static str, &'static str, i64, f64)> { /// `Scan`; each `TimeRange` keeps its own window (a tumbling window's moved /// back by the `TimeShift` below it). fn execute(root: &Rc) -> Vec { - let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default()).unwrap(); + let physical_properties = + apply_physical_properties(root, &PhysicalPropertiesAssignment::default()).unwrap(); // Tumbling windows run as their tumbling windows, each over its own window. let expansion = asap_types::ir::operator::tumbling_window_chain::expand_tumbling_window_chains_in_workload( &[QueryRoot::Operator(Rc::clone(root))], - &timed, + &physical_properties, ) .unwrap(); let dag = asap_types::ir::physical_export::compile_physical_asap_workload( diff --git a/crates/integration-tests/tests/tumbling_window_merge.rs b/crates/integration-tests/tests/tumbling_window_merge.rs index a9f205622..88a846b1c 100644 --- a/crates/integration-tests/tests/tumbling_window_merge.rs +++ b/crates/integration-tests/tests/tumbling_window_merge.rs @@ -179,8 +179,8 @@ struct Exported { } fn export(root: &Rc, assignment: &PhysicalPropertiesAssignment) -> Exported { - let timed = apply_physical_properties(root, assignment).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(root, &timed).unwrap(); + let physical_properties = apply_physical_properties(root, assignment).unwrap(); + let compiled = compile_physical_asap_dag_with_node_ids(root, &physical_properties).unwrap(); let tumbling_window = |node: &OperatorNode| -> i64 { match node.children()[0].non_asap() { Some(NonASAPOp::TimeShift { shift, .. }) => shift.offset_ms / TUMBLING_WINDOW_WIDTH_MS, @@ -304,10 +304,10 @@ fn execute_maintained_chain( &chain, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(planned, &assignment).unwrap(); + let physical_properties = apply_physical_properties(planned, &assignment).unwrap(); let expansion = expand_tumbling_window_chains_in_workload( &[QueryRoot::Operator(Rc::clone(planned))], - &timed, + &physical_properties, ) .unwrap(); let [QueryRoot::Operator(expanded)] = &expansion.roots[..] else { diff --git a/crates/logical-asap-optimization/src/pass1/memo_population.rs b/crates/logical-asap-optimization/src/pass1/memo_population.rs index 95b8b15f5..de13315fa 100644 --- a/crates/logical-asap-optimization/src/pass1/memo_population.rs +++ b/crates/logical-asap-optimization/src/pass1/memo_population.rs @@ -437,7 +437,7 @@ fn builds_a_sketch(node: &Rc) -> bool { } /// Interned structural identities: the operator over its children's identities, -/// without the schema, guarantee, timing and lifetime composition +/// without the schema and guarantee composition /// derives. Memoized per node (the cache keeps each node alive, so its /// address is not reused). #[derive(Default)] diff --git a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs index cac277c9b..fccf9b322 100644 --- a/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs +++ b/crates/logical-asap-optimization/src/pass1/per_computation_candidates.rs @@ -40,8 +40,8 @@ //! outputs then get guarantee `None`). So every candidate of a group //! computes the group's own rows. //! -//! The input must be a logical DAG: ASAP operators, execution timing or a -//! materialization are rejected. No cost model is used. The accuracy model +//! The input must be a logical DAG: ASAP operators are rejected. A logical +//! DAG has no physical properties (#1200). No cost model is used. The accuracy model //! is used only to prune summaries no query reading them can use //! ([`PerComputationCandidates::without_summaries_no_query_can_use`]); //! ranking and discarding valid candidates belong to Stage 3. diff --git a/crates/logical-asap-optimization/src/test_support.rs b/crates/logical-asap-optimization/src/test_support.rs index c3a7d8b09..99aa86db6 100644 --- a/crates/logical-asap-optimization/src/test_support.rs +++ b/crates/logical-asap-optimization/src/test_support.rs @@ -108,7 +108,6 @@ use std::time::Duration; use asap_types::ir::operator::agg_intent::AggIntent; use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction, Source}; -use asap_types::ir::properties::timing::{apply_physical_properties, PhysicalPropertiesAssignment}; use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr, TimeRangeKind}; @@ -217,26 +216,6 @@ pub fn time_range(range: Duration, child: Rc) -> Rc .unwrap() } -/// The physical properties of every node of `root` under the default -/// (every sub-DAG not materialized) assignment — what export and the -/// post-ASAP validators read. -pub fn timed(root: &Rc) -> PhysicalPropertiesAssignment { - apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) - .expect("default physical properties apply") -} - -/// The physical properties of every node of `root` with every sub-DAG -/// materialized at ingestion time where the DAG allows it. -pub fn at_ingestion(root: &Rc) -> PhysicalPropertiesAssignment { - apply_physical_properties( - root, - &PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( - std::slice::from_ref(root), - ), - ) - .expect("at-ingestion physical properties apply") -} - /// The Count-Min summary of a Stage 1 candidate of `query`. pub fn cms_summary(query: &str) -> OperatorNode { use crate::pass1::per_computation_candidates::compose_logical_candidate; diff --git a/crates/physical-asap-optimization/src/lib.rs b/crates/physical-asap-optimization/src/lib.rs index 76b116c42..34a5ccfcb 100644 --- a/crates/physical-asap-optimization/src/lib.rs +++ b/crates/physical-asap-optimization/src/lib.rs @@ -542,13 +542,13 @@ fn assigned_candidate( let split_roots = map_roots(implemented, &mut |node| { Ok(replaced[&Rc::as_ptr(node)].clone()) })?; - let timed = apply_physical_properties_to_roots(&split_roots, &options)?; - let dag = compile_physical_asap_workload_with_node_ids(&split_roots, &timed)?.dag; + let physical_properties = apply_physical_properties_to_roots(&split_roots, &options)?; + let dag = compile_physical_asap_workload_with_node_ids(&split_roots, &physical_properties)?.dag; Ok(PhysicalWorkloadPlan { id: String::new(), from_logical: from_logical.to_string(), roots: split_roots, - physical_properties: timed, + physical_properties, dag, }) } diff --git a/crates/physical-asap-optimization/src/split_by_timing.rs b/crates/physical-asap-optimization/src/split_by_timing.rs index 134945b58..c4f1d4349 100644 --- a/crates/physical-asap-optimization/src/split_by_timing.rs +++ b/crates/physical-asap-optimization/src/split_by_timing.rs @@ -264,14 +264,14 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); let (split, assignment) = split_shared_by_timing(std::slice::from_ref(&root), &assignment); - let timed = apply_physical_properties(&split[0], &assignment).unwrap(); + let physical_properties = apply_physical_properties(&split[0], &assignment).unwrap(); let summary = child(&child(&child(&split[0]))); assert!(matches!( summary.operator, Operator::ASAP(ASAPOp::SummaryAgg { .. }) )); assert_eq!( - timed.get(&summary), + physical_properties.get(&summary), Some(PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION) ); } @@ -306,10 +306,14 @@ mod tests { )); let (split, assignment) = split_shared_by_timing(&roots, &assignment); assert_eq!(split.len(), 2); - let timed = apply_physical_properties_to_roots(&as_roots(&split), &assignment).unwrap(); - let timing = |node: &OperatorNode| timed.get(node).unwrap().timing(); + let physical_properties = + apply_physical_properties_to_roots(&as_roots(&split), &assignment).unwrap(); + let timing = |node: &OperatorNode| physical_properties.get(node).unwrap().timing(); let summary = child(&child(&child(&split[0]))); - assert_eq!(timed.get(&summary).unwrap().lifetime(), MATERIALIZED_ACROSS); + assert_eq!( + physical_properties.get(&summary).unwrap().lifetime(), + MATERIALIZED_ACROSS + ); assert_eq!(timing(&child(&summary)), ExecutionTiming::IngestionTime); assert_eq!( timing(&child(&child(&split[1]))), diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index a4a9b4148..81c97e04d 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -3481,10 +3481,10 @@ mod tests { if let Some(materialized) = materialized { assignment.set(&state, materialized); } - let timed = apply_physical_properties(root, &assignment).unwrap(); + let physical_properties = apply_physical_properties(root, &assignment).unwrap(); asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids( &[QueryRoot::Operator(Rc::clone(root))], - &timed, + &physical_properties, ) .unwrap() .dag diff --git a/crates/planner/src/installable.rs b/crates/planner/src/installable.rs index c9d4f2491..fadd2db0b 100644 --- a/crates/planner/src/installable.rs +++ b/crates/planner/src/installable.rs @@ -284,11 +284,11 @@ pub fn installable_plan_of( .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) .collect(); let query_texts: Vec = entries.iter().map(|entry| entry.query.0.clone()).collect(); - // Every node keeps the physical properties Stage 3 gave it. - let timed = apply_physical_properties_to_roots(roots, physical_properties)?; + // Every node takes the physical properties Stage 3 gave it. + let physical_properties = apply_physical_properties_to_roots(roots, physical_properties)?; let expanded = tumbling_window_retention_of_expanded_tumbling_windows( roots, - &timed, + &physical_properties, tumbling_window_retention_ms, )?; let report = selection.map(|selection| report_of(selection, &expanded.installed_node_ids)); @@ -657,11 +657,11 @@ struct ExpandedWithRetention { /// exported). fn tumbling_window_retention_of_expanded_tumbling_windows( roots: &[QueryRoot], - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, tumbling_window_retention_ms: &BTreeMap, ) -> Result { use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let selected = compile_physical_asap_workload_with_node_ids(roots, timed)?; + let selected = compile_physical_asap_workload_with_node_ids(roots, physical_properties)?; // Reject unsupported raw-source shapes before expanding possibly thousands // of tumbling windows. The complete expanded-plan validation remains authoritative. selected.dag.check_raw_data_windows()?; @@ -674,7 +674,7 @@ fn tumbling_window_retention_of_expanded_tumbling_windows( .ok_or(InstallablePlanError::TumblingWindowRetentionOfUnknownChain { node_id })?; retention_by_chain.push((Rc::clone(chain), retention_ms)); } - let expansion = expand_tumbling_window_chains_in_workload(roots, timed) + let expansion = expand_tumbling_window_chains_in_workload(roots, physical_properties) .map_err(InstallablePlanError::TumblingWindowChainExpansion)?; let compiled = compile_physical_asap_workload_with_node_ids( &expansion.roots, @@ -910,9 +910,9 @@ mod tests { })) .unwrap(); let export = |root| { - let timed = + let physical_properties = apply_physical_properties(&root, &PhysicalPropertiesAssignment::default()).unwrap(); - compile_physical_asap_dag(&root, &timed).unwrap() + compile_physical_asap_dag(&root, &physical_properties).unwrap() }; let install = |dag| { InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) @@ -1003,8 +1003,8 @@ mod tests { &merged, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&merged, &assignment).unwrap(); - let dag = compile_physical_asap_dag(&merged, &timed).unwrap(); + let physical_properties = apply_physical_properties(&merged, &assignment).unwrap(); + let dag = compile_physical_asap_dag(&merged, &physical_properties).unwrap(); match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { Err(InstallablePlanError::Uninstallable( InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { @@ -1042,8 +1042,8 @@ mod tests { PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); } - let timed = apply_physical_properties(&merge, &assignment).unwrap(); - let dag = compile_physical_asap_dag(&merge, &timed).unwrap(); + let physical_properties = apply_physical_properties(&merge, &assignment).unwrap(); + let dag = compile_physical_asap_dag(&merge, &physical_properties).unwrap(); match InstallablePlan::of_dag(dag, vec![String::new()], &HashMap::new(), &HashMap::new()) { Err(InstallablePlanError::Uninstallable( InstallableDagError::IngestionBuildReadsWindowsOfDifferentOffsets { diff --git a/crates/planner/src/lib.rs b/crates/planner/src/lib.rs index ffefd3f12..3b4aa8d8f 100644 --- a/crates/planner/src/lib.rs +++ b/crates/planner/src/lib.rs @@ -33,7 +33,7 @@ pub use pass::{ }; pub use run::{ run_planning, selected_chosen_candidates, PlanSearchSettings, PlanningReport, PlanningRun, - SelectionError, Stage1ToStage3Pass, + SelectedPlan, SelectionError, Stage1ToStage3Pass, }; mod installable; diff --git a/crates/planner/src/pass.rs b/crates/planner/src/pass.rs index 0923df532..7d9105748 100644 --- a/crates/planner/src/pass.rs +++ b/crates/planner/src/pass.rs @@ -223,7 +223,8 @@ pub struct PlanOutput { /// empty for a pass that does not set it. pub tumbling_window_retention_ms: BTreeMap, /// The physical properties the pass selected for the plans' nodes, by - /// node: for [`Stage1ToStage3Pass`] every node, as Stage 3 selected it. + /// node: for [`Stage1ToStage3Pass`] every node, as Stage 3 selected it + /// (checked in debug builds). /// A node it does not list takes the timing pass's default (not /// materialized, computed when its consumer runs); empty for a pass that /// does not set it. @@ -306,7 +307,7 @@ impl PlanOutput { } /// The workload as one physical ASAP DAG, with shared sub-DAGs exported - /// once. A node keeps the physical properties its pass gave it + /// once. A node takes the physical properties its pass gave it /// (`physical_properties`): for [`Stage1ToStage3Pass`] that is Stage 3's /// selected plan as selected (e.g. ingestion-time tumbling windows), the /// document a deployment runtime executes. Only a node the pass did not @@ -325,8 +326,9 @@ impl PlanOutput { ) -> Result { // A node several roots share by `Rc` is timed and exported once. let roots = self.roots(); - let timed = apply_physical_properties_to_roots(&roots, &self.physical_properties)?; - compile_physical_asap_workload(&roots, &timed) + let physical_properties = + apply_physical_properties_to_roots(&roots, &self.physical_properties)?; + compile_physical_asap_workload(&roots, &physical_properties) } pub fn len(&self) -> usize { diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 6291706d5..381f52970 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -21,7 +21,7 @@ use asap_physical_asap_optimization::{ }; use asap_plan_selection::{select_plan, ErrorSelectingPlan, SearchReport, Selection}; use asap_types::ir::schema::schema_support::with_promql_series_identity; -use asap_types::ir::{OperatorNode, QueryRoot}; +use asap_types::ir::{OperatorNode, PhysicalPropertiesAssignment, QueryRoot}; use asap_types::stages::{ AssignmentRealizer, CandidateForMemoGroupId, DagOfSelectedCandidates, LogicalASAPMemoCandidates, Memo, MemoGroupId, PhysicalASAPMemoCandidates, @@ -107,13 +107,43 @@ pub struct PlanningRun { pub selection: Result, /// The selected plan as Stage 2 built it, with its tumbling-window retention. pub selected_physical_plan: Option, - /// The plan to run: the selected plan's roots with the weight proofs - /// Stage 3 made applied. Its nodes' physical properties are in - /// `selected_physical_plan`'s `plan.physical_properties`. - pub selected_plan: Option>, + /// The plan to run, with its nodes' physical properties. + pub selected_plan: Option, pub report: PlanningReport, } +/// The plan to run: the selected plan's roots with the weight proofs Stage 3 +/// made applied, and the physical properties of their nodes. The +/// [`RealizedPlan`] it comes from stays as Stage 2 built it. +#[derive(Debug, Clone)] +pub struct SelectedPlan { + pub roots: Vec, + pub physical_properties: PhysicalPropertiesAssignment, +} + +impl SelectedPlan { + /// `realized` with the weight proofs `selection` made applied. + pub fn of(realized: &RealizedPlan, selection: &Selection, data: &DataWorkload) -> Self { + let proven: HashSet<*const OperatorNode> = selection + .nodes_with_proven_non_negative_weights + .iter() + .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) + .map(Rc::as_ptr) + .collect(); + let mut physical_properties = realized.plan.physical_properties.clone(); + let roots = with_confirmed_weights_at( + &realized.plan.roots, + data, + &|node| proven.contains(&Rc::as_ptr(node)), + &mut physical_properties, + ); + Self { + roots, + physical_properties, + } + } +} + /// Run the planning stages over the frontends' `roots` (query `i` is /// `roots[i]`, with workload entry `entries[i]`). Only a Pass 1 failure is /// an error; a failed selection is reported in the run. @@ -221,21 +251,10 @@ pub fn run_planning( let selection = selection.map_err(SelectionError::Selection)?; report.search_report = Some(selection.search_report.clone()); // The selected plan, as Stage 2 builds it. - let mut realized = realizer + let realized = realizer .realize(physical, &selection.selected_dag) .map_err(SelectionError::Realization)?; - let proven: HashSet<*const OperatorNode> = selection - .nodes_with_proven_non_negative_weights - .iter() - .filter_map(|node| realized.operator_node_of_selected_node.get(node.0)) - .map(Rc::as_ptr) - .collect(); - selected_plan = Some(with_confirmed_weights_at( - &realized.plan.roots, - data, - &|node| proven.contains(&Rc::as_ptr(node)), - &mut realized.plan.physical_properties, - )); + selected_plan = Some(SelectedPlan::of(&realized, &selection, data)); selected_physical_plan = Some(realized); Ok(selection) })(); @@ -378,7 +397,14 @@ impl OptimizationPass for Stage1ToStage3Pass { PlanSearchSettings::default(), )?; let selection = run.selection?; - let roots = run.selected_plan.expect("a selection has a plan"); + let SelectedPlan { + roots, + physical_properties, + } = run.selected_plan.expect("a selection has a plan"); + debug_assert!( + physical_properties.lists_every_node_of(&roots), + "the selected plan's physical properties list every node of its roots" + ); let mut output = PlanOutput::new(Vec::new()); for (entry_index, root) in roots.iter().enumerate() { match root { @@ -406,8 +432,8 @@ impl OptimizationPass for Stage1ToStage3Pass { } if let Some(realized) = run.selected_physical_plan { output.tumbling_window_retention_ms = realized.tumbling_window_retention_ms; - output.physical_properties = realized.plan.physical_properties; } + output.physical_properties = physical_properties; output.selection = Some(selection); Ok(output) } diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index f32b355b4..7bc67f619 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -172,7 +172,7 @@ async fn facade_plans_match_run_planning() { facade.accepted_accuracy_per_query, selection.accepted_accuracy_per_query ); - assert_eq!(&output.roots(), run.selected_plan.as_ref().unwrap()); + assert_eq!(&output.roots(), &run.selected_plan.as_ref().unwrap().roots); } /// A repeating SQL query reaches the optimizer, not only `query_batch` entries. diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index e0903ee9f..38c1e958d 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -320,7 +320,7 @@ fn state_key(view: &MaterializedView, evaluation_ms: i64) -> String { Bound::Unbounded => Bound::Unbounded, }; let boxes = view.coverage().selection(); - let timed = boxes.iter().any(|b| b.relative_time().is_some()); + let has_relative_time = boxes.iter().any(|b| b.relative_time().is_some()); let selection: Vec = boxes .iter() .map(|selection_box| { @@ -335,7 +335,7 @@ fn state_key(view: &MaterializedView, evaluation_ms: i64) -> String { serde_json::json!({ "definition_id": view.coverage().definition_id(), "selection": selection, - "evaluation_ms": (!timed).then_some(evaluation_ms), + "evaluation_ms": (!has_relative_time).then_some(evaluation_ms), }) .to_string() } @@ -1540,7 +1540,7 @@ impl OptimizationPass for StoredAsForced { continue; } // A shared non-summary node Stage 2 built once per - // evaluation is not kept: a build forced to ingestion time + // evaluation is not materialized: a build forced to ingestion time // above it cannot read a query-time result. let selected = output .physical_properties @@ -1554,8 +1554,8 @@ impl OptimizationPass for StoredAsForced { ) => selected, _ => None, }; - if let Some(kept) = (self.store)(&node).or(own) { - assignment.set(&node, kept); + if let Some(materialized) = (self.store)(&node).or(own) { + assignment.set(&node, materialized); } } } diff --git a/crates/types/src/ir/operator/tumbling_window_chain.rs b/crates/types/src/ir/operator/tumbling_window_chain.rs index 8a724e2d1..c3b0ae9e3 100644 --- a/crates/types/src/ir/operator/tumbling_window_chain.rs +++ b/crates/types/src/ir/operator/tumbling_window_chain.rs @@ -283,7 +283,7 @@ pub struct ExpandedTumblingWindowChain { /// input's (empty for an input not yet timed). pub fn expand_tumbling_window_chains_in_workload( roots: &[QueryRoot], - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result { let mut cache = NodeToExpandedNodeCache::default(); let roots = roots @@ -330,14 +330,12 @@ pub fn expand_tumbling_window_chains_in_workload( } }) .collect::>(); - let mut physical_properties = timed.clone(); + let mut physical_properties = physical_properties.clone(); for (original, replacement) in &cache.expanded { - if let Some(properties) = timed.get(original) { - physical_properties.set(replacement, properties); - } + physical_properties.copy_assignment(original, replacement); } for expanded in &chains { - let Some(properties) = timed.get(&expanded.chain) else { + let Some(properties) = physical_properties.get(&expanded.chain) else { continue; }; for tumbling_window in &expanded.tumbling_windows { diff --git a/crates/types/src/ir/physical_export.rs b/crates/types/src/ir/physical_export.rs index 77b51e38f..38856cc87 100644 --- a/crates/types/src/ir/physical_export.rs +++ b/crates/types/src/ir/physical_export.rs @@ -377,9 +377,12 @@ impl PhysicalASAPDAG { #[cfg(any(test, feature = "test-support"))] pub fn with_every_node_at_query_time(&self) -> Result { let roots = self.query_roots()?; - let timed = + let physical_properties = apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default())?; - Ok(compile_physical_asap_workload(&roots, &timed)?) + Ok(compile_physical_asap_workload( + &roots, + &physical_properties, + )?) } /// Every node of this DAG as IR, by node id, with its exported @@ -850,9 +853,9 @@ pub struct PhysicalASAPDAGCompilation { #[cfg(any(test, feature = "test-support"))] pub fn compile_physical_asap_dag( root: &Rc, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result { - Ok(compile_physical_asap_dag_with_node_ids(root, timed)?.dag) + Ok(compile_physical_asap_dag_with_node_ids(root, physical_properties)?.dag) } /// Export the DAG below `root` with each node's physical properties from @@ -862,9 +865,12 @@ pub fn compile_physical_asap_dag( #[cfg(any(test, feature = "test-support"))] pub(crate) fn compile_physical_asap_dag_with_node_ids( root: &Rc, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result { - compile_physical_asap_workload_with_node_ids(&[QueryRoot::Operator(Rc::clone(root))], timed) + compile_physical_asap_workload_with_node_ids( + &[QueryRoot::Operator(Rc::clone(root))], + physical_properties, + ) } /// Export a batch as one DAG, each node with its physical properties from @@ -873,14 +879,14 @@ pub(crate) fn compile_physical_asap_dag_with_node_ids( /// batch. pub fn compile_physical_asap_workload( roots: &[QueryRoot], - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result { - Ok(compile_physical_asap_workload_with_node_ids(roots, timed)?.dag) + Ok(compile_physical_asap_workload_with_node_ids(roots, physical_properties)?.dag) } pub fn compile_physical_asap_workload_with_node_ids( roots: &[QueryRoot], - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result { let (flat, nodes_by_id) = flatten(roots); let mut scalar_roots = Vec::new(); @@ -900,12 +906,14 @@ pub fn compile_physical_asap_workload_with_node_ids( let mut edges = Vec::new(); for (id, flat_node) in flat.nodes.into_iter().enumerate() { let node = &nodes_by_id[id]; - let properties = timed - .get(node) - .ok_or(ExecutionDataStateError::UntimedNode { - operator: node.operator.kind_name(), - })?; - let output_state = data_state(node, timed).expect("listed node has a data state"); + let properties = + physical_properties + .get(node) + .ok_or(ExecutionDataStateError::UntimedNode { + operator: node.operator.kind_name(), + })?; + let output_state = + data_state(node, physical_properties).expect("listed node has a data state"); // Children come before their parents, so every producer is already in `nodes`. for (producer, role) in edge_roles(&flat_node.operator) { let producer_state = nodes[producer].output_state; diff --git a/crates/types/src/ir/properties/timing.rs b/crates/types/src/ir/properties/timing.rs index 661a252e8..cd9d2f704 100644 --- a/crates/types/src/ir/properties/timing.rs +++ b/crates/types/src/ir/properties/timing.rs @@ -139,6 +139,30 @@ impl PhysicalPropertiesAssignment { .map(|(_, properties)| *properties) } + /// Whether the assignment lists every node reachable from `roots`, as a + /// completed assignment ([`apply_physical_properties_to_roots`]) does. + pub fn lists_every_node_of(&self, roots: &[QueryRoot]) -> bool { + fn visit( + node: &OperatorNode, + assignment: &PhysicalPropertiesAssignment, + visited: &mut std::collections::HashSet<*const OperatorNode>, + ) -> bool { + if !visited.insert(node as *const OperatorNode) { + return true; + } + assignment.get(node).is_some() + && node + .children() + .iter() + .all(|child| visit(child, assignment, visited)) + } + let mut visited = std::collections::HashSet::new(); + roots + .iter() + .flat_map(QueryRoot::operator_refs) + .all(|node| visit(node, self, &mut visited)) + } + /// Every summary build and `MaintainPopulation` reachable from `roots` /// materialized in memory at ingestion time where the DAG allows it: not under a /// filter, and with nothing below it that can only run at query time. @@ -234,10 +258,10 @@ fn primitive(node: &OperatorNode) -> DataPrimitive { /// The data state `node`'s output carries, if `timed` lists it. pub fn data_state( node: &OperatorNode, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Option { Some(ExecutionDataState { - timing: timed.get(node)?.timing(), + timing: physical_properties.get(node)?.timing(), primitive: primitive(node), }) } @@ -310,7 +334,7 @@ pub fn apply_physical_properties_to_roots( roots: &[QueryRoot], assignment: &PhysicalPropertiesAssignment, ) -> Result { - let mut timed = PhysicalPropertiesAssignment::default(); + let mut physical_properties = PhysicalPropertiesAssignment::default(); let mut forced = HashMap::new(); let mut validated = HashMap::new(); for root in roots { @@ -319,13 +343,15 @@ pub fn apply_physical_properties_to_roots( node, ExecutionTiming::QueryTime, assignment, - &mut timed, + &mut physical_properties, &mut forced, )?; // A root stored at ingestion time is read by the query; one that // runs at ingestion time without being stored there produces rows // nothing reads. - let properties = timed.get(node).expect("written node is timed"); + let properties = physical_properties + .get(node) + .expect("written node is timed"); if properties.timing() == ExecutionTiming::IngestionTime && !properties.readable_at(ExecutionTiming::QueryTime) && primitive(node) == DataPrimitive::Raw @@ -334,10 +360,10 @@ pub fn apply_physical_properties_to_roots( operator: node.operator.kind_name(), }); } - validate(node, &timed, &mut validated)?; + validate(node, &physical_properties, &mut validated)?; } } - Ok(timed) + Ok(physical_properties) } /// Validate the sub-DAG below `root` with every sub-DAG materialized at ingestion time @@ -354,10 +380,16 @@ pub fn validate_at_ingestion( let assignment = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(root), ); - let mut timed = PhysicalPropertiesAssignment::default(); + let mut physical_properties = PhysicalPropertiesAssignment::default(); let mut forced = HashMap::new(); - write(root, root_timing, &assignment, &mut timed, &mut forced)?; - validate(root, &timed, &mut HashMap::new()) + write( + root, + root_timing, + &assignment, + &mut physical_properties, + &mut forced, + )?; + validate(root, &physical_properties, &mut HashMap::new()) } /// The data state `node` produces with every sub-DAG materialized at ingestion time where @@ -442,11 +474,11 @@ fn write( node: &Rc, consumer: ExecutionTiming, assignment: &PhysicalPropertiesAssignment, - timed: &mut PhysicalPropertiesAssignment, + physical_properties: &mut PhysicalPropertiesAssignment, forced: &mut HashMap<*const OperatorNode, bool>, ) -> Result<(), ExecutionDataStateError> { let properties = placement(node, consumer, assignment, forced)?; - if let Some(previous) = timed.get(node) { + if let Some(previous) = physical_properties.get(node) { if previous.timing() != properties.timing() { return Err(ExecutionDataStateError::ConflictingTiming { first: ExecutionDataState { @@ -462,32 +494,41 @@ fn write( return Ok(()); } for child in node.children() { - write(child, properties.timing(), assignment, timed, forced)?; + write( + child, + properties.timing(), + assignment, + physical_properties, + forced, + )?; } - timed.set(node, properties); + physical_properties.set(node, properties); Ok(()) } -fn state_of(node: &OperatorNode, timed: &PhysicalPropertiesAssignment) -> ExecutionDataState { - data_state(node, timed).expect("timed node") +fn state_of( + node: &OperatorNode, + physical_properties: &PhysicalPropertiesAssignment, +) -> ExecutionDataState { + data_state(node, physical_properties).expect("timed node") } /// Check every edge below `node` against the module-level rules. fn validate( node: &Rc, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, seen: &mut HashMap<*const OperatorNode, ()>, ) -> Result<(), ExecutionDataStateError> { if seen.insert(Rc::as_ptr(node), ()).is_some() { return Ok(()); } - let timing = timed.get(node).expect("timed node").timing(); + let timing = physical_properties.get(node).expect("timed node").timing(); match &node.operator { - Operator::ASAP(op) => validate_asap(node, op, timing, timed)?, - Operator::NonASAP(op) => validate_non_asap(node, op, timing, timed)?, + Operator::ASAP(op) => validate_asap(node, op, timing, physical_properties)?, + Operator::NonASAP(op) => validate_non_asap(node, op, timing, physical_properties)?, } for child in node.children() { - validate(child, timed, seen)?; + validate(child, physical_properties, seen)?; } Ok(()) } @@ -510,7 +551,7 @@ fn validate_asap( node: &OperatorNode, op: &ASAPOp, timing: ExecutionTiming, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result<(), ExecutionDataStateError> { match op { ASAPOp::SummaryAgg { child, .. } | ASAPOp::SummaryTumblingWindowChain { child, .. } => { @@ -518,7 +559,7 @@ fn validate_asap( ASAPOp::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain.child", _ => "SummaryAgg.child", }; - let avail = state_of(child, timed); + let avail = state_of(child, physical_properties); match avail { ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => {} s if s.primitive == DataPrimitive::SummaryState => { @@ -538,7 +579,9 @@ fn validate_asap( // `NonASAP` consumer does. if timing == ExecutionTiming::QueryTime && avail.timing == ExecutionTiming::IngestionTime - && !timed.get(child).is_some_and(|p| p.readable_at(timing)) + && !physical_properties + .get(child) + .is_some_and(|p| p.readable_at(timing)) { return Err( ExecutionDataStateError::UnstoredIngestionInputUnderQueryTimeSummary { @@ -560,7 +603,7 @@ fn validate_asap( Ok(()) } ASAPOp::SummaryEstimate { summary_input, .. } => { - let s = state_of(summary_input, timed); + let s = state_of(summary_input, physical_properties); if s.primitive != DataPrimitive::SummaryState { return Err(ExecutionDataStateError::IllegalChildDataState { edge: "SummaryEstimate.summary_input", @@ -570,13 +613,13 @@ fn validate_asap( if timing != ExecutionTiming::QueryTime { return Err(ExecutionDataStateError::IllegalChildDataState { edge: "SummaryEstimate", - child: state_of(node, timed), + child: state_of(node, physical_properties), }); } Ok(()) } ASAPOp::FinalizeExactAccumulator { child } => { - let s = state_of(child, timed); + let s = state_of(child, physical_properties); if s.primitive != DataPrimitive::SummaryState || is_exact_accumulator_state(&child.schema).is_err() || (timing == ExecutionTiming::IngestionTime && s.timing != timing) @@ -590,7 +633,7 @@ fn validate_asap( } ASAPOp::MaintainPopulation { child, population } => { let valid = population.matches_node(child) - && state_of(child, timed) + && state_of(child, physical_properties) == ExecutionDataState { timing, primitive: DataPrimitive::Raw, @@ -614,7 +657,7 @@ fn validate_asap( } ASAPOp::SummaryMerge { children, .. } => { for child in children { - let s = state_of(child, timed); + let s = state_of(child, physical_properties); if s.primitive != DataPrimitive::SummaryState || (timing == ExecutionTiming::IngestionTime && s.timing != timing) { @@ -672,13 +715,13 @@ fn validate_non_asap( node: &OperatorNode, op: &NonASAPOp, timing: ExecutionTiming, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result<(), ExecutionDataStateError> { // Every input is rows at this node's own timing. Ingestion work never // reads a query-time value; exact-accumulator state may pass through // the projection-like operators unchanged. for child in op.children() { - let s = state_of(child, timed); + let s = state_of(child, physical_properties); let passes_state = matches!( op, NonASAPOp::Project { .. } @@ -689,7 +732,9 @@ fn validate_non_asap( && (is_exact_accumulator_state(&child.schema).is_ok() || selects_states(node)); // A query-time consumer may read the stored output of a sub-DAG // materialized at ingestion time. - let reads_stored = timed.get(child).is_some_and(|p| p.readable_at(timing)); + let reads_stored = physical_properties + .get(child) + .is_some_and(|p| p.readable_at(timing)); if (s.timing != timing && !reads_stored) || (s.primitive != DataPrimitive::Raw && !passes_state) { @@ -948,12 +993,21 @@ mod tests { apply_fixing(root, &[]) } - fn timing_of(timed: &PhysicalPropertiesAssignment, node: &OperatorNode) -> ExecutionTiming { - timed.get(node).expect("timed node").timing() + fn timing_of( + physical_properties: &PhysicalPropertiesAssignment, + node: &OperatorNode, + ) -> ExecutionTiming { + physical_properties.get(node).expect("timed node").timing() } - fn lifetime_of(timed: &PhysicalPropertiesAssignment, node: &OperatorNode) -> Lifetime { - timed.get(node).expect("timed node").lifetime() + fn lifetime_of( + physical_properties: &PhysicalPropertiesAssignment, + node: &OperatorNode, + ) -> Lifetime { + physical_properties + .get(node) + .expect("timed node") + .lifetime() } const MATERIALIZED_ACROSS: Lifetime = Lifetime::AcrossEvaluations(Medium::Memory); @@ -972,18 +1026,18 @@ mod tests { #[test] fn default_assignment_materializes_nothing() { let summary = agg(scan(), kll()); - let timed = + let physical_properties = apply_physical_properties(&summary, &PhysicalPropertiesAssignment::default()).unwrap(); assert_eq!( - data_state(&summary, &timed), + data_state(&summary, &physical_properties), Some(ExecutionDataState { timing: ExecutionTiming::QueryTime, primitive: DataPrimitive::SummaryState, }) ); - assert_eq!(lifetime_of(&timed, &summary), Lifetime::None); + assert_eq!(lifetime_of(&physical_properties, &summary), Lifetime::None); assert_eq!( - data_state(&child(&summary), &timed), + data_state(&child(&summary), &physical_properties), Some(ExecutionDataState::QUERY_ROWS) ); for (lifetime, timing) in [ @@ -995,11 +1049,14 @@ mod tests { ] { let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set(&summary, properties(timing, lifetime)); - let timed = apply_physical_properties(&summary, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &summary), timing); - assert_eq!(lifetime_of(&timed, &summary), lifetime); - assert_eq!(timing_of(&timed, &child(&summary)), timing); - assert_eq!(lifetime_of(&timed, &child(&summary)), Lifetime::None); + let physical_properties = apply_physical_properties(&summary, &assignment).unwrap(); + assert_eq!(timing_of(&physical_properties, &summary), timing); + assert_eq!(lifetime_of(&physical_properties, &summary), lifetime); + assert_eq!(timing_of(&physical_properties, &child(&summary)), timing); + assert_eq!( + lifetime_of(&physical_properties, &child(&summary)), + Lifetime::None + ); } } @@ -1030,21 +1087,28 @@ mod tests { &outer, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&estimate(Rc::clone(&outer)), &assignment).unwrap(); - assert_eq!(lifetime_of(&timed, &outer), MATERIALIZED_ACROSS); - assert_eq!(lifetime_of(&timed, &inner), MATERIALIZED_ACROSS); + let physical_properties = + apply_physical_properties(&estimate(Rc::clone(&outer)), &assignment).unwrap(); + assert_eq!( + lifetime_of(&physical_properties, &outer), + MATERIALIZED_ACROSS + ); + assert_eq!( + lifetime_of(&physical_properties, &inner), + MATERIALIZED_ACROSS + ); } #[test] fn summary_agg_input_runs_at_ingestion_time() { let root = agg(scan(), kll()); - let timed = apply(&root).unwrap(); + let physical_properties = apply(&root).unwrap(); assert_eq!( - data_state(&root, &timed), + data_state(&root, &physical_properties), Some(ExecutionDataState::INGESTION_SUMMARY) ); assert_eq!( - data_state(&child(&root), &timed), + data_state(&child(&root), &physical_properties), Some(ExecutionDataState::INGESTION_ROWS) ); } @@ -1059,8 +1123,11 @@ mod tests { fn evaluation_can_feed_summary_construction_at_query_time() { let inner = estimate(agg(scan(), kll())); let root = estimate(agg(inner, kll())); - let timed = apply(&root).unwrap(); - assert_eq!(timing_of(&timed, &child(&root)), ExecutionTiming::QueryTime); + let physical_properties = apply(&root).unwrap(); + assert_eq!( + timing_of(&physical_properties, &child(&root)), + ExecutionTiming::QueryTime + ); } /// Any non-ASAP operator over an evaluation runs at query time. @@ -1074,9 +1141,9 @@ mod tests { })) .unwrap(); for root in [max(evaluation()), sorted] { - let timed = apply(&root).unwrap(); + let physical_properties = apply(&root).unwrap(); assert_eq!( - data_state(&root, &timed), + data_state(&root, &physical_properties), Some(ExecutionDataState::QUERY_ROWS) ); } @@ -1086,8 +1153,11 @@ mod tests { fn query_time_values_can_feed_query_time_summary_construction() { let post = max(estimate(agg(scan(), kll()))); let root = estimate(agg(post, kll())); - let timed = apply(&root).unwrap(); - assert_eq!(timing_of(&timed, &child(&root)), ExecutionTiming::QueryTime); + let physical_properties = apply(&root).unwrap(); + assert_eq!( + timing_of(&physical_properties, &child(&root)), + ExecutionTiming::QueryTime + ); } #[test] @@ -1100,9 +1170,9 @@ mod tests { }) ); let root = estimate(agg(Rc::clone(&operation), kll())); - let timed = apply_fixing(&root, &[&operation]).unwrap(); + let physical_properties = apply_fixing(&root, &[&operation]).unwrap(); assert_eq!( - data_state(&operation, &timed), + data_state(&operation, &physical_properties), Some(ExecutionDataState::INGESTION_ROWS) ); } @@ -1192,15 +1262,21 @@ mod tests { &tumbling_windows, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&root, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &root), ExecutionTiming::QueryTime); - assert_eq!(lifetime_of(&timed, &tumbling_windows), MATERIALIZED_ACROSS); + let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); assert_eq!( - timing_of(&timed, &tumbling_windows), + timing_of(&physical_properties, &root), + ExecutionTiming::QueryTime + ); + assert_eq!( + lifetime_of(&physical_properties, &tumbling_windows), + MATERIALIZED_ACROSS + ); + assert_eq!( + timing_of(&physical_properties, &tumbling_windows), ExecutionTiming::IngestionTime ); assert_eq!( - timing_of(&timed, &child(&tumbling_windows)), + timing_of(&physical_properties, &child(&tumbling_windows)), ExecutionTiming::IngestionTime ); @@ -1263,9 +1339,12 @@ mod tests { operator: "SummaryAgg" }) ); - let timed = apply(&estimate(Rc::clone(&filtered))).unwrap(); - assert_eq!(timing_of(&timed, &filtered), ExecutionTiming::QueryTime); - assert_eq!(lifetime_of(&timed, &filtered), Lifetime::None); + let physical_properties = apply(&estimate(Rc::clone(&filtered))).unwrap(); + assert_eq!( + timing_of(&physical_properties, &filtered), + ExecutionTiming::QueryTime + ); + assert_eq!(lifetime_of(&physical_properties, &filtered), Lifetime::None); assert!(validate_at_ingestion(&filtered, ExecutionTiming::QueryTime).is_ok()); } @@ -1280,10 +1359,11 @@ mod tests { /// for `node`'s operator kind. fn exported( root: &Rc, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, kind: &str, ) -> Lifetime { - let dag = crate::ir::physical_export::compile_physical_asap_dag(root, timed).unwrap(); + let dag = crate::ir::physical_export::compile_physical_asap_dag(root, physical_properties) + .unwrap(); dag.validate().unwrap(); dag.nodes .iter() @@ -1300,10 +1380,16 @@ mod tests { let stored = Lifetime::OneEvaluation(Medium::Disk); let mut assignment = PhysicalPropertiesAssignment::default(); assignment.set(&read, properties(ExecutionTiming::QueryTime, stored)); - let timed = apply_physical_properties(&read, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &read), ExecutionTiming::QueryTime); - assert_eq!(lifetime_of(&timed, &read), stored); - assert_eq!(exported(&read, &timed, "FinalizeExactAccumulator"), stored); + let physical_properties = apply_physical_properties(&read, &assignment).unwrap(); + assert_eq!( + timing_of(&physical_properties, &read), + ExecutionTiming::QueryTime + ); + assert_eq!(lifetime_of(&physical_properties, &read), stored); + assert_eq!( + exported(&read, &physical_properties, "FinalizeExactAccumulator"), + stored + ); } /// The Debug output names each assigned node by its operator kind, not @@ -1342,9 +1428,15 @@ mod tests { &rows, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&summary, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &summary), ExecutionTiming::QueryTime); - assert_eq!(lifetime_of(&timed, &child(&summary)), MATERIALIZED_ACROSS); + let physical_properties = apply_physical_properties(&summary, &assignment).unwrap(); + assert_eq!( + timing_of(&physical_properties, &summary), + ExecutionTiming::QueryTime + ); + assert_eq!( + lifetime_of(&physical_properties, &child(&summary)), + MATERIALIZED_ACROSS + ); } /// The same holds for exact-accumulator state: a query-time build cannot @@ -1368,9 +1460,15 @@ mod tests { &state, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&summary, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &summary), ExecutionTiming::QueryTime); - assert_eq!(lifetime_of(&timed, &child(&summary)), MATERIALIZED_ACROSS); + let physical_properties = apply_physical_properties(&summary, &assignment).unwrap(); + assert_eq!( + timing_of(&physical_properties, &summary), + ExecutionTiming::QueryTime + ); + assert_eq!( + lifetime_of(&physical_properties, &child(&summary)), + MATERIALIZED_ACROSS + ); } /// A root stored at ingestion time is valid: the query reads the stored @@ -1384,9 +1482,15 @@ mod tests { &root, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&root, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &root), ExecutionTiming::IngestionTime); - assert_eq!(exported(&root, &timed, "Aggregate"), MATERIALIZED_ACROSS); + let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); + assert_eq!( + timing_of(&physical_properties, &root), + ExecutionTiming::IngestionTime + ); + assert_eq!( + exported(&root, &physical_properties, "Aggregate"), + MATERIALIZED_ACROSS + ); let state = agg(scan(), exact_sum()); let root = finalize(Rc::clone(&state)); @@ -1399,10 +1503,13 @@ mod tests { &root, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&root, &assignment).unwrap(); - assert_eq!(timing_of(&timed, &root), ExecutionTiming::IngestionTime); + let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); + assert_eq!( + timing_of(&physical_properties, &root), + ExecutionTiming::IngestionTime + ); assert_eq!( - exported(&root, &timed, "FinalizeExactAccumulator"), + exported(&root, &physical_properties, "FinalizeExactAccumulator"), MATERIALIZED_ACROSS ); diff --git a/crates/types/src/physical/workload_plan.rs b/crates/types/src/physical/workload_plan.rs index 5fb8e5306..ac325e98c 100644 --- a/crates/types/src/physical/workload_plan.rs +++ b/crates/types/src/physical/workload_plan.rs @@ -15,9 +15,8 @@ pub struct PhysicalWorkloadPlan { /// exports. pub roots: Vec, /// The physical properties of every node of `roots`: the completed - /// assignment `dag` exports them with. The facade adds the nodes it - /// rebuilds from them (`with_confirmed_weights_at`), so it also covers the - /// plan it runs. + /// assignment `dag` exports them with. The facade's plan to run, rebuilt + /// with Stage 3's weight proofs, carries its own copy (`SelectedPlan`). pub physical_properties: PhysicalPropertiesAssignment, pub dag: PhysicalASAPDAG, } diff --git a/crates/types/src/test_support.rs b/crates/types/src/test_support.rs index 1fbdbbeab..ea7f02a5f 100644 --- a/crates/types/src/test_support.rs +++ b/crates/types/src/test_support.rs @@ -36,9 +36,9 @@ pub fn tumbling_window( /// `physical_export::compile_physical_asap_dag_with_node_ids`. pub fn compile_physical_asap_dag_with_node_ids( root: &Rc, - timed: &PhysicalPropertiesAssignment, + physical_properties: &PhysicalPropertiesAssignment, ) -> Result { - crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root, timed) + crate::ir::physical_export::compile_physical_asap_dag_with_node_ids(root, physical_properties) } /// `Operator::output_schema`. diff --git a/crates/types/src/workload/bounds.rs b/crates/types/src/workload/bounds.rs index e5d052af8..a6e90b65f 100644 --- a/crates/types/src/workload/bounds.rs +++ b/crates/types/src/workload/bounds.rs @@ -486,9 +486,7 @@ pub fn with_confirmed_weights_at( OperatorNode::with_schema(operator, node.schema.clone()) .with_guarantee(node.guarantee.clone()), ); - if let Some(properties) = physical_properties.get(node) { - physical_properties.set(&rebuilt, properties); - } + physical_properties.copy_assignment(node, &rebuilt); rebuilt } else { Rc::clone(node) diff --git a/crates/types/tests/materialized_view.rs b/crates/types/tests/materialized_view.rs index 63dbeac49..2c81ff371 100644 --- a/crates/types/tests/materialized_view.rs +++ b/crates/types/tests/materialized_view.rs @@ -102,8 +102,8 @@ fn tumbling_windows_and_kept_merge() -> (PhysicalASAPDAG, usize) { &merge, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_QUERY_TIME, ); - let timed = apply_physical_properties(&merge, &assignment).unwrap(); - let dag = compile_physical_asap_dag(&merge, &timed).unwrap(); + let physical_properties = apply_physical_properties(&merge, &assignment).unwrap(); + let dag = compile_physical_asap_dag(&merge, &physical_properties).unwrap(); let merge_id = dag.roots[0]; (dag, merge_id) } @@ -244,9 +244,9 @@ fn three_chains_over_one_scan() -> (PhysicalASAPDAG, PhysicalASAPDAG) { ) }) .collect(); - let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let unexpanded = compile_physical_asap_workload(&roots, &timed).unwrap(); - let expanded = expand_tumbling_window_chains_in_workload(&roots, &timed).unwrap(); + let physical_properties = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); + let unexpanded = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); + let expanded = expand_tumbling_window_chains_in_workload(&roots, &physical_properties).unwrap(); let mut installed = compile_physical_asap_workload(&expanded.roots, &expanded.physical_properties).unwrap(); installed @@ -378,8 +378,8 @@ fn a_shifted_tumbling_window_has_its_own_phase() { .iter() .map(|tumbling_window| QueryRoot::Operator(Rc::clone(tumbling_window))) .collect(); - let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&roots, &timed).unwrap(); + let physical_properties = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); + let mut dag = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); let mut windows: Vec<(u64, u64)> = dag .ingestion_refresh_groups() @@ -423,8 +423,8 @@ fn views_of_unbounded_windows_are_groups_of_their_own() { QueryRoot::Operator(build) }) .collect(); - let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&roots, &timed).unwrap(); + let physical_properties = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); + let mut dag = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); let groups = dag.ingestion_refresh_groups().unwrap(); assert_eq!(groups.len(), 2); @@ -455,8 +455,8 @@ fn a_view_reading_a_view_of_its_group_follows_it() { ); } let roots = [QueryRoot::Operator(Rc::clone(&rollup))]; - let timed = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); - let mut dag = compile_physical_asap_workload(&roots, &timed).unwrap(); + let physical_properties = apply_physical_properties_to_roots(&roots, &assignment).unwrap(); + let mut dag = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); dag.set_retention(&HashMap::new(), &HashMap::new()).unwrap(); let groups = dag.ingestion_refresh_groups().unwrap(); assert_eq!(groups.len(), 1); @@ -505,8 +505,8 @@ fn a_view_has_its_refresh_window_on_a_dag_without_edges() { &view, PhysicalProperties::MATERIALIZED_IN_MEMORY_ACROSS_EVALUATIONS_AT_INGESTION, ); - let timed = apply_physical_properties(&view, &assignment).unwrap(); - let mut dag = compile_physical_asap_dag(&view, &timed).unwrap(); + let physical_properties = apply_physical_properties(&view, &assignment).unwrap(); + let mut dag = compile_physical_asap_dag(&view, &physical_properties).unwrap(); let view_id = dag.roots[0]; let window = Some(RefreshWindow { width_ms: 60_000, diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index 81dccf616..0304f08b0 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -55,22 +55,22 @@ fn timed_at_ingestion() -> (Rc, PhysicalPropertiesAssignment) { root.children()[0], PhysicalProperties::new(ExecutionTiming::IngestionTime, ON_DISK).unwrap(), ); - let timed = apply_physical_properties(&root, &assignment).unwrap(); - (root, timed) + let physical_properties = apply_physical_properties(&root, &assignment).unwrap(); + (root, physical_properties) } /// `timed_at_ingestion()` exported. fn exported_at_ingestion() -> PhysicalASAPDAG { - let (root, timed) = timed_at_ingestion(); - compile_physical_asap_dag(&root, &timed).unwrap() + let (root, physical_properties) = timed_at_ingestion(); + compile_physical_asap_dag(&root, &physical_properties).unwrap() } /// A summary materialized at ingestion time is read at query time, and the /// exported DAG says where to store it. #[test] fn timed_plan_exports_with_timing() { - let (root, timed) = timed_at_ingestion(); - let dag = compile_physical_asap_dag(&root, &timed).unwrap(); + let (root, physical_properties) = timed_at_ingestion(); + let dag = compile_physical_asap_dag(&root, &physical_properties).unwrap(); dag.validate().unwrap(); let timings: Vec<_> = dag.nodes.iter().map(|n| n.output_state.timing).collect(); @@ -147,10 +147,10 @@ fn batch_shares_the_summary_and_keeps_one_root_per_query() { .into_iter() .map(QueryRoot::Operator) .collect(); - let timed = + let physical_properties = apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) .unwrap(); - let dag = compile_physical_asap_workload(&roots, &timed).unwrap(); + let dag = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); dag.validate().unwrap(); assert_eq!(dag.roots.len(), 2); assert_eq!(dag.nodes.len(), 4, "scan and summary are exported once"); @@ -163,7 +163,8 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; let read = plan(); - let timed = apply_physical_properties(&read, &PhysicalPropertiesAssignment::default()).unwrap(); + let physical_properties = + apply_physical_properties(&read, &PhysicalPropertiesAssignment::default()).unwrap(); let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&read))), @@ -175,7 +176,7 @@ fn scalar_roots_export_in_batch_order_and_reference_nodes() { QueryRoot::Scalar(scalar), QueryRoot::Operator(Rc::clone(&read)), ], - &timed, + &physical_properties, ) .unwrap(); assert_eq!(dag.nodes.len(), 3, "the read plan is exported once"); @@ -280,9 +281,9 @@ fn a_reference_read_twice_is_one_scalar_ref_edge() { child: series("a"), })) .unwrap(); - let timed = + let physical_properties = apply_physical_properties(&project, &PhysicalPropertiesAssignment::default()).unwrap(); - let dag = compile_physical_asap_dag(&project, &timed).unwrap(); + let dag = compile_physical_asap_dag(&project, &physical_properties).unwrap(); dag.validate().unwrap(); let roles: Vec<_> = dag .edges @@ -318,8 +319,9 @@ fn swapped_edge_roles_are_rejected() { right: scan("b"), })) .unwrap(); - let timed = apply_physical_properties(&join, &PhysicalPropertiesAssignment::default()).unwrap(); - let mut dag = compile_physical_asap_dag(&join, &timed).unwrap(); + let physical_properties = + apply_physical_properties(&join, &PhysicalPropertiesAssignment::default()).unwrap(); + let mut dag = compile_physical_asap_dag(&join, &physical_properties).unwrap(); dag.validate().unwrap(); for edge in &mut dag.edges { edge.role = match edge.role { @@ -387,8 +389,8 @@ fn exports_summary_over_summary_as_typed_precompute_edges() { let at_ingestion = PhysicalPropertiesAssignment::materialized_at_ingestion_where_allowed( std::slice::from_ref(&root), ); - let timed = apply_physical_properties(&root, &at_ingestion).unwrap(); - let compiled = compile_physical_asap_dag_with_node_ids(&root, &timed).unwrap(); + let physical_properties = apply_physical_properties(&root, &at_ingestion).unwrap(); + let compiled = compile_physical_asap_dag_with_node_ids(&root, &physical_properties).unwrap(); assert_eq!(compiled.node_ids.node_id(&root), Some(3)); assert_eq!(compiled.node_ids.node_id(&inner), Some(1)); let dag = compiled.dag; @@ -491,7 +493,7 @@ fn physical_node_ids_serialize_in_deterministic_binding_order() { fn every_node_at_query_time_keeps_the_operators_and_stores_nothing() { use asap_types::ir::scalar::ArithmeticOpKind; use asap_types::ir::{ExprSemantics, ScalarExpr}; - let workload = |root: Rc, timed: &PhysicalPropertiesAssignment| { + let workload = |root: Rc, physical_properties: &PhysicalPropertiesAssignment| { let scalar = ScalarExpr::Arithmetic { op: ArithmeticOpKind::Add, left: Box::new(ScalarExpr::PromqlScalarFromVector(Rc::clone(&root))), @@ -504,18 +506,19 @@ fn every_node_at_query_time_keeps_the_operators_and_stores_nothing() { QueryRoot::Scalar(scalar), QueryRoot::Operator(root), ], - timed, + physical_properties, ) .unwrap() }; - let (root, timed) = timed_at_ingestion(); - let materialized = workload(root, &timed); + let (root, physical_properties) = timed_at_ingestion(); + let materialized = workload(root, &physical_properties); let at_query_time = plan(); - let timed = apply_physical_properties(&at_query_time, &PhysicalPropertiesAssignment::default()) - .unwrap(); + let physical_properties = + apply_physical_properties(&at_query_time, &PhysicalPropertiesAssignment::default()) + .unwrap(); assert_eq!( materialized.with_every_node_at_query_time().unwrap(), - workload(at_query_time, &timed) + workload(at_query_time, &physical_properties) ); } @@ -564,10 +567,10 @@ fn exported_with_scan( asap_types::ir::physical_export::PhysicalASAPNodeId, ) { let roots: Vec = roots.into_iter().map(QueryRoot::Operator).collect(); - let timed = + let physical_properties = apply_physical_properties_to_roots(&roots, &PhysicalPropertiesAssignment::default()) .unwrap(); - let dag = compile_physical_asap_workload(&roots, &timed).unwrap(); + let dag = compile_physical_asap_workload(&roots, &physical_properties).unwrap(); let scans: Vec<_> = dag .nodes .iter() @@ -755,13 +758,14 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() { #[test] fn workload_export_matches_the_single_root_export() { use asap_types::ir::physical_export::compile_physical_asap_workload_with_node_ids; - let (root, timed) = timed_at_ingestion(); + let (root, physical_properties) = timed_at_ingestion(); let roots = [QueryRoot::Operator(Rc::clone(&root))]; - let exported = compile_physical_asap_workload_with_node_ids(&roots, &timed).unwrap(); + let exported = + compile_physical_asap_workload_with_node_ids(&roots, &physical_properties).unwrap(); assert_eq!( exported.dag, - compile_physical_asap_dag(&root, &timed).unwrap() + compile_physical_asap_dag(&root, &physical_properties).unwrap() ); let untimed = [QueryRoot::Operator(plan())]; - assert!(compile_physical_asap_workload_with_node_ids(&untimed, &timed).is_err()); + assert!(compile_physical_asap_workload_with_node_ids(&untimed, &physical_properties).is_err()); } diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index b48dd0217..523a442f0 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -258,9 +258,10 @@ fn current_series_topk() -> Rc { fn compile_physical_asap_dag(root: &Rc) -> Result<(), String> { use asap_types::ir::{apply_physical_properties, PhysicalPropertiesAssignment}; root.validate_structure().map_err(|e| e.to_string())?; - let timed = apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) - .map_err(|e| format!("{e:?}"))?; - asap_types::ir::physical_export::compile_physical_asap_dag(root, &timed) + let physical_properties = + apply_physical_properties(root, &PhysicalPropertiesAssignment::default()) + .map_err(|e| format!("{e:?}"))?; + asap_types::ir::physical_export::compile_physical_asap_dag(root, &physical_properties) .map_err(|e| format!("{e:?}"))?; Ok(()) } @@ -354,8 +355,8 @@ fn population_timing_follows_its_physical_properties() { ), (at_ingestion, IngestionTime, on_disk), ] { - let timed = apply_physical_properties(&candidate, &assignment).unwrap(); - let dag = export_timed(&candidate, &timed).unwrap(); + let physical_properties = apply_physical_properties(&candidate, &assignment).unwrap(); + let dag = export_timed(&candidate, &physical_properties).unwrap(); let [population, evaluation] = &dag.nodes[dag.nodes.len() - 2..] else { unreachable!() }; diff --git a/crates/types/tests/summary_merge_structure.rs b/crates/types/tests/summary_merge_structure.rs index 02f72cafe..279326129 100644 --- a/crates/types/tests/summary_merge_structure.rs +++ b/crates/types/tests/summary_merge_structure.rs @@ -703,8 +703,8 @@ fn summary_merge_timing() { let tumbling_windows = [state(200, "us"), state(200, "eu")]; let root = merge(tumbling_windows.to_vec()).unwrap(); let timings = |assignment: &PhysicalPropertiesAssignment| { - let timed = apply_physical_properties(&root, assignment).unwrap(); - let timing = |node: &OperatorNode| timed.get(node).unwrap().timing(); + let physical_properties = apply_physical_properties(&root, assignment).unwrap(); + let timing = |node: &OperatorNode| physical_properties.get(node).unwrap().timing(); ( timing(&root), root.children() diff --git a/crates/types/tests/tumbling_window_chain.rs b/crates/types/tests/tumbling_window_chain.rs index 14ec6ad5b..911140f33 100644 --- a/crates/types/tests/tumbling_window_chain.rs +++ b/crates/types/tests/tumbling_window_chain.rs @@ -200,10 +200,10 @@ fn expansion_carries_the_chains_timing_onto_its_tumbling_windows() { if lifetime.is_materialized() { assignment.set(&chain, PhysicalProperties::new(timing, lifetime).unwrap()); } - let timed = apply_physical_properties(&estimate, &assignment).unwrap(); + let physical_properties = apply_physical_properties(&estimate, &assignment).unwrap(); let expansion = expand_tumbling_window_chains_in_workload( &[QueryRoot::Operator(Rc::clone(&estimate))], - &timed, + &physical_properties, ) .unwrap(); let properties = |node: &OperatorNode| expansion.physical_properties.get(node).unwrap(); diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index c8bb69139..9377eee5d 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -108,8 +108,8 @@ happens: **ingestion time** or **query time**. Operator identity must not imply one of these phases. Backend capability restrictions are implementation gaps, not definitions of the operator. -The logical DAG carries no timing: `OperatorNode::timing` is `None` on every -front-end node and every candidate, and `map_children` clears it. +Operator nodes carry no physical properties: the logical DAG has no timing +or lifetime. Materialization ([#509 §Materialization](../proposals/planner-layering.md#materialization)) decides, for each sub-DAG, whether its output is materialized, and if so with which [`PhysicalProperties`](../../../crates/types/src/physical/properties.rs): a @@ -124,8 +124,8 @@ would be materialized. Any node may be named: a summary state, a population, or, #509 Example 4, the tumbling windows a query reads. `set(node, properties)` lists one; an entry whose lifetime is `None` fixes only the node's timing, and every sub-DAG the assignment does not list is not materialized. Stage 2 chooses the -assignment. Operator nodes carry no timing or lifetime (#1200): the assignment is -where physical stages keep them. +assignment. Operator nodes carry no timing or lifetime (#1200): physical +stages read them from the assignment. `apply_physical_properties(root, &assignment)` then completes it into a timing and a lifetime for every node, top-down, and returns the completed assignment: diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index ee15ead49..4bf59e9ea 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -296,7 +296,7 @@ In the worked example, `latency < 100` and `region = 'us'` move into `selection` **When two definitions are equal.** Merging (§5.6) requires equal definitions. -- They must have the same structure. Planning details are ignored: `timing`, `guarantee` and `coverage_cache`. So a tumbling window built at ingestion time can merge with one built at query time. +- They must have the same structure. Planning details are ignored: `guarantee` and `coverage_cache`. Operator nodes carry no timing (#1200), so a tumbling window built at ingestion time can merge with one built at query time. - `SummaryUpdate.weight_domain` is compared too. It is computed from the rest, so it differs only if something is wrong. - States over different tables never merge: a KLL over `m1` and one over `m2` have different definitions. To combine tables, put a `UNION ALL` with a column that marks the source table below one `SummaryAgg`; that column can then be used in `selection` or in the grouping. @@ -921,9 +921,6 @@ pub struct OperatorNode { /// The accuracy statement of the output, once known. `None` does not /// mean exact. pub guarantee: Option, - /// When the node runs: `IngestionTime` or `QueryTime`. `None` until - /// planning assigns it. - pub timing: Option, /// Cache for `coverage()`. Filled on first use, never serialized, /// ignored by equality, emptied on clone. Not a source of truth: /// coverage can always be derived again. diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 5f9deb85f..cb822d7a6 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -1,6 +1,9 @@ # Sharing Operators Between Pre-ASAP IR and Post-ASAP IR > Status: proposal, not implemented. Audience: planner designers and architects. +> Note: since #1200, operator nodes carry no `timing`; physical stages keep each +> node's timing and lifetime in a `PhysicalPropertiesAssignment`, and +> `validate_execution_timing` no longer exists. The `timing` text below predates that. > Addresses [#468](https://github.com/ProjectASAP/ASAPPlanner/issues/468). > Companion: [Decoupling operators from scalar expressions](decoupling_op_and_expr.md).