From 3bb5a6e11a3171daf0a3e0a7b0d803451ae8b9f3 Mon Sep 17 00:00:00 2001
From: zzylol <50204836+zzylol@users.noreply.github.com>
Date: Mon, 2 Nov 2026 22:13:31 +0000
Subject: [PATCH] test: ignore every test whose result depends on a wall-clock
 budget or deadline (#1067)

Follows #1166. A search budget (wall_time, time_limit_ms) or a short real
deadline (Instant::now() + a few ms or s) decides these tests' outcome on a
slow or loaded machine; e.g. structural_seed_timeout_preserves_validated_incumbent
sleeps 1.1 s against a 1 s budget and fails under load. Stage 3 is being
replaced (#1095), and #1067 moves to optimal-only stopping; until then they
are ignored. Tests whose deadline is already expired (Instant::now(),
Duration::ZERO) or whose fake evaluator never reaches its budget stay.
---
 crates/executor/src/physical_planner/mod.rs                | 2 ++
 crates/integration-tests/tests/finer_aggregate_reuse.rs    | 1 +
 crates/integration-tests/tests/identical_exact_queries.rs  | 2 ++
 crates/integration-tests/tests/independent_plan_seed.rs    | 3 +++
 crates/integration-tests/tests/joint_search_equivalence.rs | 7 +++++++
 crates/integration-tests/tests/memo_enumeration.rs         | 1 +
 .../integration-tests/tests/production_rollup_workloads.rs | 4 ++++
 .../integration-tests/tests/sharing_issue_regressions.rs   | 1 +
 .../src/pass2/memo/shared_configurations.rs                | 1 +
 crates/plan-selection/src/search/refinement/planner.rs     | 6 ++++++
 crates/plan-selection/src/search/refinement/search.rs      | 2 ++
 crates/planner/tests/runtime_install.rs                    | 2 ++
 crates/types/src/ir/flat.rs                                | 1 +
 crates/types/tests/pane_chain.rs                           | 2 ++
 crates/types/tests/physical_export.rs                      | 1 +
 15 files changed, 36 insertions(+)

diff --git a/crates/executor/src/physical_planner/mod.rs b/crates/executor/src/physical_planner/mod.rs
index a6b66ff6e..a7eac13a0 100644
--- a/crates/executor/src/physical_planner/mod.rs
+++ b/crates/executor/src/physical_planner/mod.rs
@@ -1622,6 +1622,7 @@ mod compilation_deadline_tests {
     }
 
     #[test]
+    #[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
     fn expired_compilation_is_distinct_from_invalid_graph() {
         let dag = empty_dag();
         assert!(matches!(
@@ -1755,6 +1756,7 @@ mod compilation_deadline_tests {
     }
 
     #[test]
+    #[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
     fn large_compilation_observes_deadline_during_preflight() {
         use planner_types::ir::properties::ExecutionDataState;
         use planner_types::physical::Materialization;
diff --git a/crates/integration-tests/tests/finer_aggregate_reuse.rs b/crates/integration-tests/tests/finer_aggregate_reuse.rs
index f3fc21f49..f8791fd06 100644
--- a/crates/integration-tests/tests/finer_aggregate_reuse.rs
+++ b/crates/integration-tests/tests/finer_aggregate_reuse.rs
@@ -330,6 +330,7 @@ fn cost(queries: &[String]) -> f64 {
 /// Example B end to end: adding the filtered sum to the finer sums costs
 /// only its read of the finer result, not a second scan and aggregation.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn filtered_sum_costs_little_beyond_the_finer_sums() {
     let finer =
         format!("SELECT city, host, SUM(value) FROM metrics WHERE {WINDOW} GROUP BY city, host");
diff --git a/crates/integration-tests/tests/identical_exact_queries.rs b/crates/integration-tests/tests/identical_exact_queries.rs
index 800cc4783..17ec2a164 100644
--- a/crates/integration-tests/tests/identical_exact_queries.rs
+++ b/crates/integration-tests/tests/identical_exact_queries.rs
@@ -53,6 +53,7 @@ fn cost(w: &FuzzWorkload) -> f64 {
 
 /// #922: two identical exact percentile queries pay for their result once.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn identical_exact_percentiles_share_work() {
     let related = workload();
     let mut base = related.clone();
@@ -67,6 +68,7 @@ fn identical_exact_percentiles_share_work() {
 
 /// Different ranks share the source window but still compute their own values.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn different_exact_percentiles_share_the_input_window() {
     let mut related = workload();
     related.queries[1].entry.query.0 = related.queries[1].entry.query.0.replace("0.99", "0.5");
diff --git a/crates/integration-tests/tests/independent_plan_seed.rs b/crates/integration-tests/tests/independent_plan_seed.rs
index 316756fdf..f7b31f130 100644
--- a/crates/integration-tests/tests/independent_plan_seed.rs
+++ b/crates/integration-tests/tests/independent_plan_seed.rs
@@ -82,6 +82,7 @@ fn assert_no_costlier_than_separate(queries: &[String]) -> (f64, f64) {
 /// chain; together the search used to stop at a query-time plan about twice
 /// as costly.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn per_city_percentiles_cost_no_more_than_separate_plans() {
     assert_no_costlier_than_separate(&[
         format!("SELECT {P90} FROM metrics WHERE city = 'NYC' AND {W5}"),
@@ -94,6 +95,7 @@ fn per_city_percentiles_cost_no_more_than_separate_plans() {
 /// ingestion-time state; together the search used to stop at a query-time
 /// plan almost three times as costly.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn percentile_and_max_over_one_scan_cost_no_more_than_separate_plans() {
     assert_no_costlier_than_separate(&[
         format!("SELECT {P90} FROM metrics WHERE {W5}"),
@@ -103,6 +105,7 @@ fn percentile_and_max_over_one_scan_cost_no_more_than_separate_plans() {
 
 /// The same with a grouped count over the shared scan.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn percentile_and_grouped_count_over_one_scan_cost_no_more_than_separate_plans() {
     assert_no_costlier_than_separate(&[
         format!("SELECT {P90} FROM metrics WHERE {W5}"),
diff --git a/crates/integration-tests/tests/joint_search_equivalence.rs b/crates/integration-tests/tests/joint_search_equivalence.rs
index 7749fd077..5a5b47361 100644
--- a/crates/integration-tests/tests/joint_search_equivalence.rs
+++ b/crates/integration-tests/tests/joint_search_equivalence.rs
@@ -140,6 +140,7 @@ fn run_workload(
 // validated output semantics here, not inventory sizes from different catalogs.
 // Fixed-catalog terminal/producer scaling is tested in candidates::scopes.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn repeated_queries_preserve_all_outputs_and_exact_guarantees() {
     let settings = CostInputs::default();
     let deployment = DeploymentCapabilities::default();
@@ -186,6 +187,7 @@ fn different_windows_and_raw_retention_policies_produce_valid_plans() {
 
 // Factoring a fused TopK candidate must not absorb the separately requested sum.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn fused_top_k_preserves_its_independently_requested_inner_sum() {
     let workload = workload_of(
         &[
@@ -203,6 +205,7 @@ fn fused_top_k_preserves_its_independently_requested_inner_sum() {
 
 // Grouped rollups coexist with temporal quantiles without merging distinct outputs.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn sketch_rollup_coexists_with_other_quantile_outputs() {
     let workload = workload_of(
         &[
@@ -221,6 +224,7 @@ fn sketch_rollup_coexists_with_other_quantile_outputs() {
 
 // Local sketch contracts do not replace composed output accuracy validation.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn composed_quantile_and_sum_outputs_are_validated_together() {
     let workload = workload_of(
         &[
@@ -288,6 +292,7 @@ fn zero_budget_returns_no_unvalidated_plan() {
 
 // The SQL frontend keeps repeated and different-window COUNT DISTINCT outputs.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn repeated_sql_queries_preserve_separate_output_references() {
     let (mut workload, catalog) = planner_layering_common::example2();
     let entries = workload.query_workload.repeating_queries.as_mut().unwrap();
@@ -311,6 +316,7 @@ fn repeated_sql_queries_preserve_separate_output_references() {
 // Request work on literal outputs still consumes CPU and latency budgets even
 // when there is no operator node to which a fee could be attributed.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn rootless_request_overhead_respects_capacity_and_latency() {
     let mut workload = workload_of(&[("1", 1)], AccuracyTarget::exact());
     let roots = lower_promql_roots(&workload).unwrap();
@@ -346,6 +352,7 @@ fn rootless_request_overhead_respects_capacity_and_latency() {
 // Unsupported exact operators cannot become accepted merely because no summary
 // candidate exists; failure must leave the exported physical plan absent.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn unsupported_exact_operation_never_returns_an_unvalidated_plan() {
     let workload = workload_of(
         &[("stddev_over_time(m[5m])", 60_000)],
diff --git a/crates/integration-tests/tests/memo_enumeration.rs b/crates/integration-tests/tests/memo_enumeration.rs
index f331894f4..61c28df7d 100644
--- a/crates/integration-tests/tests/memo_enumeration.rs
+++ b/crates/integration-tests/tests/memo_enumeration.rs
@@ -338,6 +338,7 @@ fn assert_pass1_memo_and_whole_workload_plans(
 /// The example's memo after each production step, and the `accuracy_budget_parts`
 /// of the state-configuration rule's rewrites (one test: one planning run).
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn ratio_of_quantiles_production_memo() {
     // With the built-in model, Pass 1 prunes every summary of the quantile
     // groups: no query has a guarantee through `/` and `+` over a KLL rank
diff --git a/crates/integration-tests/tests/production_rollup_workloads.rs b/crates/integration-tests/tests/production_rollup_workloads.rs
index 6e2fe94f6..1dd0e5cfa 100644
--- a/crates/integration-tests/tests/production_rollup_workloads.rs
+++ b/crates/integration-tests/tests/production_rollup_workloads.rs
@@ -165,6 +165,7 @@ async fn sql(queries: &[String], accuracy: AccuracyTarget) -> PlanningRun {
 }
 
 #[tokio::test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 async fn production_three_sql_groupings_share_kll() {
     let accuracy = AccuracyTarget::max_error(0.05)
         .failure_probability(0.05)
@@ -175,6 +176,7 @@ async fn production_three_sql_groupings_share_kll() {
 }
 
 #[tokio::test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 async fn production_sql_distinct_region_service_rollup() {
     let accuracy = AccuracyTarget::max_error(0.05)
         .failure_probability(0.05)
@@ -185,6 +187,7 @@ async fn production_sql_distinct_region_service_rollup() {
 }
 
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 // The earlier controlled fixture omitted raw retention. This production
 // workload explicitly holds one current sample for each of its 10,000 series.
 fn production_promql_compares_rollup_with_exact() {
@@ -277,6 +280,7 @@ fn independent_kll_cost(
 // Exercise multi-input operators through the production route under more than
 // three heterogeneous query schedules, not just the local pricing kernel.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn production_binary_queries_use_joint_prices_with_mixed_schedules() {
     use asap_types::workload::{DemandEstimate, Rate};
     let accuracy = AccuracyTarget::exact();
diff --git a/crates/integration-tests/tests/sharing_issue_regressions.rs b/crates/integration-tests/tests/sharing_issue_regressions.rs
index ca64962f9..f2c56e230 100644
--- a/crates/integration-tests/tests/sharing_issue_regressions.rs
+++ b/crates/integration-tests/tests/sharing_issue_regressions.rs
@@ -489,6 +489,7 @@ fn sql_count_rollup_executes_with_integer_results() {
 
 /// #922: duplicate exact TopK readers share their result and inner window work.
 #[test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 fn duplicate_exact_topk_shares_work() {
     let mut w = workload(924);
     for query in &mut w.queries {
diff --git a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs b/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs
index 1c9a5ff8f..7a462325d 100644
--- a/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs
+++ b/crates/logical-asap-optimization/src/pass2/memo/shared_configurations.rs
@@ -574,6 +574,7 @@ mod tests {
     // Cancellation is checked during divisor enumeration, not only between
     // configurations: this grid cannot be fully factored within the budget.
     #[test]
+    #[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
     fn large_grid_enumeration_honors_deadline() {
         let deadline = std::time::Instant::now() + std::time::Duration::from_millis(2);
         assert!(
diff --git a/crates/plan-selection/src/search/refinement/planner.rs b/crates/plan-selection/src/search/refinement/planner.rs
index 002196ab4..75b7ca950 100644
--- a/crates/plan-selection/src/search/refinement/planner.rs
+++ b/crates/plan-selection/src/search/refinement/planner.rs
@@ -1537,6 +1537,7 @@ mod tests {
     // cannot establish an exact top-k ranking; contract-aware priorities find a
     // validated seed without changing the feasible candidate catalog.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn accuracy_baseline_validates_composed_value_and_topk_workload() {
         let target = AccuracyTarget::max_error(0.05).failure_probability(0.3);
         let roots: Vec<_> = [
@@ -2042,6 +2043,7 @@ mod tests {
 
     // Both scalar price objectives use the full evaluator, with one frozen calibration.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn real_kll_search_cost_matches_independent_plan_selector_evaluation() {
         let (configured, roots, entries, data) = fixture();
         for provisioned in [false, true] {
@@ -2107,6 +2109,7 @@ mod tests {
 
     // A one-evaluation budget returns the validated full plan and an unfinished status.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn bounded_search_keeps_its_validated_plan() {
         let (configured, roots, entries, data) = fixture();
         let context = CandidatePlanningContext {
@@ -2137,6 +2140,7 @@ mod tests {
     // Workload eligibility can reject one offered configuration; exclude only
     // that complete assignment and continue to a validated legal configuration.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn rejected_materialization_does_not_stop_search_for_legal_plan() {
         let (configured, roots, mut entries, data) = fixture();
         // The catalog was generated for predictable readers. This deliberately
@@ -2217,6 +2221,7 @@ mod tests {
 
     // A broken output contract is an adapter error, never evidence of infeasibility.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn unsupported_output_contract_stops_without_an_infeasibility_cut() {
         let (configured, mut roots, entries, data) = fixture();
         let QueryRoot::Operator(node) = &mut roots[0] else {
@@ -2254,6 +2259,7 @@ mod tests {
 
     // A cluster objective must not silently become scalar price optimization.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn scalar_price_search_does_not_scalarize_cluster_objectives() {
         let (configured, roots, entries, data) = fixture();
         let settings = asap_cost::CostInputs {
diff --git a/crates/plan-selection/src/search/refinement/search.rs b/crates/plan-selection/src/search/refinement/search.rs
index f9320572f..b255b44a4 100644
--- a/crates/plan-selection/src/search/refinement/search.rs
+++ b/crates/plan-selection/src/search/refinement/search.rs
@@ -1214,6 +1214,7 @@ mod tests {
 
     // A baseline validation that overruns the shared deadline still retains its plan.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn structural_seed_timeout_preserves_validated_incumbent() {
         let result = search_candidates_with_baseline(
             &choices(&[0.0]),
@@ -1394,6 +1395,7 @@ mod tests {
     }
     // The callback cannot be preempted, but a late validated result remains usable.
     #[test]
+    #[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
     fn callback_overrun_preserves_validated_incumbent() {
         let result = search_candidates(
             &choices(&[1.0]),
diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs
index 2690a27cf..c98e8b7e0 100644
--- a/crates/planner/tests/runtime_install.rs
+++ b/crates/planner/tests/runtime_install.rs
@@ -1989,6 +1989,7 @@ async fn a_non_negative_gauges_merged_ddsketch_panes_carry_the_relative_bound_th
 /// reason (`NoErrorBoundForQueryTarget`) and Stage 2 the exact candidates,
 /// which scan raw data at query time.
 #[tokio::test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 async fn an_undeclared_gauges_merged_ddsketch_panes_get_no_relative_bound_through_avg() {
     let error = plan_p99_avg(false).await.expect_err("no valid plan");
     assert_plan_selection_selected_none(&error);
@@ -2207,6 +2208,7 @@ fn exact_average_of_p99s(samples: &[RawDataSample]) -> BTreeMap<String, f64> {
 /// a guarantee, and it runs: equal to its query-time reference and within a
 /// loose tolerance of the exact answer.
 #[tokio::test]
+#[ignore = "flaky: depends on wall-clock search budgets (#1067)"]
 async fn ignoring_accuracy_guarantees_selects_a_sketch_plan_that_runs() {
     let workload = average_of_p99s_workload();
     let capabilities = DeploymentCapabilities {
diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs
index e35b9cf3c..cfc631f55 100644
--- a/crates/types/src/ir/flat.rs
+++ b/crates/types/src/ir/flat.rs
@@ -164,6 +164,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
     fn bounded_flatten_preserves_identity_and_abandons_partial_export() {
         use std::time::{Duration, Instant};
         let shared = values();
diff --git a/crates/types/tests/pane_chain.rs b/crates/types/tests/pane_chain.rs
index 35a78bcd1..48e7d500c 100644
--- a/crates/types/tests/pane_chain.rs
+++ b/crates/types/tests/pane_chain.rs
@@ -434,6 +434,7 @@ fn a_rollup_of_the_newest_panes_covers_their_window() {
 /// A fine-width chain remains a legal candidate; interrupted expansion is a
 /// typed budget stop, never a malformed chain or a partially successful plan.
 #[test]
+#[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
 fn fine_pane_expansion_stops_at_the_planning_deadline() {
     use asap_types::ir::operator::pane_chain::PaneChainExpansionError;
     use asap_types::ir::QueryRoot;
@@ -463,6 +464,7 @@ fn fine_pane_expansion_stops_at_the_planning_deadline() {
 
 /// Supplying ample budget preserves complete expansion and shared pane identity.
 #[test]
+#[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
 fn budgeted_expansion_preserves_every_pane_and_shared_reader() {
     use asap_types::ir::QueryRoot;
     use asap_types::test_support::expand_pane_chains_in_workload_before;
diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs
index 4efef66bd..158a6edc0 100644
--- a/crates/types/tests/physical_export.rs
+++ b/crates/types/tests/physical_export.rs
@@ -768,6 +768,7 @@ fn high_fan_in_edges_preserve_multiplicity_without_order_dependence() {
 }
 
 #[test]
+#[ignore = "flaky: depends on a wall-clock deadline (#1067)"]
 fn physical_export_deadline_is_typed_and_preserves_unbounded_behavior() {
     use asap_types::ir::physical_export::PhysicalASAPCompilationError;
     use asap_types::test_support::compile_physical_asap_workload_with_node_ids_before;