From da1821162ff7f6b341580a99633ef78d8e6f3232 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:55:34 +0100 Subject: [PATCH] fix: evaluate every rule over the whole project on selected runs A selected plan or build evaluated model-scoped rules on its slice of the project, so a custom rule reading ctx.graph.dependents saw only the selection's dependents and could report findings that sqb compile does not. Selected runs now evaluate built-in and custom rules over the whole project with compile's rules config and every exception, reusing compile's rules cache, and report only findings on selected models and on files that are not models. SQL lint rules still check each selected file on its own. A project that compiles cleanly now passes every selection. Closes CHI-794 --- .../src/bindings/_helpers/rules/constants.rs | 4 +- .../bindings/_helpers/rules/fix_session.rs | 2 +- .../bindings/_helpers/rules/rule_dispatch.rs | 162 +++++------------- .../src/bindings/_helpers/rules/rule_plan.rs | 10 +- .../src/bindings/_helpers/rules/sql_rules.rs | 2 +- .../concepts/rules/execution-and-caching.md | 10 +- .../cli/commands/main/rules/_test_types.py | 12 ++ .../rules/test_selected_project_wide_rules.py | 95 +++++++++- .../concepts/rules/execution-and-caching.mdx | 10 +- 9 files changed, 168 insertions(+), 139 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index eef0addb63..71fe51be7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -18,8 +18,8 @@ pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = pub(crate) const RULES_NATIVE_API_VERSION: i64 = 1; /// File name of the custom-rule result cache under the project cache directory. pub(crate) const CUSTOM_RULES_CACHE_FILE: &str = "custom-rules.json"; -/// Bulk cache subdirectory of a selected run's whole-project pass, kept apart from its slice's. -pub(crate) const WHOLE_PROJECT_CACHE_DIRECTORY: &str = "whole-project"; +/// Code prefix of the SQL lint rules, which check each selected file on its own. +pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; /// Suffix of a custom host scratch directory abandoned by a cancelled run. pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; /// Marker file whose presence tells split custom hosts the run was cancelled. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 98cfc3b12c..04800a40a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -29,6 +29,7 @@ use crate::bindings::_helpers::records::fields::{field, replaced}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::authored_sql::SqlExpansionInputs; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; +use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; use crate::bindings::_helpers::rules::lint_expansion::{ @@ -43,7 +44,6 @@ const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; const APPLIED_REASON: &str = "Meaning-preserving rewrite; compiler invariants verified"; const MAX_PASSES: usize = 128; -const SQL_RULE_PREFIX: &str = "SQBRSQL"; struct FixSession<'py> { project_dir: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 824501ea1d..ff0b48d956 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -30,7 +30,6 @@ use crate::bindings::_helpers::rules::catalogue::{ }; use crate::bindings::_helpers::rules::constants::RULES_MEMO_REUSES; use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; -use crate::bindings::_helpers::rules::constants::WHOLE_PROJECT_CACHE_DIRECTORY; use crate::bindings::_helpers::rules::custom_evaluation::{ CustomRun, evaluate_custom_rules_cached, outcome, }; @@ -123,37 +122,6 @@ struct NativeRun<'py> { verify_determinism: bool, initial_findings: Option>, cancel: HostCancel, - /// Which of the selected custom rules this evaluation runs. - custom_scope: CustomScope, -} - -/// The custom rules one evaluation of a selected run runs, and where it caches. -enum CustomScope { - /// Every selected custom rule, caching in the bulk cache. - All, - /// The slice of a selection: every selected custom rule except these project-wide ones. - Slice(Vec), - /// The whole-project pass of a selection: only these, caching in its own directory. - WholeProject(Vec), -} - -impl CustomScope { - /// Whether this evaluation runs the selected custom rule `code`. - fn runs(&self, code: &String) -> bool { - match self { - Self::All => true, - Self::Slice(skipped) => !skipped.contains(code), - Self::WholeProject(codes) => codes.contains(code), - } - } - - /// The bulk cache subdirectory this evaluation caches in, the bulk cache itself when `None`. - fn cache_directory(&self) -> Option<&'static str> { - match self { - Self::WholeProject(_) => Some(WHOLE_PROJECT_CACHE_DIRECTORY), - Self::All | Self::Slice(_) => None, - } - } } /// How a native evaluation obtains its custom-rule outcome. @@ -199,7 +167,6 @@ fn evaluate_project_rules<'py>( verify_determinism, initial_findings: None, cancel: HostCancel::default(), - custom_scope: CustomScope::All, }; evaluate_native(&run, (false, None), CustomSource::Deferred, None) } @@ -224,7 +191,6 @@ fn evaluate_native_rules<'py>( verify_determinism: arguments.flag("verify_determinism")?, initial_findings: (!initial_findings.is_none()).then_some(initial_findings), cancel: HostCancel::default(), - custom_scope: CustomScope::All, }; evaluate_native( &run, @@ -298,16 +264,24 @@ pub(crate) fn evaluate_rules<'py>( ], )?))?; let whole_codes: Vec = whole_project_codes(&plan)?; + let selection: bool = !field(&plan, "model_paths")?.is_none(); let run: NativeRun<'py> = NativeRun { - project: field(&plan, "project")?, - config: field(&plan, "config")?, + project: if selection { + graph_project.clone() + } else { + field(&plan, "project")? + }, + config: if selection { + whole_project_config(&plan)? + } else { + field(&plan, "config")? + }, project_dir: field(&plan, "project_dir")?, catalogue: field(&plan, "catalogue")?, dialect: dialect.extract()?, verify_determinism: false, initial_findings: None, cancel: HostCancel::default(), - custom_scope: CustomScope::Slice(whole_codes.clone()), }; let payloads: Bound<'py, PyAny> = field(&plan, "custom_payloads")?; let custom: Option = start_custom_rules(&run, &payloads)?; @@ -341,7 +315,6 @@ pub(crate) fn evaluate_rules<'py>( verify_determinism: false, initial_findings: None, cancel: native_cancel, - custom_scope: CustomScope::All, }; evaluate_native( &run, @@ -373,111 +346,68 @@ pub(crate) fn evaluate_rules<'py>( } let sql_result: Bound<'py, PyAny> = sql_result?; let result: Bound<'py, PyAny> = result?; - let result: Bound<'py, PyAny> = if whole_codes.is_empty() { - result + let result: Bound<'py, PyAny> = if selection { + selected_findings(&plan, &whole_codes, &run, &result)? } else { - with_whole_project_rules( - (&plan, &whole_codes), - (&graph_project, &payloads), - (&run, &result), - )? + result }; finish_rule_run(&plan, &sql_result, &result) } -/// `codes` and every selected custom rule, which the custom-rule coverage rule checks. -fn with_custom_codes(plan: &Bound<'_, PyAny>, codes: &[String]) -> PyResult> { - let mut select: Vec = codes.to_vec(); +/// The rules config a whole-project compile evaluates: every selected rule and every exception. +fn whole_project_config<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = plan.py(); + let mut codes: Vec> = Vec::new(); for rule in field(plan, "selected")?.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - let code: String = field(&rule, "code")?.extract()?; - if field(&rule, "custom")?.is_truthy()? && !codes.contains(&code) { - select.push(code); - } + codes.push(field(&rule?, "code")?); } - Ok(select) + replaced( + &field(plan, "effective_config")?, + &[ + ("select", PyTuple::new(py, codes)?.into_any()), + ("ignore", PyTuple::empty(py).into_any()), + ], + ) } -/// `sliced` with `codes` evaluated over the whole project as compile does, kept to the slice. -fn with_whole_project_rules<'py>( - (plan, codes): (&Bound<'py, PyAny>, &[String]), - (project, payloads): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (slice, sliced): (&NativeRun<'py>, &Bound<'py, PyAny>), +/// `whole`'s `codes` findings kept to the selected models and to files that are not models. +fn selected_findings<'py>( + plan: &Bound<'py, PyAny>, + codes: &[String], + run: &NativeRun<'py>, + whole: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = plan.py(); - let select: Vec = with_custom_codes(plan, codes)?; - let whole: NativeRun<'py> = NativeRun { - project: project.clone(), - config: replaced( - &field(plan, "effective_config")?, - &[ - ("select", PyTuple::new(py, select)?.into_any()), - ("ignore", PyTuple::empty(py).into_any()), - ], - )?, - project_dir: slice.project_dir.clone(), - catalogue: slice.catalogue.clone(), - dialect: slice.dialect.clone(), - verify_determinism: false, - initial_findings: None, - cancel: slice.cancel.clone(), - custom_scope: CustomScope::WholeProject(codes.to_vec()), - }; - let evaluated: PyResult> = evaluate_native( - &whole, - (true, Some(payloads.clone())), - CustomSource::Deferred, - None, - ); - cancel_on_interrupt(py, &evaluated, &whole.cancel); - let evaluated: Bound<'py, PyAny> = evaluated?; let finalized: Bound<'py, PyTuple> = finalize_rule_findings_rows( - &whole.config, - &whole.project_dir, + &run.config, + &run.project_dir, codes.to_vec(), - field(&evaluated, "findings")? + field(whole, "findings")? .try_iter()? .collect::>()?, )?; - let project_wide: HashSet<&str> = codes.iter().map(String::as_str).collect(); - let mut findings: Vec> = Vec::new(); - for finding in field(sliced, "findings")?.try_iter()? { - let finding: Bound<'py, PyAny> = finding?; - if !project_wide.contains(field(&finding, "code")?.extract::()?.as_str()) { - findings.push(finding); - } - } let selected: Bound<'py, PyAny> = field(plan, "model_paths")?; - let models: HashSet = project_models(project)? + let models: HashSet = project_models(&run.project)? .into_iter() .map(|model| model.posix) .collect(); + let mut findings: Vec> = Vec::new(); for finding in finalized.iter() { let path: String = rule_finding(&finding)?.path; if !models.contains(&path) || selected.contains(&path)? { findings.push(finding); } } - let total = |name: &str| -> PyResult> { - (field(sliced, name)?.extract::()? + field(&evaluated, name)?.extract::()?) - .into_bound_py_any(py) - }; - new_record_with( - py, - RecordId::RulesResult, + replaced( + whole, &[ ("findings", PyTuple::new(py, findings)?.into_any()), ( "evaluated_models", - project_models(&slice.project)? + project_models(&field(plan, "project")?)? .len() .into_bound_py_any(py)?, ), - ("cache_hits", total("cache_hits")?), - ("cache_misses", total("cache_misses")?), - ("built_in_ms", total("built_in_ms")?), - ("custom_ms", total("custom_ms")?), - ("custom_cpu_ms", total("custom_cpu_ms")?), ], ) } @@ -665,9 +595,6 @@ fn request_options<'py>( ), ], )?; - if let Some(directory) = run.custom_scope.cache_directory() { - options.set_item("cache_directory", directory)?; - } Ok(options) } @@ -686,7 +613,6 @@ pub(crate) fn speculative_rules_request<'py>( verify_determinism: false, initial_findings: None, cancel: HostCancel::default(), - custom_scope: CustomScope::All, }; rules_request_over( &request_options(&run, &field(plan, "custom_payloads")?)?, @@ -710,10 +636,7 @@ fn start_custom_rules( for rule in run.catalogue.try_iter()? { let rule: Bound<'_, PyAny> = rule?; let code: String = field(&rule, "code")?.extract()?; - if field(&rule, "custom")?.is_truthy()? - && selected.contains(&code) - && run.custom_scope.runs(&code) - { + if field(&rule, "custom")?.is_truthy()? && selected.contains(&code) { rules.push(rule.unbind()); } } @@ -742,7 +665,6 @@ fn start_custom_rules( let dialect: String = run.dialect.clone(); let verify_determinism: bool = run.verify_determinism; let cancel: HostCancel = run.cancel.clone(); - let cache_directory: Option<&'static str> = run.custom_scope.cache_directory(); PhaseThread::spawn(CUSTOM_THREAD, move || { Python::attach(|py| { let [project, config, project_dir] = inputs; @@ -755,7 +677,7 @@ fn start_custom_rules( verify_determinism, implementation_fingerprints: fingerprints, cancel, - cache_directory, + cache_directory: None, }) .map(Bound::unbind) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 070528ed50..af2973d9ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::rules::catalogue::{ references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, verify_custom_rules, }; -use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; +use crate::bindings::_helpers::rules::constants::{SQL_RULE_PREFIX, TYPE_PROOF_RULE_CODES}; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; @@ -142,16 +142,16 @@ pub(crate) fn rule_run_plan<'py>( ) } -/// The selected project-wide codes a model selection still evaluates over the whole project. +/// The selected non-SQL-lint codes a model selection evaluates over the whole project. pub(crate) fn whole_project_codes(plan: &Bound<'_, PyAny>) -> PyResult> { let mut codes: Vec = Vec::new(); if field(plan, "model_paths")?.is_none() { return Ok(codes); } for rule in field(plan, "selected")?.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - if field(&rule, "project_wide")?.is_truthy()? { - codes.push(field(&rule, "code")?.extract()?); + let code: String = field(&rule?, "code")?.extract()?; + if !code.starts_with(SQL_RULE_PREFIX) { + codes.push(code); } } Ok(codes) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index eb0982207b..4587763c2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; +use crate::bindings::_helpers::rules::constants::SQL_RULE_PREFIX; use crate::bindings::_helpers::rules::early_sql_lint::{ EarlyLintResult, EarlySqlLint, PreparedSqlLint, sql_lint_settings, sql_rule_cache_path, }; @@ -51,7 +52,6 @@ const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; const PATHLIB_MODULE: &str = "pathlib"; -const SQL_RULE_PREFIX: &str = "SQBRSQL"; const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSQL000"; const DEFAULT_REMEDIATION: &str = "Update the authored SQL to satisfy this rule."; const UTF8: &str = "utf-8"; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md index c74480221a..3088640fdf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -24,10 +24,12 @@ compiler path and reject Rule findings before opening a warehouse connection. `sqb rules run` is intentionally focused. Use it to inspect one exact code or family, but run the normal compiler before treating the complete project as valid. -With `--select`, model-subject Rules evaluate only the selected models. Project-subject Rules, which -compare models with each other, still evaluate the whole project with every exception, as -`sqb compile` does, and then report only their findings on selected models and on files that are not -models. A selection never reports a project-subject finding that `sqb compile` would not. +With a selection (`sqb rules run --select`, `sqb plan`, `sqb build`), built-in and custom Rules still +evaluate the whole project with every exception, as `sqb compile` does, so a Rule that reads the graph +sees every dependency and dependent. The selection only filters the report to findings on selected +models and on files that are not models, so a selection never reports a finding that `sqb compile` +would not, and it reuses the cache `sqb compile` writes. SQL lint Rules check each selected file on its +own. ## Dependency-aware caching diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index b6c548ef83..a8a6b4ddf9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -303,3 +303,15 @@ class SelectedRuleCacheCase: edited_path: str expected_entries: tuple[str, ...] expected_recompile_cache_hits: int + + +@dataclass(frozen=True) +class SelectedGraphRuleCase: + """One selected plan whose graph-reading model Rule must agree with a whole-project compile.""" + + description: str + view_sql: str + selection: tuple[str, ...] + expected_compile_findings: tuple[tuple[str, str], ...] + expected_plan_findings: tuple[tuple[str, str], ...] + expected_plan_exit: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_selected_project_wide_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_selected_project_wide_rules.py index dda87ec3f5..45cb90150b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_selected_project_wide_rules.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_selected_project_wide_rules.py @@ -1,4 +1,4 @@ -"""E2E coverage of project-wide Rules on a selected plan, which must agree with compile.""" +"""E2E coverage of Rules on a selected plan, which must agree with a whole-project compile.""" from __future__ import annotations @@ -11,11 +11,13 @@ from sqlbuild._native import rules_bulk_cache_path from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + SelectedGraphRuleCase, SelectedProjectWideRuleCase, SelectedRuleCacheCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( compile_diagnostics, + custom_rule_source, duckdb_rules_project_toml, rule_cache_hits, run_compile_cli, @@ -23,7 +25,7 @@ write_model_files, ) -_FINDING_LINE: re.Pattern[str] = re.compile(r"^(\S+):\d+:\d+ \[(SQBR\w+)\] ", re.MULTILINE) +_FINDING_LINE: re.Pattern[str] = re.compile(r"^(\S+):\d+:\d+ \[(X?SQBR\w+)\] ", re.MULTILINE) _RETURNS_PATH: str = "models/sales/mart/sales__mart__returns__finance.sql" _REFUNDS_PATH: str = "models/sales/mart/sales__mart__refunds__audit.sql" _ORDERS_PATH: str = "models/sales/mart/sales__mart__orders.sql" @@ -71,6 +73,17 @@ 'reason = "Reviewed qualified variants."\n' ) +_ORDERS_BASE_PATH: str = "models/orders_base.sql" +_PUBLISHED_VIEW_RULE: str = custom_rule_source( + code="XSQBRPUB001", + body=( + ' if model.name != "orders_base":\n' + " return []\n" + ' published: bool = any(item.name == "orders_published" for item in ctx.graph.dependents(model))\n' + " return [] if published else [ctx.finding(subject=model)]\n" + ), +) + @pytest.mark.parametrize( "test_case", @@ -178,5 +191,83 @@ def test_given_compile_cache_when_planning_selection_then_compile_and_plan_share assert rule_cache_hits(recompiled) == test_case.expected_recompile_cache_hits, recompiled.stdout +@pytest.mark.parametrize( + "test_case", + [ + SelectedGraphRuleCase( + description="an unselected dependent still exempts its upstream model", + view_sql='SELECT order_id FROM __ref("orders_base")\n', + selection=("+orders_report",), + expected_compile_findings=(), + expected_plan_findings=(), + expected_plan_exit=0, + ), + SelectedGraphRuleCase( + description="a finding compile reports on a selected model is still reported", + view_sql="SELECT 1 AS order_id\n", + selection=("+orders_report",), + expected_compile_findings=(("XSQBRPUB001", _ORDERS_BASE_PATH),), + expected_plan_findings=(("XSQBRPUB001", _ORDERS_BASE_PATH),), + expected_plan_exit=1, + ), + SelectedGraphRuleCase( + description="a finding on an unselected model is left out", + view_sql="SELECT 1 AS order_id\n", + selection=("orders_published",), + expected_compile_findings=(("XSQBRPUB001", _ORDERS_BASE_PATH),), + expected_plan_findings=(), + expected_plan_exit=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_selection_when_planning_then_graph_reading_model_rules_see_whole_project( + test_case: SelectedGraphRuleCase, + tmp_path: Path, +) -> None: + write_model_files( + project_dir=tmp_path, + toml=duckdb_rules_project_toml( + project_dir=tmp_path, + rules_toml=( + 'select = ["XSQBRPUB001"]\n\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n' + ), + ), + files=( + ("rules/published.py", _PUBLISHED_VIEW_RULE), + (_ORDERS_BASE_PATH, "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n"), + ( + "models/orders_report.sql", + "MODEL (description 'Order report.');\n" + 'SELECT order_id FROM __ref("orders_base")\n', + ), + ( + "models/orders_published.sql", + "MODEL (description 'Published orders.', materialized view);\n" + + test_case.view_sql, + ), + ), + ) + + compiled: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + plans: tuple[subprocess.CompletedProcess[str], ...] = tuple( + run_unevaluated_rules_cli(tmp_path, "plan", "--select", *test_case.selection) + for _ in ("cold", "warm") + ) + + compile_findings: tuple[tuple[str, str], ...] = tuple( + sorted({(str(item["code"]), str(item["path"])) for item in compile_diagnostics(compiled)}) + ) + assert compile_findings == test_case.expected_compile_findings, compiled.stdout + for planned in plans: + plan_output: str = planned.stdout + planned.stderr + plan_findings: tuple[tuple[str, str], ...] = tuple( + sorted({(code, path) for path, code in _FINDING_LINE.findall(plan_output)}) + ) + assert plan_findings == test_case.expected_plan_findings, plan_output + assert planned.returncode == test_case.expected_plan_exit, plan_output + assert "Traceback" not in plan_output, plan_output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx index 8f5d93a55d..7d9170ac54 100644 --- a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx +++ b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx @@ -21,10 +21,12 @@ compiler path and reject Rule findings before opening a warehouse connection. `sqb rules run` is intentionally focused. Use it to inspect one exact code or family, but run the normal compiler before treating the complete project as valid. -With `--select`, model-subject Rules evaluate only the selected models. Project-subject Rules, which -compare models with each other, still evaluate the whole project with every exception, as -`sqb compile` does, and then report only their findings on selected models and on files that are not -models. A selection never reports a project-subject finding that `sqb compile` would not. +With a selection (`sqb rules run --select`, `sqb plan`, `sqb build`), built-in and custom Rules still +evaluate the whole project with every exception, as `sqb compile` does, so a Rule that reads the graph +sees every dependency and dependent. The selection only filters the report to findings on selected +models and on files that are not models, so a selection never reports a finding that `sqb compile` +would not, and it reuses the cache `sqb compile` writes. SQL lint Rules check each selected file on its +own. ## Dependency-aware caching