diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index eb008b7177..e0fe2aa3ab 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -100,8 +100,10 @@ jobs: runner: ubuntu-24.04-arm target: aarch64 - name: macos-x86_64 - runner: macos-15-intel + runner: macos-15 target: x86_64 + python-architecture: x64 + smoke-arch: x86_64 - name: macos-arm64 runner: macos-15 target: aarch64 @@ -116,6 +118,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" + architecture: ${{ matrix.platform.python-architecture }} - uses: dtolnay/rust-toolchain@master if: inputs.prebuilt-run-id == '' with: @@ -144,6 +147,8 @@ jobs: github-token: ${{ github.token }} - name: Smoke test installed wheels and native console entry points shell: bash + env: + SMOKE_ARCH: ${{ matrix.platform.smoke-arch }} run: | python -m venv "${RUNNER_TEMP}/wheel-smoke" if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then @@ -151,13 +156,19 @@ jobs: else smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python" fi - "${smoke_python}" -m pip install --find-links dist dist/*.whl + # A cross-compiled wheel runs under Rosetta; the cached Python is universal, so force + # the wheel's architecture or it starts as arm64 and rejects the wheel. + run_as=() + if [[ -n "${SMOKE_ARCH}" ]]; then + run_as=(arch "-${SMOKE_ARCH}") + fi + "${run_as[@]}" "${smoke_python}" -m pip install --find-links dist dist/*.whl cd "${RUNNER_TEMP}" - "${smoke_python}" -c "import sqlbuild._native" + "${run_as[@]}" "${smoke_python}" -c "import sqlbuild._native" smoke_scripts="$(dirname "${smoke_python}")" - "${smoke_scripts}/sqb" --version - "${smoke_scripts}/sqb" --help - "${smoke_scripts}/sqlbuild" --help + "${run_as[@]}" "${smoke_scripts}/sqb" --version + "${run_as[@]}" "${smoke_scripts}/sqb" --help + "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help - name: Check wheel contents and run a native command if: matrix.platform.name == 'linux-x86_64' run: | diff --git a/Makefile b/Makefile index c1725a5a3d..2451eea424 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py diff --git a/crates/sqlbuild-cli/src/dispatch/main/closing_status.rs b/crates/sqlbuild-cli/src/dispatch/main/closing_status.rs new file mode 100644 index 0000000000..53b8d3a897 --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/main/closing_status.rs @@ -0,0 +1,13 @@ +//! The closing line after an incomplete-delivery notice, so the command's own outcome is the last word. + +/// `sqb `'s outcome by exit code, naming the only thing a preceding notice affects. +#[must_use] +pub fn closing_status(command: &str, exit_code: Option, affected: &str) -> String { + match exit_code { + Some(0) => { + format!("sqb {command} completed successfully; only its {affected} is affected.\n") + } + Some(code) => format!("sqb {command} failed with exit code {code}.\n"), + None => format!("sqb {command} failed.\n"), + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/main/mod.rs b/crates/sqlbuild-cli/src/dispatch/main/mod.rs index d96f4ca211..7cc59e2303 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/mod.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/mod.rs @@ -1,3 +1,4 @@ +pub mod closing_status; pub mod command_dispatch; pub mod event_export_warning; pub mod failure_report; diff --git a/crates/sqlbuild-cli/src/dispatch/tests.rs b/crates/sqlbuild-cli/src/dispatch/tests.rs index 2844cc4653..1c47579fbf 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_closing_status.rs"] +mod closing_status; #[path = "tests/test_command_dispatch.rs"] mod command_dispatch; #[path = "tests/test_event_export_warning.rs"] diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs new file mode 100644 index 0000000000..188b2b2071 --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs @@ -0,0 +1,32 @@ +use crate::dispatch::main::closing_status::closing_status; +use crate::dispatch::tests::test_types::ClosingStatusTestCase; + +#[test] +fn given_command_outcome_when_closing_a_notice_then_the_outcome_is_the_last_line() { + let test_cases = [ + ClosingStatusTestCase { + description: "success names the only affected output", + exit_code: Some(0), + expected_status: "sqb audit completed successfully; only its lifecycle event export \ + is affected.\n", + }, + ClosingStatusTestCase { + description: "failure reports the exit code", + exit_code: Some(2), + expected_status: "sqb audit failed with exit code 2.\n", + }, + ClosingStatusTestCase { + description: "unknown exit code reports failure", + exit_code: None, + expected_status: "sqb audit failed.\n", + }, + ]; + for test_case in test_cases { + assert_eq!( + closing_status("audit", test_case.exit_code, "lifecycle event export"), + test_case.expected_status, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs index a075ba8d89..7f2415c4b8 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs @@ -18,3 +18,10 @@ pub(super) struct EventExportWarningTestCase { pub(super) command_succeeded: bool, pub(super) expected_warning: Option<&'static str>, } + +/// A command, its exit code and what a preceding notice affects, and the closing line. +pub(super) struct ClosingStatusTestCase { + pub(super) description: &'static str, + pub(super) exit_code: Option, + pub(super) expected_status: &'static str, +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs index c956dd94fe..ef22baf413 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs @@ -3,12 +3,12 @@ use std::collections::BTreeMap; use std::path::Path; -use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config; use sqlbuild_rules::models::{RuleMetadata, RulesConfig}; use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::main::string_tuple_repr::string_tuple_repr; use sqlbuild_rules::rule_catalogue::models::{ CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleListing, RuleText, }; @@ -18,21 +18,31 @@ use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; use crate::native_commands::_helpers::project_facts::has_custom_rules; use crate::native_commands::constants::{ - EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, PROJECT_SUBJECT, - RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, + DICT_TYPE_NAME, EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, + PROJECT_SUBJECT, RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, RulesAction, RulesOptions}; -/// `RuleOption.boolean(...)` or `RuleOption.string_list(...)` as the catalogue validates it. +/// `RuleOption.boolean(...)`, `.string_list(...)` or a table list, as the catalogue validates it. fn option_spec(option: &BuiltinOption) -> OptionSpec { let (value_type, default, item_type) = match option.kind { BuiltinOptionKind::Boolean => (OptionKind::Bool, OptionValue::Bool(false), None), - BuiltinOptionKind::StringList => ( + BuiltinOptionKind::StringList(defaults) => ( OptionKind::Tuple, - OptionValue::Tuple(Vec::new()), + OptionValue::Tuple( + defaults + .iter() + .map(|item| OptionValue::Str((*item).to_owned())) + .collect(), + ), Some(OptionKind::Str), ), + BuiltinOptionKind::TableList => ( + OptionKind::Tuple, + OptionValue::Tuple(Vec::new()), + Some(OptionKind::Other(DICT_TYPE_NAME.to_owned())), + ), }; OptionSpec { name: option.name.to_owned(), @@ -49,15 +59,16 @@ fn option_spec(option: &BuiltinOption) -> OptionSpec { /// The option as `rules show` prints it: the `repr` of its default, which is also effective. fn option_text(option: &BuiltinOption) -> OptionText { - let default_repr: &str = match option.kind { - BuiltinOptionKind::Boolean => FALSE_REPR, - BuiltinOptionKind::StringList => EMPTY_TUPLE_REPR, + let default_repr: String = match option.kind { + BuiltinOptionKind::Boolean => FALSE_REPR.to_owned(), + BuiltinOptionKind::StringList(defaults) => string_tuple_repr(defaults), + BuiltinOptionKind::TableList => EMPTY_TUPLE_REPR.to_owned(), }; OptionText { name: option.name.to_owned(), description: option.description.to_owned(), - default_repr: default_repr.to_owned(), - effective_repr: default_repr.to_owned(), + effective_repr: default_repr.clone(), + default_repr, } } @@ -155,7 +166,7 @@ pub(crate) fn run_rules( return Err(PythonNeeded::Project); } let config: RulesConfig = load_rules_config(project_dir).map_err(|_| PythonNeeded::Failure)?; - if forbidden_exceptions(&config).is_some() || !config.rule_options.is_empty() { + if !config.rule_options.is_empty() { return Err(PythonNeeded::Project); } let rules: Vec = sorted_catalogue()?; diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 5b1dd8ff84..0d395432ff 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -63,6 +63,8 @@ pub(crate) const PROJECT_SUBJECT: &str = "project"; /// Python `repr` of the built-in option defaults: `False` and `()`. pub(crate) const FALSE_REPR: &str = "False"; pub(crate) const EMPTY_TUPLE_REPR: &str = "()"; +/// The Python type name of each `allowed_qualifiers`-style table option item. +pub(crate) const DICT_TYPE_NAME: &str = "dict"; pub(crate) const UNKNOWN_RULE_PREFIX: &str = "Unknown rule: "; /// The whitespace `int()` strips once other spaces are ASCII spaces. pub(crate) const INT_ASCII_SPACES: &str = " \t\n\x0b\x0c\r"; diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index 840e99a048..b7a0f2b172 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -210,4 +210,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["query"], }, }, + TargetField { + group: None, + name: "concurrency", + rule: MergeRule::Overlay { + decided_by: &["concurrency"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs new file mode 100644 index 0000000000..16a6aa60eb --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs @@ -0,0 +1,28 @@ +//! `effective_concurrency`: the worker limit of one command and the layer it came from. + +use crate::effective::models::{ConcurrencyLayers, ConcurrencySource, EffectiveConcurrency}; + +/// `--concurrency`, else the target's `concurrency`, else the local-then-project setting. +#[must_use] +pub fn effective_concurrency(layers: &ConcurrencyLayers<'_>) -> EffectiveConcurrency { + if let Some(limit) = layers.requested { + return EffectiveConcurrency { + limit, + source: ConcurrencySource::CommandLine, + }; + } + if let Some((target, limit)) = layers.target { + return EffectiveConcurrency { + limit, + source: ConcurrencySource::Target(target.to_owned()), + }; + } + EffectiveConcurrency { + limit: layers.settings, + source: if layers.local_override { + ConcurrencySource::LocalSettings + } else { + ConcurrencySource::ProjectSettings + }, + } +} diff --git a/crates/sqlbuild-config/src/effective/main/mod.rs b/crates/sqlbuild-config/src/effective/main/mod.rs index 80f728929d..1e419479b5 100644 --- a/crates/sqlbuild-config/src/effective/main/mod.rs +++ b/crates/sqlbuild-config/src/effective/main/mod.rs @@ -1,4 +1,5 @@ pub mod default_connection_name; +pub mod effective_concurrency; pub mod effective_connection; pub mod effective_settings; pub mod effective_target_name; diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs index 34cb57057d..d26481562b 100644 --- a/crates/sqlbuild-config/src/effective/models.rs +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -80,3 +80,43 @@ pub struct TargetChoice { pub default: Option, pub defined: Vec, } + +/// The concurrency each configuration layer sets, highest precedence first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ConcurrencyLayers<'a> { + /// `--concurrency` on the command line. + pub requested: Option, + /// The selected target's name and its `concurrency`, when it sets one. + pub target: Option<(&'a str, i64)>, + /// `[settings] concurrency` after any `sqlbuild_local.toml` override. + pub settings: i64, + /// Whether `sqlbuild_local.toml` `[settings]` overrides `concurrency`. + pub local_override: bool, +} + +/// Where the effective concurrency came from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConcurrencySource { + CommandLine, + Target(String), + LocalSettings, + ProjectSettings, +} + +impl std::fmt::Display for ConcurrencySource { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::CommandLine => formatter.write_str("--concurrency"), + Self::Target(name) => write!(formatter, "targets.{name}.concurrency"), + Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"), + Self::ProjectSettings => formatter.write_str("settings.concurrency"), + } + } +} + +/// One command's worker limit and the layer that set it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EffectiveConcurrency { + pub limit: i64, + pub source: ConcurrencySource, +} diff --git a/crates/sqlbuild-config/src/effective/tests/test_effective.rs b/crates/sqlbuild-config/src/effective/tests/test_effective.rs index e4c38903f3..dd2ffc3b5f 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_effective.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_effective.rs @@ -1,17 +1,18 @@ use crate::effective::errors::EffectiveConfigError; use crate::effective::main::default_connection_name::default_connection_name; +use crate::effective::main::effective_concurrency::effective_concurrency; use crate::effective::main::effective_connection::effective_connection; use crate::effective::main::effective_settings::effective_settings; use crate::effective::main::effective_target_name::effective_target_name; use crate::effective::main::effective_vars::effective_vars; use crate::effective::main::merged_target::merged_target; use crate::effective::models::{ - ConnectionChoice, ConnectionLayers, FieldValue, TargetChoice, TargetLayer, + ConcurrencyLayers, ConnectionChoice, ConnectionLayers, FieldValue, TargetChoice, TargetLayer, }; use crate::effective::tests::helpers::{entries, field_value, local_layer, project_layer}; use crate::effective::tests::test_types::{ - ConnectionTestCase, DefaultConnectionTestCase, MergeTestCase, MessageTestCase, Pairs, - TargetNameTestCase, UnionTestCase, + ConcurrencyTestCase, ConnectionTestCase, DefaultConnectionTestCase, MergeTestCase, + MessageTestCase, Pairs, TargetNameTestCase, UnionTestCase, }; use crate::effective::types::Entries; @@ -295,3 +296,60 @@ fn given_ambiguous_connection_when_rendering_then_names_join_into_the_python_mes ); } } + +#[test] +fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings() { + let test_cases = [ + ConcurrencyTestCase { + description: "the command line beats the target", + requested: Some(2), + target: Some(("prod", 8)), + settings: 16, + local_override: true, + expected_limit: 2, + expected_source: "--concurrency", + }, + ConcurrencyTestCase { + description: "the target beats a local settings override", + requested: None, + target: Some(("prod", 8)), + settings: 4, + local_override: true, + expected_limit: 8, + expected_source: "targets.prod.concurrency", + }, + ConcurrencyTestCase { + description: "a local settings override applies without a target value", + requested: None, + target: None, + settings: 4, + local_override: true, + expected_limit: 4, + expected_source: "sqlbuild_local.toml settings.concurrency", + }, + ConcurrencyTestCase { + description: "the project setting is the default", + requested: None, + target: None, + settings: 16, + local_override: false, + expected_limit: 16, + expected_source: "settings.concurrency", + }, + ]; + + for test_case in test_cases { + let resolved = effective_concurrency(&ConcurrencyLayers { + requested: test_case.requested, + target: test_case.target, + settings: test_case.settings, + local_override: test_case.local_override, + }); + assert_eq!( + (resolved.limit, resolved.source.to_string().as_str()), + (test_case.expected_limit, test_case.expected_source), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/effective/tests/test_types.rs b/crates/sqlbuild-config/src/effective/tests/test_types.rs index 0199cdea3a..6653868d84 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_types.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_types.rs @@ -48,3 +48,13 @@ pub(super) struct MessageTestCase { pub(super) names: &'static [&'static str], pub(super) expected_message: &'static str, } + +pub(super) struct ConcurrencyTestCase { + pub(super) description: &'static str, + pub(super) requested: Option, + pub(super) target: Option<(&'static str, i64)>, + pub(super) settings: i64, + pub(super) local_override: bool, + pub(super) expected_limit: i64, + pub(super) expected_source: &'static str, +} diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index 2ffa133ab5..0ba2fcebd1 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -15,7 +15,7 @@ use crate::project::_helpers::keys::{ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ - CLONE_KEYS, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, MAX_FULL_ROWS_KEY, + CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, MAX_FULL_ROWS_KEY, RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; use crate::project::errors::ConfigFailure; @@ -127,6 +127,24 @@ pub(crate) fn seconds_value(seconds: &str) -> ConfigValue { ) } +/// A target's worker limit, which overrides `[settings] concurrency` for that target. +fn target_concurrency(mapping: &Entries, target: TargetPlace<'_>) -> Loaded { + let invalid = || { + ConfigFailure::at( + target.file, + format!( + "targets.{}.concurrency must be an integer >= 1", + target.name + ), + ) + }; + let value = optional_strict_int(mapping, CONCURRENCY_KEY).map_err(|_| invalid())?; + if value.as_ref().is_some_and(|number| int_below(number, 1)) { + return Err(invalid()); + } + Ok(value_or_none(value)) +} + fn target_connection(mapping: &Entries, target: TargetPlace<'_>) -> Loaded<(Built, Built)> { match get(mapping, "connection") { None => Ok((Built::Value(ConfigValue::Map(Vec::new())), none())), @@ -396,6 +414,7 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded>()?; fields.push(("clone", object("LocalClonePolicy", clone_fields))); + fields.push(("concurrency", target_concurrency(mapping, target)?)); Ok(fields) } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index faa5439e1b..4c745d97c2 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -171,7 +171,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "valid_from_column", "valid_to_column", ]; -pub(crate) const TARGET_KEYS: [&str; 19] = [ +pub(crate) const TARGET_KEYS: [&str; 20] = [ "connection", "vars", "database", @@ -191,6 +191,7 @@ pub(crate) const TARGET_KEYS: [&str; 19] = [ "diff", "warehouses", "recall", + "concurrency", ]; pub(crate) const CLONE_KEYS: [&str; 2] = ["allow_as_clone_origin", "allow_as_clone_destination"]; pub(crate) const EXECUTION_LIMIT_KEYS: [&str; 5] = [ diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index 6c902db1f2..b84acd88e7 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -30,7 +30,15 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { ("warehouse", warehouse), ( "concurrency", - format!("{} configured limit", facts.concurrency), + if facts.concurrency_source.is_empty() { + format!("{} configured limit", facts.concurrency) + } else { + format!( + "{} configured limit ({})", + facts.concurrency, + sanitize(&facts.concurrency_source) + ) + }, ), ("full_refresh", facts.full_refresh.to_string()), ( diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index 952e90d832..bf8b0d29e7 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -179,6 +179,8 @@ pub struct RunContextFacts { /// The warehouse and where it came from. pub warehouse: Option<(String, String)>, pub concurrency: i64, + /// The configuration layer the concurrency came from. + pub concurrency_source: String, pub full_refresh: bool, pub selected_count: usize, pub total_count: usize, diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index a2611194dd..f9bb0b89ad 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -121,12 +121,13 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { command: "sqb build".to_owned(), run_id: Some("run\u{1b}1".to_owned()), concurrency: 4, + concurrency_source: "targets.prod.concurrency".to_owned(), selected_count: 1200, total_count: 1_234_567, date_vars: (Some("2026-01-01".to_owned()), Some("2026-01-31".to_owned())), ..RunContextFacts::default() }, - expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", + expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit (targets.prod.concurrency)\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", }, ContextTestCase { description: "a warehouse, one date and selector files", diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs index 3b9c8a22a1..a9f8414daa 100644 --- a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod allocation; pub(crate) mod decimal; +pub(crate) mod query_text; diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs new file mode 100644 index 0000000000..53a4e23f66 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs @@ -0,0 +1,34 @@ +//! The `QUERY_TEXT` column of a query-history read, selected by the run's query tag. + +use crate::cost_allocation::constants::{ + HOOK_TAG_FRAGMENT, NULL_QUERY_TEXT, SQLBUILD_TAG_FRAGMENT, +}; + +/// One run's query-tag fragment as `json.dumps` writes it, else the SQLBuild app fragment. +fn run_tag_fragment(run_id: &str) -> String { + let plain: bool = run_id + .chars() + .all(|character| character.is_ascii_graphic() && character != '"' && character != '\\'); + if plain && !run_id.is_empty() { + format!("\"run_id\":\"{run_id}\"") + } else { + SQLBUILD_TAG_FRAGMENT.to_owned() + } +} + +fn like_contains(fragment: &str) -> String { + format!("'%{}%'", fragment.replace('\'', "''")) +} + +/// CREATE text of the run's non-hook statements, selected by query tag, or masked text. +pub(crate) fn query_text_expression(run_id: Option<&str>) -> String { + let Some(run_id) = run_id else { + return NULL_QUERY_TEXT.to_owned(); + }; + format!( + "CASE WHEN (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') \ +AND QUERY_TAG LIKE {} AND QUERY_TAG NOT LIKE {} THEN QUERY_TEXT END AS QUERY_TEXT", + like_contains(&run_tag_fragment(run_id)), + like_contains(HOOK_TAG_FRAGMENT), + ) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/constants.rs b/crates/sqlbuild-core/src/cost_allocation/constants.rs index 09c13bd940..57789c24a3 100644 --- a/crates/sqlbuild-core/src/cost_allocation/constants.rs +++ b/crates/sqlbuild-core/src/cost_allocation/constants.rs @@ -25,6 +25,10 @@ pub(crate) const CREDITS_PER_HOUR: [(&str, u64); 10] = [ pub const QUERY_HISTORY_RESULT_LIMIT: usize = 10_000; pub(crate) const QUERY_HISTORY_FUNCTION: &str = "INFORMATION_SCHEMA.QUERY_HISTORY"; pub(crate) const NULL_QUERY_TEXT: &str = "NULL AS QUERY_TEXT"; +/// The compact, key-sorted query-tag fragment every SQLBuild-tagged statement carries. +pub(crate) const SQLBUILD_TAG_FRAGMENT: &str = "\"app\":\"sqlbuild\""; +/// The query-tag fragment of hook statements, whose text is never requested. +pub(crate) const HOOK_TAG_FRAGMENT: &str = "\"resource_type\":\"hook\""; /// The rounding digit at which `ROUND_HALF_EVEN` looks past the first dropped digit. pub(crate) const HALF_DIGIT: u8 = 5; /// `str(Decimal)` switches to exponent form at or below this many left digits. diff --git a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs index 40b6028249..54fa520146 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs @@ -1,6 +1,7 @@ //! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window. -use crate::cost_allocation::constants::{NULL_QUERY_TEXT, QUERY_HISTORY_FUNCTION}; +use crate::cost_allocation::_helpers::query_text::query_text_expression; +use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION; use crate::cost_allocation::models::QueryHistoryWindow; fn query_history_function(database: Option<&str>) -> String { @@ -13,24 +14,6 @@ fn query_history_function(database: Option<&str>) -> String { } } -fn query_text_expression(query_ids: &[String]) -> String { - if query_ids.is_empty() { - return NULL_QUERY_TEXT.to_owned(); - } - let mut sorted: Vec<&String> = query_ids.iter().collect(); - sorted.sort(); - sorted.dedup(); - let literals: Vec = sorted - .into_iter() - .map(|query_id| format!("'{}'", query_id.replace('\'', "''"))) - .collect(); - format!( - "CASE WHEN QUERY_ID IN ({}) AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') \ -THEN QUERY_TEXT END AS QUERY_TEXT", - literals.join(", ") - ) -} - /// Query-history SQL for queries ending after `start`, and before `end` when it is bounded. pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String { let end_range: String = match window.end { @@ -58,7 +41,7 @@ FROM TABLE({}( RESULT_LIMIT => {} )) ORDER BY START_TIME, QUERY_ID", - query_text_expression(window.classification_query_ids), + query_text_expression(window.text_run_id), query_history_function(window.database), window.start, window.result_limit, diff --git a/crates/sqlbuild-core/src/cost_allocation/models.rs b/crates/sqlbuild-core/src/cost_allocation/models.rs index 349f862fbe..d23c4e9be0 100644 --- a/crates/sqlbuild-core/src/cost_allocation/models.rs +++ b/crates/sqlbuild-core/src/cost_allocation/models.rs @@ -63,12 +63,12 @@ pub struct RunAllocation { pub limitations: Vec, } -/// The time window and classified query IDs of one query-history read. +/// One query-history read's window and the run whose CREATE text it requests by query tag. #[derive(Clone, Debug, PartialEq, Eq)] pub struct QueryHistoryWindow<'a> { pub database: Option<&'a str>, pub start: &'a str, pub end: Option<&'a str>, - pub classification_query_ids: &'a [String], + pub text_run_id: Option<&'a str>, pub result_limit: usize, } diff --git a/crates/sqlbuild-core/src/event_export/constants.rs b/crates/sqlbuild-core/src/event_export/constants.rs index c3bc6e2626..a5fe675243 100644 --- a/crates/sqlbuild-core/src/event_export/constants.rs +++ b/crates/sqlbuild-core/src/event_export/constants.rs @@ -19,6 +19,17 @@ pub const EVENT_EXPORT_KINDS: [&str; 8] = [ /// Invocation terminals, delivered only after every other queued event. pub const INVOCATION_TERMINAL_EVENT_TYPES: [&str; 2] = ["invocation_completed", "invocation_failed"]; +/// Run and invocation outcomes: always admitted, and the only events delivered in the shutdown reserve. +pub const OUTCOME_EVENT_TYPES: [&str; 6] = [ + "invocation_started", + "invocation_completed", + "invocation_failed", + "run_started", + "run_completed", + "run_failed", +]; +/// The final share of the shutdown budget kept for outcome events once other backlog is dropped. +pub const OUTCOME_RESERVE_FRACTION: f64 = 0.25; pub const RETRY_EVENT_TYPE: &str = "retry_scheduled"; pub const RETRY_EVENT_KIND: &str = "retry"; pub const RESOURCE_EVENT_PREFIX: &str = "resource"; diff --git a/crates/sqlbuild-core/src/event_export/main/mod.rs b/crates/sqlbuild-core/src/event_export/main/mod.rs index 59d4f3ce19..be3a987742 100644 --- a/crates/sqlbuild-core/src/event_export/main/mod.rs +++ b/crates/sqlbuild-core/src/event_export/main/mod.rs @@ -2,6 +2,7 @@ pub mod drop_lifecycle_attempts; pub mod export_snapshot; pub mod lifecycle_export_policy; pub mod queue_lifecycle_event; +pub mod retain_outcome_events; pub mod route_lifecycle_event; pub mod sanitized_exception_type; pub mod severity_at_least; diff --git a/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs index 1a585982c3..f8a7878ea1 100644 --- a/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs +++ b/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs @@ -2,15 +2,30 @@ use crate::event_export::models::{EventQueue, QueueOutcome, QueuedEvent}; -/// Queue `event`, or reject it when every queued event has at least its priority. +/// Queue `event` or reject it when full of events at least its priority; outcome events are never displaced and always admitted. pub fn queue_lifecycle_event( mut queue: EventQueue, event: QueuedEvent, ) -> QueueOutcome { let mut displaced: Option> = None; if queue.items.len() >= queue.capacity { - let lowest: Option = queue.items.iter().map(|queued| queued.priority).min(); - let Some(lowest) = lowest.filter(|lowest| *lowest < event.priority) else { + let lowest: Option = queue + .items + .iter() + .filter(|queued| !queued.outcome) + .map(|queued| queued.priority) + .min(); + let displaceable = lowest.filter(|lowest| event.outcome || *lowest < event.priority); + let Some(lowest) = displaceable else { + if event.outcome { + queue.items.push(event); + return QueueOutcome { + queue, + inserted: true, + displaced: None, + rejected: None, + }; + } return QueueOutcome { queue, inserted: false, @@ -22,7 +37,7 @@ pub fn queue_lifecycle_event( .items .iter() .enumerate() - .filter(|(_, queued)| queued.priority == lowest) + .filter(|(_, queued)| !queued.outcome && queued.priority == lowest) .min_by_key(|(_, queued)| queued.sequence) .map(|(position, _)| position); displaced = position.map(|position| queue.items.remove(position)); diff --git a/crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs b/crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs new file mode 100644 index 0000000000..7e110abd21 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs @@ -0,0 +1,12 @@ +//! Keep only outcome events once the shutdown reserve begins. + +use crate::event_export::models::{EventQueue, QueuedEvent}; + +/// The queue with only its outcome events, and every other event it held. +#[must_use] +pub fn retain_outcome_events(mut queue: EventQueue) -> (EventQueue, Vec>) { + let (kept, dropped): (Vec>, Vec>) = + queue.items.drain(..).partition(|queued| queued.outcome); + queue.items = kept; + (queue, dropped) +} diff --git a/crates/sqlbuild-core/src/event_export/models.rs b/crates/sqlbuild-core/src/event_export/models.rs index 66a5e2159c..9ee6750807 100644 --- a/crates/sqlbuild-core/src/event_export/models.rs +++ b/crates/sqlbuild-core/src/event_export/models.rs @@ -44,6 +44,8 @@ pub struct QueuedEvent { pub priority: u8, /// Whether the event ends the invocation and so waits behind every other queued event. pub invocation_terminal: bool, + /// Whether the event reports a run or invocation outcome, which the queue always admits. + pub outcome: bool, pub eligible_exporters: Vec, } diff --git a/crates/sqlbuild-core/src/event_export/tests/helpers.rs b/crates/sqlbuild-core/src/event_export/tests/helpers.rs index 03087a70d4..2817d06c0a 100644 --- a/crates/sqlbuild-core/src/event_export/tests/helpers.rs +++ b/crates/sqlbuild-core/src/event_export/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::event_export::models::QueuedEvent; +use crate::event_export::models::{EventQueue, QueuedEvent}; /// A queued unit event with `sequence`, `(priority, invocation terminal)` and one eligible exporter. pub(crate) fn queued( @@ -10,6 +10,22 @@ pub(crate) fn queued( item: (), priority, invocation_terminal, + outcome: false, eligible_exporters: vec![0], } } + +/// A queued non-terminal event with `sequence`, `(priority, outcome)` and one eligible exporter. +pub(crate) fn event((priority, outcome): (u8, bool), sequence: u64) -> QueuedEvent<()> { + QueuedEvent { + outcome, + ..queued(sequence, (priority, false)) + } +} + +/// The sequences `queue` holds, ascending. +pub(crate) fn sequences(queue: &EventQueue<()>) -> Vec { + let mut sequences: Vec = queue.items.iter().map(|item| item.sequence).collect(); + sequences.sort_unstable(); + sequences +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs index 95aa2ce728..2ae6361f1d 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs @@ -1,8 +1,107 @@ use crate::event_export::main::queue_lifecycle_event::queue_lifecycle_event; +use crate::event_export::main::retain_outcome_events::retain_outcome_events; use crate::event_export::main::take_lifecycle_event::take_lifecycle_event; use crate::event_export::models::EventQueue; -use crate::event_export::tests::helpers::queued; -use crate::event_export::tests::test_types::{DeliveryOrderTestCase, DisplacementTestCase}; +use crate::event_export::tests::helpers::{event, queued, sequences}; +use crate::event_export::tests::test_types::{ + DeliveryOrderTestCase, DisplacementTestCase, OutcomeAdmissionTestCase, OutcomeReserveTestCase, +}; + +#[test] +fn given_full_queue_when_outcome_or_other_event_arrives_then_outcomes_are_admitted_and_kept() { + let test_cases = [ + OutcomeAdmissionTestCase { + description: "outcome displaces an equal-priority event", + queued: &[(2, false), (2, false)], + incoming: (2, true), + expected_inserted: true, + expected_displaced: Some(0), + expected_sequences: &[1, 2], + }, + OutcomeAdmissionTestCase { + description: "outcome displaces a higher-priority event when nothing lower is queued", + queued: &[(3, false), (2, true)], + incoming: (0, true), + expected_inserted: true, + expected_displaced: Some(0), + expected_sequences: &[1, 2], + }, + OutcomeAdmissionTestCase { + description: "outcome exceeds capacity when only outcomes are queued", + queued: &[(2, true), (0, true)], + incoming: (2, true), + expected_inserted: true, + expected_displaced: None, + expected_sequences: &[0, 1, 2], + }, + OutcomeAdmissionTestCase { + description: "failure never displaces a queued outcome", + queued: &[(2, true), (0, true)], + incoming: (3, false), + expected_inserted: false, + expected_displaced: None, + expected_sequences: &[0, 1], + }, + ]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(test_case.queued.len()); + for (sequence, dimensions) in (0_u64..).zip(test_case.queued) { + queue = queue_lifecycle_event(queue, event(*dimensions, sequence)).queue; + } + let incoming_sequence: u64 = u64::try_from(test_case.queued.len()).unwrap_or(u64::MAX); + let outcome = queue_lifecycle_event(queue, event(test_case.incoming, incoming_sequence)); + assert_eq!( + ( + outcome.inserted, + outcome.displaced.as_ref().map(|event| event.sequence), + sequences(&outcome.queue), + ), + ( + test_case.expected_inserted, + test_case.expected_displaced, + test_case.expected_sequences.to_vec(), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_mixed_backlog_when_reserve_begins_then_only_outcome_events_remain() { + let test_cases = [OutcomeReserveTestCase { + description: "other events are handed back for accounting", + queued: &[(2, false), (2, true), (0, false), (2, true)], + expected_kept: &[1, 3], + expected_dropped: &[0, 2], + }]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(8); + for (sequence, dimensions) in (0_u64..).zip(test_case.queued) { + queue = queue_lifecycle_event(queue, event(*dimensions, sequence)).queue; + } + + let (kept, dropped) = retain_outcome_events(queue); + + assert_eq!( + ( + sequences(&kept), + dropped + .iter() + .map(|item| item.sequence) + .collect::>() + ), + ( + test_case.expected_kept.to_vec(), + test_case.expected_dropped.to_vec() + ), + "{}", + test_case.description + ); + } +} #[test] fn given_full_queue_when_priority_pair_arrives_then_displacement_policy_is_exact() { diff --git a/crates/sqlbuild-core/src/event_export/tests/test_types.rs b/crates/sqlbuild-core/src/event_export/tests/test_types.rs index d3960e9af0..0338b49a40 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_types.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_types.rs @@ -58,3 +58,21 @@ pub(crate) struct SnapshotTestCase { pub(crate) expected_accepted: u64, pub(crate) expected_flush_complete: bool, } + +/// A full queue of `(priority, outcome)` events receiving one more, and what it keeps. +pub(crate) struct OutcomeAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) queued: &'static [(u8, bool)], + pub(crate) incoming: (u8, bool), + pub(crate) expected_inserted: bool, + pub(crate) expected_displaced: Option, + pub(crate) expected_sequences: &'static [u64], +} + +/// Queued `(priority, outcome)` events and what the shutdown reserve keeps and drops, by sequence. +pub(crate) struct OutcomeReserveTestCase { + pub(crate) description: &'static str, + pub(crate) queued: &'static [(u8, bool)], + pub(crate) expected_kept: &'static [u64], + pub(crate) expected_dropped: &'static [u64], +} diff --git a/crates/sqlbuild-core/src/integration_results/constants.rs b/crates/sqlbuild-core/src/integration_results/constants.rs index 512d74e0aa..6139ea82ef 100644 --- a/crates/sqlbuild-core/src/integration_results/constants.rs +++ b/crates/sqlbuild-core/src/integration_results/constants.rs @@ -48,7 +48,8 @@ pub(crate) const CHECK_STATUSES: [&str; 5] = ["error", "fail", "insufficient", " pub(crate) const CHECK_NON_FAIL_STATUSES: [&str; 2] = ["insufficient", "pass"]; /// `AuditSeverity` values, in declaration order. pub(crate) const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; -pub(crate) const CHECK_ATTACHMENT_KINDS: [&str; 3] = ["end", "model", "source"]; +/// Every audit attachment kind: model, source and seed attachments, and end-of-run audits. +pub(crate) const CHECK_ATTACHMENT_KINDS: [&str; 4] = ["end", "model", "seed", "source"]; pub(crate) const CHECK_RUN_SCOPE_PHASES: [&str; 2] = ["delta_and_final", "final"]; pub(crate) const EVALUATION_MODES: [&str; 2] = ["violations", "measurement"]; pub(crate) const BOUND_OPERATORS: [&str; 2] = ["below", "above"]; diff --git a/crates/sqlbuild-core/src/integration_results/tests/helpers.rs b/crates/sqlbuild-core/src/integration_results/tests/helpers.rs index 53f50afe2f..d1a0702766 100644 --- a/crates/sqlbuild-core/src/integration_results/tests/helpers.rs +++ b/crates/sqlbuild-core/src/integration_results/tests/helpers.rs @@ -50,3 +50,40 @@ pub(super) fn orders_asset(status: &str) -> JsonValue { ("microbatch", object(vec![("run_type", text("normal"))])), ]) } + +/// A completed audit record whose single check carries the given attachment kind. +pub(super) fn audit_check_record(attachment_kind: &str) -> JsonValue { + object(vec![ + ("schema_version", integer(1)), + ("record_kind", text("integration_result")), + ("event_id", text("e1")), + ("event_sequence", integer(0)), + ("event_type", text("resource_attempt_completed")), + ("occurred_at", text("2026-09-02T12:00:00Z")), + ("invocation_id", text("i1")), + ("run_id", text("r1")), + ("resource_id", text("audit:not_null:orders")), + ("resource_attempt_id", text("a1")), + ("operation_id", JsonValue::Null), + ("statement_id", JsonValue::Null), + ("resource_kind", text("audit")), + ("resource_name", text("not_null")), + ("attempt_number", integer(1)), + ("duration_ms", integer(3)), + ("output_kind", text("check")), + ("command", text("audit")), + ("asset", JsonValue::Null), + ( + "checks", + JsonValue::Array(vec![object(vec![ + ("kind", text("audit")), + ("name", text("not_null")), + ("check_id", text("audit:not_null:orders")), + ("passed", JsonValue::Bool(true)), + ("status", text("pass")), + ("attachment_kind", text(attachment_kind)), + ("audit_definition_name", text("not_null")), + ])]), + ), + ]) +} diff --git a/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs b/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs index 5dbf05e0a4..ba01a2bcd7 100644 --- a/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs +++ b/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs @@ -1,8 +1,10 @@ use crate::integration_results::main::encode_integration_value::encode_integration_value; use crate::integration_results::main::normalize_integration_record::normalize_integration_record; use crate::integration_results::main::strip_optional_enrichment::strip_optional_enrichment; -use crate::integration_results::tests::helpers::{integer, object, orders_asset, record, text}; -use crate::integration_results::tests::test_types::NormalizeTestCase; +use crate::integration_results::tests::helpers::{ + audit_check_record, integer, object, orders_asset, record, text, +}; +use crate::integration_results::tests::test_types::{AttachmentKindTestCase, NormalizeTestCase}; use crate::json::models::JsonValue; #[test] @@ -99,3 +101,46 @@ fn given_newer_enrichment_when_stripping_then_record_keeps_core_fields() { ); } } + +#[test] +fn given_every_audit_attachment_kind_when_normalizing_check_then_it_is_supported() { + let test_cases = [ + AttachmentKindTestCase { + description: "a model audit check is supported", + attachment_kind: "model", + expected_error: None, + }, + AttachmentKindTestCase { + description: "a source audit check is supported", + attachment_kind: "source", + expected_error: None, + }, + AttachmentKindTestCase { + description: "a seed audit check is supported", + attachment_kind: "seed", + expected_error: None, + }, + AttachmentKindTestCase { + description: "an end-of-run audit check is supported", + attachment_kind: "end", + expected_error: None, + }, + AttachmentKindTestCase { + description: "an unknown attachment kind is rejected", + attachment_kind: "warehouse", + expected_error: Some("integration check attachment_kind is unsupported"), + }, + ]; + + for test_case in test_cases { + let normalized = + normalize_integration_record(audit_check_record(test_case.attachment_kind)); + let error: Option = normalized.as_ref().err().map(|error| error.message.clone()); + assert_eq!( + error.as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/integration_results/tests/test_types.rs b/crates/sqlbuild-core/src/integration_results/tests/test_types.rs index bacd65819f..fee2995e50 100644 --- a/crates/sqlbuild-core/src/integration_results/tests/test_types.rs +++ b/crates/sqlbuild-core/src/integration_results/tests/test_types.rs @@ -6,3 +6,9 @@ pub(super) struct NormalizeTestCase { pub(super) expected_error: Option<&'static str>, pub(super) expected_encoded: Option<&'static str>, } + +pub(super) struct AttachmentKindTestCase { + pub(super) description: &'static str, + pub(super) attachment_kind: &'static str, + pub(super) expected_error: Option<&'static str>, +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs index 1859d7aa66..c4f5c1211c 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs @@ -5,6 +5,8 @@ use crate::project_validation::errors::HeaderError; use crate::project_validation::models::CsvSettings; use sqlbuild_core::text::main::decode_text::decode_text; +/// The quote character seed loaders use when a seed declares none. +const DEFAULT_QUOTECHAR: char = '"'; /// The size of the chunks `TextIOWrapper` decodes. const DECODE_CHUNK: usize = 8192; const UNQUOTED_NEWLINE: &str = @@ -320,6 +322,60 @@ fn first_record( Ok(None) } +/// Non-blank records in `bytes`, at most `limit`; `None` for a prefix ending before `limit`. +pub(crate) fn csv_record_count( + bytes: &[u8], + settings: &CsvSettings, + complete: bool, + limit: usize, +) -> Result, HeaderError> { + let mut dialect: Dialect = Dialect::new(settings, usize::MAX); + if settings.quotechar.is_none() { + dialect.quotechar = Some(DEFAULT_QUOTECHAR); + } + let text: String = match std::str::from_utf8(bytes) { + Ok(text) => text.to_owned(), + Err(error) if error.error_len().is_none() && !complete => { + String::from_utf8_lossy(&bytes[..error.valid_up_to()]).into_owned() + } + Err(_) => decoded_prefix(bytes, bytes.len())?, + }; + let mut parser: Parser<'_> = Parser { + settings: &dialect, + state: State::StartRecord, + field: String::new(), + field_len: 0, + fields: Vec::new(), + }; + let mut count: usize = 0; + let mut lines = lines(&text).peekable(); + while let Some(line) = lines.next() { + let terminated: bool = line.ends_with(['\n', '\r']); + if lines.peek().is_none() && !complete && !terminated_line(line, terminated, &text) { + return Ok(None); + } + for character in line.chars() { + parser.process(Input::Char(character))?; + } + parser.process(Input::Eol)?; + if parser.state == State::StartRecord { + if !std::mem::take(&mut parser.fields).is_empty() { + count += 1; + } + if count >= limit { + return Ok(Some(count)); + } + } + } + if !complete { + return Ok(None); + } + if parser.field_len != 0 || parser.state == State::InQuotedField || !parser.fields.is_empty() { + count += 1; + } + Ok(Some(count.min(limit))) +} + /// Whether a final line is known to be whole: it ends with `\n`, or `\r` not followed later. fn terminated_line(line: &str, terminated: bool, text: &str) -> bool { terminated && (line.ends_with('\n') || !text.ends_with('\r')) diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index 6122e20503..e03d600d5a 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -8,3 +8,7 @@ pub(crate) const CURRENT_DIRECTORY: &str = "."; pub(crate) const KNOWN_FOLDER_LIMIT: usize = 5; /// Model names SQLBuild reserves for compiled output structure. pub(crate) const RESERVED_MODEL_NAMES: [&str; 1] = ["_chain_"]; +/// The first seed read, grown until the header and two data rows are seen or the file ends. +pub(crate) const INITIAL_SEED_READ_BYTES: u64 = 64 * 1024; +/// A header and two data rows: enough to know a seed holds more than one data row. +pub(crate) const RECORDS_FOR_MANY_ROWS: usize = 3; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs b/crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs new file mode 100644 index 0000000000..168e69135f --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs @@ -0,0 +1,30 @@ +//! Whether a seed CSV holds a header and at most one data row. + +use std::fs::File; +use std::io::Read; +use std::path::Path; + +use crate::project_validation::_helpers::csv_header::csv_record_count; +use crate::project_validation::constants::{INITIAL_SEED_READ_BYTES, RECORDS_FOR_MANY_ROWS}; +use crate::project_validation::models::CsvSettings; + +/// Whether the CSV at `path` has at most one data row; any read or parse failure answers `false`. +#[must_use] +pub fn at_most_one_csv_data_row(path: &Path, settings: &CsvSettings) -> bool { + let Ok(mut file) = File::open(path) else { + return false; + }; + let mut bytes: Vec = Vec::new(); + let mut want: u64 = INITIAL_SEED_READ_BYTES; + loop { + let Ok(read) = (&mut file).take(want).read_to_end(&mut bytes) else { + return false; + }; + let complete: bool = (read as u64) < want; + match csv_record_count(&bytes, settings, complete, RECORDS_FOR_MANY_ROWS) { + Ok(Some(count)) => return count < RECORDS_FOR_MANY_ROWS, + Ok(None) if !complete => want = want.saturating_mul(2), + Ok(None) | Err(_) => return false, + } + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs index 1fc67badd8..2c19ca312a 100644 --- a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs +++ b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs @@ -1 +1,2 @@ +pub mod at_most_one_csv_data_row; pub mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/tests.rs b/crates/sqlbuild-discovery/src/project_validation/tests.rs index ff3236ffdb..d7aebec514 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_at_most_one_csv_data_row.rs"] +mod at_most_one_csv_data_row; #[path = "tests/test_csv_header.rs"] mod csv_header; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs new file mode 100644 index 0000000000..dc07731a42 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs @@ -0,0 +1,85 @@ +use std::path::PathBuf; + +use crate::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; +use crate::project_validation::models::CsvSettings; +use crate::project_validation::tests::test_types::OneDataRowTestCase; + +#[test] +fn given_seed_csv_text_when_counting_data_rows_then_only_one_row_files_qualify() { + let test_cases = [ + OneDataRowTestCase { + description: "header and one row", + text: "version,currency\n1,EUR\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "header only", + text: "version,currency\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "one row without a final newline", + text: "version,currency\n1,EUR".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "blank lines are not rows", + text: "version,currency\n\n1,EUR\n\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "a quoted newline stays inside one row", + text: "version,note\n1,\"first\nsecond\"\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "two rows", + text: "version,currency\n1,EUR\n2,USD\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: false, + }, + OneDataRowTestCase { + description: "two rows with a custom delimiter", + text: "version;currency\n1;EUR\n2;USD\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + expected_one_row: false, + }, + OneDataRowTestCase { + description: "many rows past the first read", + text: format!("id\n{}", "1\n".repeat(70_000)), + settings: CsvSettings::default(), + expected_one_row: false, + }, + OneDataRowTestCase { + description: "one row longer than the first read", + text: format!("id\n\"{}\"\n", "x".repeat(200_000)), + settings: CsvSettings::default(), + expected_one_row: true, + }, + ]; + let directory = tempfile::TempDir::new().unwrap_or_else(|error| panic!("{error}")); + + for test_case in test_cases { + let path: PathBuf = directory.path().join("seed.csv"); + std::fs::write(&path, &test_case.text).unwrap_or_else(|error| panic!("{error}")); + + assert_eq!( + at_most_one_csv_data_row(&path, &test_case.settings), + test_case.expected_one_row, + "{}", + test_case.description + ); + } + assert!(!at_most_one_csv_data_row( + &directory.path().join("missing.csv"), + &CsvSettings::default() + )); +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs index 6881934f58..08e6d0affd 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs @@ -42,3 +42,10 @@ pub(super) fn csv_error(message: &str) -> Result, String> { pub(super) fn failure(kind: FailureKind, message: &str) -> Result<(), String> { Err(format!("{kind:?}: {message}")) } + +pub(crate) struct OneDataRowTestCase { + pub(crate) description: &'static str, + pub(crate) text: String, + pub(crate) settings: CsvSettings, + pub(crate) expected_one_row: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index c6047e2bd5..6555702ee3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -15,6 +15,7 @@ use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressRepo use crate::bindings::_helpers::cli::progress::current_native_progress_projector; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_text::write_execution_header; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords, within}; @@ -184,15 +185,12 @@ fn prepare_run<'py>( let py: Python<'py> = request.py(); let entries: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?.getattr("audit_entries")?; - let requested: Bound<'py, PyAny> = native_field(request, "concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field(pipeline, "project")? - .getattr("settings")? - .getattr("concurrency")? - .extract::()? - } else { - requested.extract::()? - }; + let concurrency: i64 = resolve_concurrency( + &native_field(request, "concurrency")?, + &native_field(invocation, "discovered_inputs")?, + &native_field(request, "selected_target")?, + )? + .limit; let audit_count: usize = entries.len()?; let worker_count: i64 = concurrency.min(i64::try_from(audit_count).unwrap_or(i64::MAX)); let stream: Bound<'py, PyAny> = native_field(invocation, "progress_stream")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index ec7ba2f74d..d20adc24a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -1,8 +1,10 @@ //! Prepare build execution, run the build pipeline and the post-build Python checks. +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_core::build_command::main::render_run_context::render_run_context; use sqlbuild_core::build_command::models::{RunContextFacts, SelectorFileFact}; use sqlbuild_core::cli_text::models::CliStyle; @@ -81,7 +83,7 @@ fn validated_date(py: Python<'_>, value: Option>) -> PyResult, pipeline: &Bound<'_, PyAny>, - concurrency: &Bound<'_, PyAny>, + concurrency: &EffectiveConcurrency, ) -> PyResult { let py: Python<'_> = context.py(); let project: Bound<'_, PyAny> = native_field(pipeline, "project")?; @@ -142,7 +144,8 @@ fn run_context_facts( database: display_text(&native_field(&project, "effective_target_database")?)?, schema: display_text(&native_field(&project, "effective_target_schema")?)?, warehouse, - concurrency: concurrency.extract()?, + concurrency: concurrency.limit, + concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, selected_count, total_count, @@ -159,7 +162,7 @@ pub(crate) fn write_run_context( context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>, ) -> PyResult<()> { - let concurrency: Bound<'_, PyAny> = context.concurrency(&native_field(pipeline, "project")?)?; + let concurrency: EffectiveConcurrency = context.concurrency()?; let facts: RunContextFacts = run_context_facts(context, pipeline, &concurrency)?; emit( &context.stream, @@ -220,8 +223,7 @@ pub(crate) fn prepare_build_execution<'py>( ) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency: Bound<'py, PyAny> = - context.concurrency(&native_field(pipeline, "project")?)?; + let concurrency: Bound<'py, PyAny> = context.concurrency()?.limit.into_pyobject(py)?.into_any(); if context.verbose { write_run_context(context, pipeline)?; } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 1742421c42..c2fca8e628 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -3,6 +3,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_core::build_command::errors::BuildCommandError; use sqlbuild_core::build_command::main::duration_limit_error::duration_limit_error; use sqlbuild_core::build_command::models::DurationLimit; @@ -12,6 +13,7 @@ use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -51,11 +53,13 @@ impl<'py> BuildContext<'py> { native_field(&self.invocation, name) } - /// The requested concurrency, or the project setting. - pub(crate) fn concurrency(&self, project: &Bound<'py, PyAny>) -> PyResult> { - or_none(self.request.getattr("concurrency")?, || { - native_field(project, "settings")?.getattr("concurrency") - }) + /// `--concurrency`, else the selected target's `concurrency`, else `[settings]`. + pub(crate) fn concurrency(&self) -> PyResult { + resolve_concurrency( + &self.request.getattr("concurrency")?, + &self.get("discovered_inputs")?, + &native_field(&self.request, "selected_target")?, + ) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 0e9d5993c4..dc05338074 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -70,6 +70,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult, Py)>, /// Context managers entered, innermost last. entered: Vec>, + /// The command name a closing status line reports. + command: String, } /// Log a cleanup failure without letting logging fail the command. @@ -197,18 +203,20 @@ fn export_summary(summary: &Bound<'_, PyAny>) -> PyResult { }) } -/// Print the incomplete-export warning to stderr when events were dropped or failed. -fn warn_incomplete_export(summary: &Bound<'_, PyAny>, command_succeeded: bool) -> PyResult<()> { - let py: Python<'_> = summary.py(); +/// Print the incomplete-export warning to stderr, closed by the command's success, ignoring write failures. +fn warn_incomplete_export( + summary: &Bound<'_, PyAny>, + (command, command_succeeded): (&str, bool), +) -> PyResult<()> { let Some(message) = event_export_warning(&export_summary(summary)?, command_succeeded) else { return Ok(()); }; - let options = pyo3::types::PyDict::new(py); - pyo3::types::PyDictMethods::set_item(&options, "file", py.import("sys")?.getattr("stderr")?)?; - pyo3::types::PyDictMethods::set_item(&options, "flush", true)?; - py.import("builtins")? - .getattr("print")? - .call((message,), Some(&options))?; + let closing: String = if command_succeeded { + closing_status(command, Some(0), LIFECYCLE_EVENT_EXPORT) + } else { + String::new() + }; + drop(write_stderr(summary.py(), &format!("{message}\n{closing}"))); Ok(()) } @@ -295,7 +303,7 @@ impl Observed { let succeeded: bool = self.outcome.get().succeeded.load(Ordering::SeqCst); run_cleanup( py, - warn_incomplete_export(&summary, succeeded), + warn_incomplete_export(&summary, (&self.command, succeeded)), "event_exporter_warning", ); } @@ -539,6 +547,9 @@ impl CliObservabilityScope { unsubscribe_outcome: None, recall_sinks: None, entered: Vec::new(), + command: native_field(args, "command")? + .extract::>()? + .unwrap_or_default(), }; if let Err(error) = observed.install(py, (args, self.project_dir.bind(py)), &terminal_index) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs new file mode 100644 index 0000000000..3008e81201 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs @@ -0,0 +1,62 @@ +//! The worker limit of one command: `--concurrency`, the target, then `[settings]`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; +use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; + +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_target_config, resolve_target_name, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +const CONCURRENCY: &str = "concurrency"; + +/// The effective concurrency of a command on `selected_target`, and the layer that set it. +pub(crate) fn resolve_concurrency( + requested: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + resolve_configured_concurrency( + requested, + ( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + ), + selected_target, + ) +} + +/// `resolve_concurrency` from the project and local configs themselves. +pub(crate) fn resolve_configured_concurrency( + requested: &Bound<'_, PyAny>, + (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + let py = project_config.py(); + let target: Option<(String, i64)> = + match resolve_target_name(py, project_config, local_config, selected_target)? { + Some(name) => { + let config: Bound<'_, PyAny> = + resolve_target_config(py, project_config, local_config, name.as_any())?; + let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; + limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) + } + None => None, + } + .transpose()?; + let settings: i64 = native_field( + &build_effective_settings(project_config, local_config)?, + CONCURRENCY, + )? + .extract()?; + let local_override: bool = + native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; + Ok(effective_concurrency(&ConcurrencyLayers { + requested: requested.extract()?, + target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), + settings, + local_override, + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index 2d06334d13..fa6ea5ff50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod adapters; +pub(crate) mod concurrency; pub(crate) mod connection; pub(crate) mod invocation; pub(crate) mod preview_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index d1e6f34625..9d29ac2895 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -193,6 +193,13 @@ pub(crate) fn reject_invalid_project_configuration( config: &ProjectConfiguration<'_, '_>, ) -> PyResult<()> { reject_local_sql_analysis_opt_out(config)?; + reject_invalid_connection_configuration(config) +} + +/// Validate the target warehouses and target connections a warehouse connection depends on. +pub(crate) fn reject_invalid_connection_configuration( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { reject_unsupported_target_warehouses(config)?; reject_invalid_target_connections(config) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 407cf70237..33d0a026eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -99,6 +99,7 @@ pub(crate) fn pipeline_result<'py>( "full_refresh", "reload_sources", "max_microbatches", + "concurrency", ], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 6cef6bd314..8303b8a13a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -324,7 +324,6 @@ const R_LINT_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("header_rules_enabled", FieldDefault::True), FieldSpec::new("max_literal_length", FieldDefault::Int(100)), FieldSpec::new("max_ranking_order_by", FieldDefault::Int(5)), - FieldSpec::new("allow_suppressions", FieldDefault::True), FieldSpec { name: "relation_keys", default: FieldDefault::EmptyDict, @@ -395,6 +394,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("resolve_python_run_selectors", FieldDefault::False), FieldSpec::new("no_cache", FieldDefault::False), FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("selection_diagnostics", FieldDefault::False), FieldSpec::new("plan_sql_tests", FieldDefault::True), FieldSpec::new("record_migration_fingerprints", FieldDefault::True), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 8df9e95178..925d7e16ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -146,6 +146,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { "warehouses", FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), + FieldSpec::new("concurrency", FieldDefault::None), ], }; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { @@ -184,6 +185,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { "warehouses", FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), + FieldSpec::new("concurrency", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index b090f852da..518b8f373c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_core::cost_command::constants::{ HISTORY_SELECTOR, LATEST_SELECTOR, REFRESHABLE_STATUSES, }; @@ -26,6 +26,9 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::replaced_keywords; +/// Collection budget of an explicit `sqb cost` refresh, which a user waits for on purpose. +const REFRESH_BUDGET_SECONDS: f64 = 300.0; + fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { Ok(CostCoverage { status: native_field(summary, "status")? @@ -167,6 +170,10 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { ("statement_ledger_path", ledger), ("usd_per_credit", native_field(¤t, "usd_per_credit")?), ("rate_source", native_field(¤t, "rate_source")?), + ( + "budget_seconds", + REFRESH_BUDGET_SECONDS.into_pyobject(py)?.into_any(), + ), ], )?; if accepts_cost_refresh(&coverage(¤t)?, &coverage(&summary)?) != Some(true) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 405562e494..96b5c7561b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -3,8 +3,10 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use pyo3::exceptions::{PyTimeoutError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; @@ -25,8 +27,13 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::RecordId; -const DEADLINE_SECONDS: f64 = 10.0; -const QUERY_TIMEOUT_SECONDS: &str = "5"; +/// Default budget of one collection, which the at-exit collection keeps short. +const DEADLINE_SECONDS: f64 = 30.0; +/// Statement timeout of one query-history read. +const QUERY_TIMEOUT_SECONDS: &str = "10"; +/// Narrowest slice still split in half after a statement timeout. +const MIN_TIMEOUT_SPLIT_SECONDS: i64 = 60; +const STATEMENT_TIMEOUT_CODE: &str = "000630"; const RUNNING_STATUS: &str = "RUNNING"; const HOOK_RESOURCE_TYPE: &str = "hook"; const ROW_WIDTH: usize = 14; @@ -43,11 +50,12 @@ const PARTIAL_STATUS: &str = "PARTIAL"; const COMPLETE_STATUS: &str = "COMPLETE"; const UNAVAILABLE_STATUS: &str = "UNAVAILABLE"; -/// The connection, window inputs and deadline every query-history read shares. +/// Connection, window inputs and deadline of every read; text is kept only for classified IDs. struct History<'py> { connection: Bound<'py, PyAny>, database: Option, - classification_query_ids: Vec, + text_run_id: Option, + classification_query_ids: HashSet, deadline: f64, result_limit: usize, } @@ -92,7 +100,7 @@ fn history_rows<'py>( database: history.database.as_deref(), start: &start_text, end: end_text.as_deref(), - classification_query_ids: &history.classification_query_ids, + text_run_id: history.text_run_id.as_deref(), result_limit: history.result_limit, }); let parameters: Bound<'py, PyDict> = PyDict::new(py); @@ -134,20 +142,46 @@ fn merged_rows<'py>( Ok(by_query_id.into_values().collect()) } -/// End-time-bounded rows, halving the window while a read reaches the result limit. +/// Whether a query-history read failed on Snowflake's statement timeout. +pub(crate) fn statement_timeout(py: Python<'_>, error: &PyErr) -> bool { + error + .value(py) + .str() + .is_ok_and(|text| text.to_string_lossy().contains(STATEMENT_TIMEOUT_CODE)) +} + +/// End-time-bounded rows, halving the window while a read reaches the result limit or times out. fn bounded_rows<'py>( history: &History<'py>, (start, end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult<(Vec>, bool)> { let py: Python<'py> = start.py(); - let rows: Vec> = history_rows(history, (start, Some(end)))?; + let width: Bound<'py, PyAny> = end.sub(start)?; + let rows: Vec> = match history_rows(history, (start, Some(end))) { + Ok(rows) => rows, + Err(error) + if statement_timeout(py, &error) + && width.gt(timedelta(py, ("seconds", MIN_TIMEOUT_SPLIT_SECONDS))?)? => + { + return split_rows(history, (start, end), &width); + } + Err(error) => return Err(error), + }; if rows.len() < history.result_limit { return Ok((rows, false)); } - let width: Bound<'py, PyAny> = end.sub(start)?; if width.le(timedelta(py, ("milliseconds", 1))?)? { return Ok((rows, true)); } + split_rows(history, (start, end), &width) +} + +/// Rows of both halves of a bounded window, merged by query ID. +fn split_rows<'py>( + history: &History<'py>, + (start, end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + width: &Bound<'py, PyAny>, +) -> PyResult<(Vec>, bool)> { let midpoint: Bound<'py, PyAny> = start.add(width.div(2)?)?; let (left, left_saturated) = bounded_rows(history, (start, &midpoint))?; let (right, right_saturated) = bounded_rows(history, (&midpoint, end))?; @@ -194,7 +228,7 @@ fn optional_str<'py>(value: &Bound<'py, PyAny>) -> PyResult> { /// A history row as an observation, or the reason it carries no attributable compute. fn row_observation<'py>( row: &Bound<'py, PyAny>, - require_complete: bool, + (require_complete, classify): (bool, bool), ) -> PyResult, &'static str>> { let py: Python<'py> = row.py(); let fields: Vec> = unpacked(row)?; @@ -224,7 +258,7 @@ fn row_observation<'py>( } else { String::new() }; - let query_text: Option = if query_text.is_instance_of::() { + let query_text: Option = if classify && query_text.is_instance_of::() { Some(query_text.extract()?) } else { None @@ -288,7 +322,36 @@ fn row_observation<'py>( Ok(Ok(observation)) } -/// Observations of the run window: open-ended history, bounded windows when saturated. +/// Open-ended rows from `start`, which include running queries, or bounded windows if saturated. +fn open_rows<'py>( + history: &History<'py>, + (start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Vec>, bool)> { + let py: Python<'py> = start.py(); + let open_rows: Vec> = match history_rows(history, (start, None)) { + Ok(rows) => rows, + Err(error) if statement_timeout(py, &error) => { + return bounded_rows(history, (start, window_end)); + } + Err(error) => return Err(error), + }; + if open_rows.len() < history.result_limit { + return Ok((open_rows, false)); + } + let (bounded, bounded_saturated) = bounded_rows(history, (start, window_end))?; + let mut still_running: Vec> = Vec::new(); + for row in open_rows { + if running(&row.get_item(7)?, &row.get_item(10)?)? { + still_running.push(row); + } + } + Ok(( + merged_rows(still_running.into_iter().chain(bounded))?, + bounded_saturated, + )) +} + +/// Observations of the run window: one open read, halved windows when saturated or timed out. fn query_observations<'py>( history: &History<'py>, (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -297,29 +360,16 @@ fn query_observations<'py>( let py: Python<'py> = started_at.py(); let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; - let open_rows: Vec> = history_rows(history, (&window_start, None))?; - let mut saturated: bool = open_rows.len() >= history.result_limit; - let rows: Vec> = if saturated { - let (bounded, bounded_saturated) = - bounded_rows(history, (&window_start, &completed_at.add(&skew)?))?; - let mut still_running: Vec> = Vec::new(); - for row in open_rows { - if running(&row.get_item(7)?, &row.get_item(10)?)? { - still_running.push(row); - } - } - saturated = bounded_saturated; - merged_rows(still_running.into_iter().chain(bounded))? - } else { - open_rows - }; + let window_end: Bound<'py, PyAny> = completed_at.add(&skew)?; + let (rows, saturated) = open_rows(history, (&window_start, &window_end))?; let observations: Bound<'py, PyList> = PyList::empty(py); let mut observed: HashSet = HashSet::new(); let mut skipped: Vec<(String, &'static str)> = Vec::new(); for row in rows { let id: String = query_id(&row)?; observed.insert(id.clone()); - match row_observation(&row, expected.contains(&id))? { + let classify: bool = history.classification_query_ids.contains(&id); + match row_observation(&row, (expected.contains(&id), classify))? { Ok(observation) => observations.append(observation)?, Err(reason) => match skipped.iter_mut().find(|(skipped_id, _)| *skipped_id == id) { Some(entry) => entry.1 = reason, @@ -340,6 +390,43 @@ fn query_observations<'py>( }) } +/// The budget a query-history read exhausted, or `None` for any other failure. +fn budget_exhausted(py: Python<'_>, error: &PyErr, budget_seconds: f64) -> Option { + if error.is_instance_of::(py) + && error + .value(py) + .str() + .is_ok_and(|text| text.to_string_lossy() == DEADLINE_MESSAGE) + { + return Some(format!( + "{budget_seconds:.0}-second collection deadline exceeded" + )); + } + statement_timeout(py, error).then(|| { + format!("query-history read reached its {QUERY_TIMEOUT_SECONDS}-second statement timeout") + }) +} + +/// A pending summary for a run whose history did not return within the collection budget. +fn budget_pending( + (run_id, cause): (&str, &str), + statement_count: usize, + failure: Option<&str>, +) -> Summary { + let mut pending: Summary = with_ledger_failure( + partial(&format!( + "Snowflake query history did not return within the cost collection budget \ +({cause}). Run `sqb cost {run_id}` to retry." + )), + failure, + ); + if failure.is_none() { + pending.state.status = PENDING_STATUS; + } + pending.state.counts = [statement_count, 0, statement_count, 0]; + pending +} + /// A collected summary: the allocation it prices, if any, and its derived fields. struct Summary { allocation: Option, @@ -465,6 +552,7 @@ struct Collection<'py> { rate_source: Bound<'py, PyAny>, attempts: i64, retry_delay_seconds: f64, + budget_seconds: f64, } fn collect<'py>( @@ -473,7 +561,7 @@ fn collect<'py>( ledger_path: &Bound<'py, PyAny>, ) -> PyResult { let py: Python<'py> = ledger_path.py(); - history.deadline = monotonic(py)? + DEADLINE_SECONDS; + history.deadline = monotonic(py)? + collection.budget_seconds; let run_id: String = collection.run_id.extract()?; let ledger: Vec> = read_ledger(ledger_path, &collection.run_id)?; let failure: Option = consume_ledger_failure(py, &run_id)?; @@ -485,14 +573,15 @@ fn collect<'py>( } } let mut expected_ids: HashSet = HashSet::new(); - let mut classification: Vec = Vec::new(); + let mut classification: HashSet = HashSet::new(); for (id, entry) in pyo3::types::PyDictMethods::iter(&by_query_id) { let id: String = id.extract()?; if !native_field(&entry, "resource_type")?.eq(HOOK_RESOURCE_TYPE)? { - classification.push(id.clone()); + classification.insert(id.clone()); } expected_ids.insert(id); } + history.text_run_id = (!classification.is_empty()).then(|| run_id.clone()); history.classification_query_ids = classification; if !ledger.is_empty() && expected_ids.is_empty() { let mut summary: Summary = with_ledger_failure( @@ -514,11 +603,26 @@ fn collect<'py>( saturated: false, }; for attempt in 0..collection.attempts { - observed = query_observations( + observed = match query_observations( &history, (&collection.started_at, &collection.completed_at), &expected_ids, - )?; + ) { + Ok(observed) => observed, + Err(error) => { + let Some(cause) = budget_exhausted(py, &error, collection.budget_seconds) else { + return Err(error); + }; + if summary.is_some() { + break; + } + return Ok(budget_pending( + (&run_id, &cause), + ledger.len(), + failure.as_deref(), + )); + } + }; let attributed: Vec> = observed .observations .iter() @@ -621,6 +725,7 @@ fn collect_snowflake_cost<'py>( "attempts", "retry_delay_seconds", "result_limit", + "budget_seconds", ], ), )?; @@ -640,11 +745,15 @@ fn collect_snowflake_cost<'py>( attempts: optional("attempts", 3_i64.into_bound_py_any(py)?)?.extract()?, retry_delay_seconds: optional("retry_delay_seconds", 1.0_f64.into_bound_py_any(py)?)? .extract()?, + budget_seconds: optional("budget_seconds", py.None().into_bound(py))? + .extract::>()? + .unwrap_or(DEADLINE_SECONDS), }; let history: History<'py> = History { connection: arguments.value("connection")?, database: arguments.value("database")?.extract()?, - classification_query_ids: Vec::new(), + text_run_id: None, + classification_query_ids: HashSet::new(), deadline: 0.0, result_limit: optional( "result_limit", @@ -684,14 +793,23 @@ fn snowflake_query_observations<'py>( "classification_query_ids", "deadline", ], - &["database", "result_limit"], + &["database", "result_limit", "run_id"], ), )?; let result_limit: Bound<'py, PyAny> = arguments.value("result_limit")?; + let classification_query_ids: HashSet = + strings(&arguments.value("classification_query_ids")?)? + .into_iter() + .collect(); let history: History<'py> = History { connection: arguments.value("connection")?, database: arguments.value("database")?.extract()?, - classification_query_ids: strings(&arguments.value("classification_query_ids")?)?, + text_run_id: if classification_query_ids.is_empty() { + None + } else { + arguments.value("run_id")?.extract()? + }, + classification_query_ids, deadline: arguments.value("deadline")?.extract()?, result_limit: if result_limit.is_none() { QUERY_HISTORY_RESULT_LIMIT @@ -737,8 +855,8 @@ fn render_snowflake_query_history_sql( "render_snowflake_query_history_sql", options, ( - &["started_at", "classification_query_ids"], - &["database", "completed_at", "result_limit"], + &["started_at"], + &["database", "completed_at", "result_limit", "run_id"], ), )?; let start: String = arguments @@ -753,11 +871,12 @@ fn render_snowflake_query_history_sql( }; let result_limit: Bound<'_, PyAny> = arguments.value("result_limit")?; let database: Option = arguments.value("database")?.extract()?; + let run_id: Option = arguments.value("run_id")?.extract()?; Ok(query_history_sql(&QueryHistoryWindow { database: database.as_deref(), start: &start, end: end.as_deref(), - classification_query_ids: &strings(&arguments.value("classification_query_ids")?)?, + text_run_id: run_id.as_deref(), result_limit: if result_limit.is_none() { QUERY_HISTORY_RESULT_LIMIT } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs index 2fb97dc797..3914bb5227 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs @@ -27,6 +27,7 @@ const CHECK_ASSET_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_f const MATERIALIZE: &str = "materialize"; const MATERIALIZATION: &str = "materialization"; const CHECK: &str = "check"; +const AUDIT_CHECK_KIND: &str = "audit"; /// DAG nodes upstream first, or in artifact order when edges form a cycle. pub(crate) fn sorted_nodes<'py>(dag: &Bound<'py, PyAny>) -> PyResult>> { @@ -241,6 +242,23 @@ fn dag_check_for<'py>( return Ok(Some(dag_check)); } } + singular_audit_check(dag, check) +} + +/// The DAG check `audit:` of a singular audit whose result names where it ran. +fn singular_audit_check<'py>( + dag: &Bound<'py, PyAny>, + check: &Bound<'py, PyAny>, +) -> PyResult>> { + if text_of(check, "kind")? != AUDIT_CHECK_KIND { + return Ok(None); + } + let wanted: String = format!("{AUDIT_CHECK_KIND}:{}", text_of(check, "name")?); + for dag_check in items(dag, "checks")? { + if text_of(&dag_check, "id")? == wanted && get(&dag_check, "attachment_kind")?.is_none() { + return Ok(Some(dag_check)); + } + } Ok(None) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 4450fc5cda..9a3746dd1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_command::support::{boolean, emit, none, text, tuple}; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::dbt_interop::execution::render_dbt_execution_summary_footer; @@ -302,6 +303,17 @@ pub(crate) fn run_dbt_sqlbuild_work<'py>( native_field(&project, "effective_target_name")?, )?, ), + ( + "concurrency", + resolve_concurrency( + &py.None().into_bound(py), + &discovered, + &native_field(&project, "effective_target_name")?, + )? + .limit + .into_pyobject(py)? + .into_any(), + ), ], )?; execute_dbt_sqlbuild_work( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 9c6b9873ad..27ce82fdec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -138,7 +138,7 @@ pub(crate) fn execute_dbt_sqlbuild_work<'py>( native_field(&command, "value")?.extract::()? ) }; - let concurrency = native_field(&project, "settings")?.getattr("concurrency")?; + let concurrency = native_field(&context, "concurrency")?; let header = pyo3::IntoPyObjectExt::into_bound_py_any( format_execution_header( &text(py, &display).extract::()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index b7ff43f7eb..d0b17cbd19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -3,6 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; +use crate::bindings::_helpers::compile_driver::config_validation::{ + ProjectConfiguration, reject_invalid_connection_configuration, +}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::contracts::project_config::{ load_local_config_file, load_project_config_file, @@ -16,6 +19,22 @@ use crate::bindings::_helpers::records::ids::RecordId; pub(crate) fn discover_project_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + configuration_inputs(py, project_dir, false) +} + +/// `discover_project_configuration` with the target warehouses and connections validated. +pub(crate) fn discover_connection_configuration<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + configuration_inputs(py, project_dir, true) +} + +fn configuration_inputs<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + validate_connections: bool, ) -> PyResult> { let lifecycle: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::OperationLifecycle)?, @@ -28,19 +47,24 @@ pub(crate) fn discover_project_configuration<'py>( ], )?; within(&lifecycle, |_| { + let project_config: Bound<'py, PyAny> = + load_project_config_file(py, project_dir)?.into_bound(py); + let local_config: Bound<'py, PyAny> = + load_local_config_file(py, project_dir)?.into_bound(py); + if validate_connections { + reject_invalid_connection_configuration(&ProjectConfiguration { + project_dir, + project_config: &project_config, + local_config: &local_config, + })?; + } new_record_with( py, RecordId::DiscoveredProjectInputs, &[ ("project_dir", project_dir.clone()), - ( - "project_config", - load_project_config_file(py, project_dir)?.into_bound(py), - ), - ( - "local_config", - load_local_config_file(py, project_dir)?.into_bound(py), - ), + ("project_config", project_config), + ("local_config", local_config), ], ) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index 4a8806e1bf..639f068c24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -156,16 +156,21 @@ pub(crate) fn column_name(column: &Bound<'_, PyAny>) -> PyResult { native_field(column, "name")?.extract() } -fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult { +/// The CSV reader settings a seed declaration authors, without the dialect check. +pub(crate) fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { let settings: Bound<'_, PyAny> = native_field(entry, "csv_settings")?; - let mut csv: CsvSettings = CsvSettings { + Ok(CsvSettings { delimiter: optional_text(&settings, "delimiter")?, quotechar: optional_text(&settings, "quotechar")?, escapechar: optional_text(&settings, "escapechar")?, doublequote: native_field(&settings, "doublequote")?.extract()?, skipinitialspace: native_field(&settings, "skipinitialspace")?.extract()?, dialect_error: None, - }; + }) +} + +fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult { + let mut csv: CsvSettings = seed_csv_settings(entry)?; csv.dialect_error = csv_dialect_error(py, &csv)?; Ok(SeedEntryFact { name: column_name(entry)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs index 7e9756e372..6ceb9fe54e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -5,6 +5,7 @@ pub(crate) const DISPATCHER_THREAD_NAME: &str = "sqlbuild-event-exporter-dispatc pub(crate) const NOTIFIER_THREAD_NAME: &str = "sqlbuild-event-exporter-notifier"; pub(crate) const HEALTH_THREAD_NAME: &str = "sqlbuild-event-exporter-health"; pub(crate) const OUTPUT_SINK_THREAD_NAME: &str = "sqlbuild-command-output-sink"; +/// Prefix of each exporter's long-lived delivery thread name. pub(crate) const INVOCATION_THREAD_PREFIX: &str = "sqlbuild-event-exporter-"; /// Attribute naming the worker of a long-lived dispatcher thread, for the exit hook. pub(crate) const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; @@ -14,5 +15,9 @@ pub(crate) const POLL_SECONDS: f64 = 0.01; pub(crate) const JOIN_GRACE_SECONDS: f64 = 0.05; /// How long shutdown and interpreter exit wait for each stopped dispatcher thread. pub(crate) const EXIT_JOIN_SECONDS: f64 = 1.0; +/// The most attempts one exporter's delivery thread holds, at most the queue capacity; more stay queued. +pub(crate) const LANE_WINDOW: usize = 64; +/// The longest a finalizer waits for sink calls still running after shutdown before it runs anyway. +pub(crate) const IDLE_WAIT_SECONDS: f64 = 1.0; /// No time left before a deadline. pub(crate) const NO_SECONDS: f64 = 0.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index a736ac6130..9bdef06fc7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -1,6 +1,6 @@ //! `EventExporterDispatcher`: bounded priority delivery of lifecycle events on Python daemon threads. -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; @@ -14,6 +14,7 @@ use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_TERMINAL_EVENT_TYPES, + OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, }; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; @@ -28,8 +29,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::event_exporting::constants::{ - DISPATCHER_THREAD_NAME, EXIT_JOIN_SECONDS, HEALTH_THREAD_NAME, JOIN_GRACE_SECONDS, NO_SECONDS, - NOTIFIER_THREAD_NAME, OUTPUT_SINK_THREAD_NAME, POLL_SECONDS, WORKER_ATTRIBUTE, + DISPATCHER_THREAD_NAME, EXIT_JOIN_SECONDS, HEALTH_THREAD_NAME, IDLE_WAIT_SECONDS, + JOIN_GRACE_SECONDS, NO_SECONDS, NOTIFIER_THREAD_NAME, OUTPUT_SINK_THREAD_NAME, POLL_SECONDS, + WORKER_ATTRIBUTE, }; use crate::bindings::_helpers::event_exporting::models::{ BoundEventExporter, EventExportSummary, EventExporterFailure, @@ -37,7 +39,7 @@ use crate::bindings::_helpers::event_exporting::models::{ use crate::bindings::_helpers::event_exporting::values::{ export_policy, input_error, severity_from, state_error, }; -use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, WorkerRole}; +use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, Lane, WorkerRole}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; @@ -80,12 +82,23 @@ pub(crate) struct State { sequence: u64, accepting: bool, pub(crate) deadline: Option, + /// When only outcome events remain deliverable, ahead of the deadline. + pub(crate) reserve_at: Option, + pub(crate) reserved: bool, + /// Each exporter's long-lived delivery thread, once started and until retired. + pub(crate) lanes: Vec>, + /// The exporter each delivered-but-unreported attempt token belongs to. + pub(crate) tokens: HashMap, + pub(crate) next_token: u64, + /// Retired lanes, whose threads are joined before finalizers run. + pub(crate) retired: Vec, + /// Whether a timed-out sink call may still be running. + pub(crate) abandoned_calls: bool, pub(crate) stopping: bool, pub(crate) force_stop: bool, pub(crate) blocked: HashSet, pub(crate) live_invocations: HashSet, pub(crate) next_invocation: u64, - idle_finalizers: Vec>, dispatcher_running: bool, shutdown_started: bool, shutdown_complete: bool, @@ -103,6 +116,10 @@ pub(crate) struct State { /// What the dispatcher and its threads share. pub(crate) struct Shared { condition: Py, + /// The `SimpleQueue` every delivery thread reports `(token, error)` outcomes to. + pub(crate) results: Py, + /// Attempt tokens a delivery thread claimed (`True`) before running, or the dispatcher cancelled (`False`). + pub(crate) claims: Py, state: Mutex, pub(crate) invocation_timeout: Duration, shutdown_timeout: Duration, @@ -162,13 +179,6 @@ fn remaining_seconds(deadline: Instant) -> f64 { .as_secs_f64() } -/// Run finalizers, ignoring their failures. -pub(crate) fn run_finalizers(py: Python<'_>, finalizers: Vec>) { - for finalizer in finalizers { - drop(finalizer.bind(py).call0()); - } -} - /// End a retired command-output sink thread once it has run every call queued before. fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { if let Some(calls) = retired { @@ -338,6 +348,34 @@ pub(crate) fn stop_event_exporter_threads(py: Python<'_>) -> PyResult<()> { Ok(()) } +impl State { + /// Count one failed attempt and queue its sanitized diagnostic, returning whether one was queued. + pub(crate) fn note_failure( + &mut self, + diagnostics: Option, + (index, kind, severity): (usize, &str, ExportSeverity), + error_type: String, + ) -> bool { + let mut exporter_name: String = String::new(); + if let Some(counts) = self.counts.get_mut(index) { + counts.failed += 1; + } + if let Some(exporter) = self.exporters.get(index) { + exporter_name.clone_from(&exporter.name); + } + if diagnostics.is_none_or(|capacity| self.failures.len() >= capacity) { + return false; + } + self.failures.push_back(EventExporterFailure { + exporter_name, + error_type, + event_kind: kind.to_owned(), + event_severity: severity, + }); + true + } +} + impl Shared { pub(crate) fn hold<'py>(&self, py: Python<'py>) -> PyResult> { let condition = self.condition.bind(py).clone(); @@ -385,7 +423,8 @@ impl Shared { queue_capacity: state.queue.capacity, drained: final_snapshot && !state.dispatcher_running - && state.live_invocations.is_empty(), + && state.live_invocations.is_empty() + && !state.abandoned_calls, }) } @@ -395,81 +434,76 @@ impl Shared { Ok(Self::snapshot(&*self.locked()?, false)) } - /// Count one failed attempt and queue its sanitized diagnostic. - pub(crate) fn record_failure( - &self, - py: Python<'_>, - (index, kind, severity): (usize, &str, ExportSeverity), - error_type: String, - ) -> PyResult<()> { - let _held = self.hold(py)?; - let mut state = self.locked()?; - let mut exporter_name: String = String::new(); - if let Some(counts) = state.counts.get_mut(index) { - counts.failed += 1; - } - if let Some(exporter) = state.exporters.get(index) { - exporter_name.clone_from(&exporter.name); - } - if !self.notifications - || self.failure_callback.is_none() - || state.failures.len() >= self.notification_capacity - { - return Ok(()); - } - state.failures.push_back(EventExporterFailure { - exporter_name, - error_type, - event_kind: kind.to_owned(), - event_severity: severity, - }); - drop(state); - self.notify_all(py) + /// The diagnostic queue capacity when failures are reported, else `None`. + pub(crate) fn diagnostics(&self) -> Option { + (self.notifications && self.failure_callback.is_some()) + .then_some(self.notification_capacity) } - /// Forget one finished invocation; once idle, run the finalizers waiting for it. - pub(crate) fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { - let (finalizers, retired): (Vec>, Option>) = { + /// Forget one finished invocation and wake a caller waiting for delivery to become idle. + fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { + let retired: Option> = { let _held = self.hold(py)?; let mut state = self.locked()?; state.live_invocations.remove(&invocation); - if state.live_invocations.is_empty() && !state.dispatcher_running { - let retired: Option> = if state.shutdown_complete { - state.output_lane.take() - } else { - None - }; - (std::mem::take(&mut state.idle_finalizers), retired) + let retired: Option> = if Self::idle(&state) && state.shutdown_complete { + state.output_lane.take() } else { - (Vec::new(), None) - } + None + }; + drop(state); + self.notify_all(py)?; + retired }; - retire_output_lane(py, retired)?; - run_finalizers(py, finalizers); - Ok(()) + retire_output_lane(py, retired) } - /// Mark the dispatcher thread finished and run idle finalizers. + /// Mark the dispatcher thread finished. pub(crate) fn dispatcher_finished(&self, py: Python<'_>) -> PyResult<()> { - let finalizers: Vec> = { + let _held = self.hold(py)?; + self.locked()?.dispatcher_running = false; + self.notify_all(py) + } + + /// Whether the dispatcher thread and every sink call have finished. + fn idle(state: &State) -> bool { + state.live_invocations.is_empty() && !state.dispatcher_running + } + + /// Wait until delivery and every sink call are idle, the idle wait ends or interpreter exit begins. + fn await_idle(&self, py: Python<'_>) -> PyResult<()> { + let deadline: Instant = + Instant::now() + self.invocation_timeout.min(seconds(IDLE_WAIT_SECONDS)); + let threads: Vec> = { let _held = self.hold(py)?; - let mut state = self.locked()?; - state.dispatcher_running = false; - let finalizers = if state.live_invocations.is_empty() { - std::mem::take(&mut state.idle_finalizers) - } else { - Vec::new() - }; - drop(state); - self.notify_all(py)?; - finalizers + loop { + if interpreter_exiting(py)? { + return Ok(()); + } + let state = self.locked()?; + if Self::idle(&state) { + break state + .retired + .iter() + .map(|lane| lane.thread().clone_ref(py)) + .collect(); + } + drop(state); + let remaining: f64 = remaining_seconds(deadline); + if remaining <= NO_SECONDS { + return Ok(()); + } + self.wait(py, remaining.min(POLL_SECONDS))?; + } }; - run_finalizers(py, finalizers); + for thread in threads { + join(thread.bind(py), remaining_seconds(deadline))?; + } Ok(()) } /// Register a running invocation, returning its identity. - pub(crate) fn begin_invocation(&self, py: Python<'_>) -> PyResult { + fn begin_invocation(&self, py: Python<'_>) -> PyResult { let _held = self.hold(py)?; let mut state = self.locked()?; state.next_invocation += 1; @@ -644,6 +678,8 @@ impl EventExporterDispatcher { let has_summary: bool = summary_callback.is_some(); let shared = Arc::new(Shared { condition: imported(py, "threading", "Condition")?.call0()?.unbind(), + results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), + claims: PyDict::new(py).unbind(), state: Mutex::new(State { bound: !exporters.is_none(), counts: vec![ExporterCounts::default(); bound_exporters.len()], @@ -655,12 +691,18 @@ impl EventExporterDispatcher { sequence: 0, accepting: true, deadline: None, + reserve_at: None, + reserved: false, + lanes: Vec::new(), + tokens: HashMap::new(), + next_token: 0, + retired: Vec::new(), + abandoned_calls: false, stopping: false, force_stop: false, blocked: HashSet::new(), live_invocations: HashSet::new(), next_invocation: 0, - idle_finalizers: Vec::new(), dispatcher_running: true, shutdown_started: false, shutdown_complete: false, @@ -759,6 +801,7 @@ impl EventExporterDispatcher { }, priority: policy.priority, invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), + outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), eligible_exporters: eligible, }, ); @@ -862,7 +905,14 @@ impl EventExporterDispatcher { if owns { state.shutdown_started = true; state.accepting = false; - state.deadline = Some(Instant::now() + shared.shutdown_timeout); + let started: Instant = Instant::now(); + state.deadline = Some(started + shared.shutdown_timeout); + state.reserve_at = Some( + started + + shared + .shutdown_timeout + .mul_f64(1.0 - OUTCOME_RESERVE_FRACTION), + ); state.stopping = true; state.health_stopping = true; } @@ -922,17 +972,10 @@ impl EventExporterDispatcher { Ok(summary) } - /// Run `finalizer` once the dispatcher thread and every live invocation have finished. + /// Run `finalizer` on the calling thread, never a daemon delivery thread, once delivery is idle or the idle wait ends. fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { - { - let _held = self.shared.hold(py)?; - let mut state = self.shared.locked()?; - if !state.live_invocations.is_empty() || state.dispatcher_running { - state.idle_finalizers.push(finalizer); - return Ok(()); - } - } - run_finalizers(py, vec![finalizer]); + self.shared.await_idle(py)?; + drop(finalizer.bind(py).call0()); Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs index ebe2562456..40d42b4598 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::event_exporting::models::{ BoundEventExporter, EventExportSummary, EventExporterAccounting, EventExporterCounts, EventExporterFailure, }; -use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, InvocationCompletion}; +use crate::bindings::_helpers::event_exporting::workers::DispatcherWorker; use crate::bindings::_helpers::event_exporting::{policy, scope}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,7 +21,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - module.add_class::()?; let hook = wrap_pyfunction!(stop_event_exporter_threads, module)?; module.add_function(hook.clone())?; imported(module.py(), "atexit", "register")?.call1((hook,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs index 45396ddb7b..f3a5b2a94c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs @@ -41,7 +41,7 @@ pub(crate) struct EventExporterCommandScope { state: Mutex, } -/// Releases and force-closes a provider session once sink delivery is idle. +/// Releases and force-closes a provider session on the closing thread once sink delivery is idle. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct ProviderSessionCloser { session: Py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs index 1cde088682..ca07475d2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs @@ -1,28 +1,30 @@ -//! The dispatcher's delivery, notification, health and per-invocation thread targets. +//! The dispatcher's delivery, notification and health thread targets, and each exporter's delivery lane. -use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::{Duration, Instant}; -use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_core::event_export::constants::TIMEOUT_ERROR_TYPE; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; +use sqlbuild_core::event_export::main::retain_outcome_events::retain_outcome_events; use sqlbuild_core::event_export::main::take_lifecycle_event::take_lifecycle_event; -use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEvent}; +use sqlbuild_core::event_export::models::{ + EventQueue, ExportSeverity, ExportSnapshot, QueuedEvent, +}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::event_exporting::constants::{ - INVOCATION_THREAD_PREFIX, NO_SECONDS, POLL_SECONDS, + INVOCATION_THREAD_PREFIX, LANE_WINDOW, NO_SECONDS, POLL_SECONDS, }; -use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared}; +use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared, State}; use crate::bindings::_helpers::event_exporting::models::{ EventExportSummary, EventExporterFailure, }; use crate::bindings::_helpers::event_exporting::values::error_type_of; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::native_threads::exit_signal::interpreter_exiting; /// Which long-lived dispatcher thread a worker runs. #[derive(Clone, Copy)] @@ -39,19 +41,28 @@ pub(crate) struct DispatcherWorker { role: WorkerRole, } -/// Whether an invocation finished, and the sanitized type of what it raised. -#[derive(Default)] -struct Completion { - done: bool, - error_type: Option, +/// One attempt handed to a lane and not yet reported. +pub(crate) struct Attempt { + token: u64, + kind: String, + severity: ExportSeverity, + outcome: bool, } -/// Reports how one sink call on its own daemon thread ended. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct InvocationCompletion { - shared: Arc, - invocation: u64, - completion: Arc>, +/// One exporter's long-lived delivery thread, its call queue and the attempts it still owes. +pub(crate) struct Lane { + calls: Py, + thread: Py, + pending: VecDeque, + /// When the attempt at the front of `pending` began, approximately. + head_started: Instant, +} + +/// What one dispatch round reads from the dispatcher besides its state. +#[derive(Clone, Copy)] +pub(crate) struct Round<'a> { + shared: &'a Shared, + diagnostics: Option, } /// The next thing the notification thread delivers. @@ -62,12 +73,308 @@ enum Notification { Stop, } -/// Whether the call finished, and the sanitized type of what it raised. -fn completion_state(completion: &Mutex) -> PyResult<(bool, Option)> { - let completion = completion - .lock() - .map_err(|_| PyRuntimeError::new_err("event exporter invocation lock is poisoned"))?; - Ok((completion.done, completion.error_type.clone())) +impl Lane { + /// Start `exporter`'s delivery thread, which claims each call in `claims` and reports to `results`. + fn start(py: Python<'_>, exporter: &Exporter, round: Round<'_>) -> PyResult { + let calls: Bound<'_, PyAny> = imported(py, "queue", "SimpleQueue")?.call0()?; + let arguments = PyDict::new(py); + arguments.set_item("calls", &calls)?; + arguments.set_item("results", round.shared.results.bind(py))?; + arguments.set_item("claims", round.shared.claims.bind(py))?; + let thread = call_with( + &imported(py, "threading", "Thread")?, + &[ + ( + "target", + host::entry(py, UserCodeHost::InvokeLifecycleEventSink)?, + ), + ("kwargs", arguments.into_any()), + ( + "name", + PyString::new(py, &format!("{INVOCATION_THREAD_PREFIX}{}", exporter.name)) + .into_any(), + ), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.call_method0("start")?; + Ok(Self { + calls: calls.unbind(), + thread: thread.unbind(), + pending: VecDeque::new(), + head_started: Instant::now(), + }) + } + + /// The lane's delivery thread. + pub(crate) fn thread(&self) -> &Py { + &self.thread + } + + /// Queue `None` so the thread ends after the calls queued before it. + fn retire(&self, py: Python<'_>) -> PyResult<()> { + self.calls.bind(py).call_method1("put", (py.None(),))?; + Ok(()) + } +} + +impl State { + /// Account reported attempts against their lanes, returning whether a diagnostic was queued. + fn record_reports( + &mut self, + py: Python<'_>, + reports: Vec<(u64, Option)>, + round: Round<'_>, + ) -> PyResult { + let claims: &Bound<'_, PyDict> = round.shared.claims.bind(py); + let mut notify: bool = false; + for (token, error_type) in reports { + if claims.contains(token)? { + claims.del_item(token)?; + } + let Some(index) = self.tokens.remove(&token) else { + continue; + }; + let Some(lane) = self.lanes.get_mut(index).and_then(Option::as_mut) else { + continue; + }; + let Some(position) = lane + .pending + .iter() + .position(|attempt| attempt.token == token) + else { + continue; + }; + let Some(attempt) = lane.pending.remove(position) else { + continue; + }; + lane.head_started = Instant::now(); + match error_type { + Some(error_type) => { + notify |= self.note_failure( + round.diagnostics, + (index, &attempt.kind, attempt.severity), + error_type, + ); + } + None => { + if let Some(counts) = self.counts.get_mut(index) { + counts.delivered += 1; + } + } + } + } + Ok(notify) + } + + /// Cancel `attempts` the delivery thread has not claimed (checked and marked under the GIL), counting each dropped; returns the claimed. + fn cancel_unclaimed( + &mut self, + py: Python<'_>, + index: usize, + attempts: impl IntoIterator, + claims: &Py, + ) -> PyResult> { + let claims: &Bound<'_, PyDict> = claims.bind(py); + let mut claimed: Vec = Vec::new(); + for attempt in attempts { + let already_claimed: bool = match claims.get_item(attempt.token)? { + Some(claim) => claim.is_truthy()?, + None => false, + }; + if already_claimed { + claimed.push(attempt); + continue; + } + claims.set_item(attempt.token, false)?; + self.tokens.remove(&attempt.token); + self.counts = drop_lifecycle_attempts(std::mem::take(&mut self.counts), &[index]); + } + Ok(claimed) + } + + /// Retire `index`'s lane: calls it already claimed fail as timed out and the rest are dropped. + fn abandon_lane(&mut self, py: Python<'_>, index: usize, round: Round<'_>) -> PyResult { + let Some(mut lane) = self.lanes.get_mut(index).and_then(Option::take) else { + return Ok(false); + }; + let attempts: VecDeque = std::mem::take(&mut lane.pending); + let claimed: Vec = + self.cancel_unclaimed(py, index, attempts, &round.shared.claims)?; + let claims: &Bound<'_, PyDict> = round.shared.claims.bind(py); + let mut notify: bool = false; + for running in claimed { + claims.del_item(running.token)?; + self.tokens.remove(&running.token); + self.blocked.insert(index); + self.abandoned_calls = true; + notify |= self.note_failure( + round.diagnostics, + (index, &running.kind, running.severity), + TIMEOUT_ERROR_TYPE.to_owned(), + ); + } + lane.retire(py)?; + self.retired.push(lane); + Ok(notify) + } + + /// Abandon every lane whose current sink call outlived `timeout`. + fn expire_lanes( + &mut self, + py: Python<'_>, + timeout: Duration, + round: Round<'_>, + ) -> PyResult { + let mut notify: bool = false; + for index in 0..self.lanes.len() { + let expired: bool = self.lanes[index].as_ref().is_some_and(|lane| { + !lane.pending.is_empty() && lane.head_started.elapsed() >= timeout + }); + if expired { + notify |= self.abandon_lane(py, index, round)?; + } + } + Ok(notify) + } + + /// Drop every queued and lane-waiting event that is not an outcome, counting each attempt. + fn reserve_for_outcomes(&mut self, py: Python<'_>, claims: &Py) -> PyResult<()> { + let capacity: usize = self.queue.capacity; + let (queue, dropped) = retain_outcome_events(std::mem::replace( + &mut self.queue, + EventQueue::new(capacity), + )); + self.queue = queue; + for item in &dropped { + self.counts = + drop_lifecycle_attempts(std::mem::take(&mut self.counts), &item.eligible_exporters); + } + for index in 0..self.lanes.len() { + let Some(lane) = self.lanes[index].as_mut() else { + continue; + }; + let (kept, skipped): (VecDeque, VecDeque) = + lane.pending.drain(..).partition(|attempt| attempt.outcome); + let claimed: Vec = self.cancel_unclaimed(py, index, skipped, claims)?; + let Some(lane) = self.lanes[index].as_mut() else { + continue; + }; + let mut pending: Vec = claimed.into_iter().chain(kept).collect(); + pending.sort_by_key(|attempt| attempt.token); + lane.pending = pending.into(); + } + drop(dropped); + Ok(()) + } + + /// Move queued events onto exporter lanes while every eligible lane has room. + fn feed(&mut self, py: Python<'_>, round: Round<'_>) -> PyResult { + let mut notify: bool = false; + while !self.queue.items.is_empty() { + let capacity: usize = self.queue.capacity; + let (queue, next) = take_lifecycle_event(std::mem::replace( + &mut self.queue, + EventQueue::new(capacity), + )); + self.queue = queue; + let Some(item) = next else { + break; + }; + let full: bool = item.eligible_exporters.iter().any(|index| { + !self.blocked.contains(index) + && self + .lanes + .get(*index) + .and_then(Option::as_ref) + .is_some_and(|lane| lane.pending.len() >= LANE_WINDOW.min(capacity)) + }); + if full { + self.queue.items.push(item); + break; + } + for index in &item.eligible_exporters { + if self.blocked.contains(index) { + self.counts = + drop_lifecycle_attempts(std::mem::take(&mut self.counts), &[*index]); + continue; + } + notify |= self.offer(py, *index, &item, round)?; + } + } + Ok(notify) + } + + /// Queue one attempt on `index`'s lane, starting the lane's thread on first use. + fn offer( + &mut self, + py: Python<'_>, + index: usize, + item: &QueuedEvent, + round: Round<'_>, + ) -> PyResult { + let Some(exporter) = self.exporters.get(index).map(Arc::clone) else { + return Ok(false); + }; + if self.lanes.len() <= index { + self.lanes.resize_with(index + 1, || None); + } + if self.lanes[index].is_none() { + match Lane::start(py, &exporter, round) { + Ok(lane) => self.lanes[index] = Some(lane), + Err(error) => { + return Ok(self.note_failure( + round.diagnostics, + (index, &item.item.kind, item.item.severity), + error_type_of(py, &error), + )); + } + } + } + let token: u64 = self.next_token; + self.next_token += 1; + let Some(lane) = self.lanes[index].as_mut() else { + return Ok(false); + }; + lane.calls.bind(py).call_method1( + "put", + (( + exporter.function.bind(py), + exporter.provider_arguments.bind(py), + item.item.event.bind(py), + token, + ),), + )?; + if lane.pending.is_empty() { + lane.head_started = Instant::now(); + } + lane.pending.push_back(Attempt { + token, + kind: item.item.kind.clone(), + severity: item.item.severity, + outcome: item.outcome, + }); + self.tokens.insert(token, index); + Ok(false) + } + + /// Whether any lane still owes a reported attempt. + fn busy(&self) -> bool { + self.lanes + .iter() + .flatten() + .any(|lane| !lane.pending.is_empty()) + } + + /// End every lane once it has run what was queued before, keeping its thread to join. + fn retire_lanes(&mut self, py: Python<'_>) -> PyResult<()> { + let lanes: Vec = self.lanes.drain(..).flatten().collect(); + for lane in lanes { + lane.retire(py)?; + self.retired.push(lane); + } + Ok(()) + } } impl DispatcherWorker { @@ -97,165 +404,78 @@ impl DispatcherWorker { } } - /// The next queued event, `Ok(None)` to keep polling and `Err(())` once delivery stops. - fn next_event(&self, py: Python<'_>) -> PyResult>, ()>> { - let shared = &self.shared; - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if Shared::past_deadline(&state) || state.force_stop { - let drained: Vec> = std::mem::take(&mut state.queue.items); - for item in &drained { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &item.eligible_exporters, - ); - } - drop(state); - drop(drained); - return Ok(Err(())); - } - if state.queue.items.is_empty() { - drop(state); - shared.wait(py, POLL_SECONDS)?; - state = shared.locked()?; - } - let capacity: usize = state.queue.capacity; - let (queue, next) = take_lifecycle_event(std::mem::replace( - &mut state.queue, - EventQueue::new(capacity), - )); - state.queue = queue; - match next { - Some(item) => Ok(Ok(Some(item))), - None if state.stopping => Ok(Err(())), - None => Ok(Ok(None)), - } - } - fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { let result = (|| -> PyResult<()> { if !self.await_bound(py)? { return Ok(()); } - while let Ok(item) = self.next_event(py)? { - if let Some(item) = item { - self.deliver(py, &item)?; - } - } + while self.dispatch_round(py)? {} Ok(()) })(); + let retired: PyResult<()> = (|| { + let _held = self.shared.hold(py)?; + self.shared.locked()?.retire_lanes(py) + })(); self.shared.dispatcher_finished(py)?; + retired?; result } - /// Offer one event to each eligible exporter in order, before the drain deadline. - fn deliver(&self, py: Python<'_>, item: &QueuedEvent) -> PyResult<()> { - let shared = &self.shared; - for (position, index) in item.eligible_exporters.iter().enumerate() { - let exporter: Arc = { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if Shared::past_deadline(&state) { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &item.eligible_exporters[position..], - ); - return Ok(()); - } - if state.blocked.contains(index) { - state.counts = - drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[*index]); - continue; - } - let Some(exporter) = state.exporters.get(*index) else { - continue; - }; - Arc::clone(exporter) - }; - self.invoke(py, (*index, exporter), item)?; - } - Ok(()) - } - - /// Call one exporter on its own daemon thread and account for its outcome. - fn invoke( - &self, - py: Python<'_>, - (index, exporter): (usize, Arc), - item: &QueuedEvent, - ) -> PyResult<()> { + /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. + fn dispatch_round(&self, py: Python<'_>) -> PyResult { let shared = &self.shared; - let failure = (index, item.item.kind.as_str(), item.item.severity); - let completion: Arc> = Arc::new(Mutex::new(Completion::default())); - let name: String = format!("{INVOCATION_THREAD_PREFIX}{}", exporter.name); - let invocation: u64 = shared.begin_invocation(py)?; - let reporter = InvocationCompletion::new(Arc::clone(shared), invocation, &completion); - let started = Py::new(py, reporter).and_then(|reporter| { - let arguments = PyDict::new(py); - arguments.set_item("function", exporter.function.bind(py))?; - arguments.set_item("provider_arguments", exporter.provider_arguments.bind(py))?; - arguments.set_item("event", item.item.event.bind(py))?; - arguments.set_item("completion", reporter)?; - let thread = call_with( - &imported(py, "threading", "Thread")?, - &[ - ( - "target", - host::entry(py, UserCodeHost::InvokeLifecycleEventSink)?, - ), - ("kwargs", arguments.into_any()), - ("name", PyString::new(py, &name).into_any()), - ("daemon", PyBool::new(py, true).to_owned().into_any()), - ], - )?; - thread.call_method0("start") - }); - if let Err(error) = started { - shared.invocation_finished(py, invocation)?; - return shared.record_failure(py, failure, error_type_of(py, &error)); - } - let completed: bool = self.await_invocation(py, &completion)?; - let (_, error_type) = completion_state(&completion)?; - if !completed { - { - let _held = shared.hold(py)?; - shared.locked()?.blocked.insert(index); + let round = Round { + shared, + diagnostics: shared.diagnostics(), + }; + let _held = shared.hold(py)?; + let reports: Vec<(u64, Option)> = self.reported(py)?; + let mut state = shared.locked()?; + let mut notify: bool = state.record_reports(py, reports, round)?; + let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { + let drained: Vec> = std::mem::take(&mut state.queue.items); + for item in &drained { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &item.eligible_exporters, + ); } - return shared.record_failure(py, failure, TIMEOUT_ERROR_TYPE.to_owned()); - } - if let Some(error_type) = error_type { - return shared.record_failure(py, failure, error_type); + for index in 0..state.lanes.len() { + notify |= state.abandon_lane(py, index, round)?; + } + drop(drained); + true + } else { + notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; + if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { + state.reserved = true; + state.reserve_for_outcomes(py, &shared.claims)?; + } + notify |= state.feed(py, round)?; + state.stopping && !state.busy() && state.queue.items.is_empty() + }; + drop(state); + if notify { + shared.notify_all(py)?; } - let _held = shared.hold(py)?; - if let Some(counts) = shared.locked()?.counts.get_mut(index) { - counts.delivered += 1; + if !stopped { + shared.wait(py, POLL_SECONDS)?; } - Ok(()) + Ok(!stopped) } - /// Wait for an invocation until its own timeout or the drain deadline, whichever is first. - fn await_invocation(&self, py: Python<'_>, completion: &Mutex) -> PyResult { - let shared = &self.shared; - let invocation_deadline: Instant = Instant::now() + shared.invocation_timeout; - let _held = shared.hold(py)?; - loop { - let state = shared.locked()?; - let effective: Instant = state.deadline.map_or(invocation_deadline, |deadline| { - deadline.min(invocation_deadline) - }); - let force_stop: bool = state.force_stop || interpreter_exiting(py)?; - drop(state); - let remaining: f64 = effective - .saturating_duration_since(Instant::now()) - .as_secs_f64(); - if remaining <= NO_SECONDS || force_stop { - return Ok(completion_state(completion)?.0); - } - shared.wait(py, remaining.min(POLL_SECONDS))?; - if completion_state(completion)?.0 { - return Ok(true); - } + /// Every `(token, sanitized error type)` delivery threads reported since the last round. + fn reported(&self, py: Python<'_>) -> PyResult)>> { + let results: &Bound<'_, PyAny> = self.shared.results.bind(py); + let mut reports: Vec<(u64, Option)> = Vec::new(); + while !results.call_method0("empty")?.is_truthy()? { + let (token, error): (u64, Bound<'_, PyAny>) = + results.call_method0("get_nowait")?.extract()?; + let error_type: Option = + (!error.is_none()).then(|| error_type_of(py, &PyErr::from_value(error))); + reports.push((token, error_type)); } + Ok(reports) } fn next_notification(&self, py: Python<'_>) -> PyResult { @@ -376,34 +596,3 @@ impl DispatcherWorker { } } } - -impl InvocationCompletion { - /// A reporter that records into `completion` and ends `invocation`. - fn new(shared: Arc, invocation: u64, completion: &Arc>) -> Self { - Self { - shared, - invocation, - completion: Arc::clone(completion), - } - } -} - -#[pymethods] -impl InvocationCompletion { - #[pyo3(signature = (error, /))] - fn __call__(&self, py: Python<'_>, error: Option<&Bound<'_, PyAny>>) -> PyResult<()> { - if let Some(error) = error { - let error_type: String = error_type_of(py, &PyErr::from_value(error.clone())); - if let Ok(mut completion) = self.completion.lock() { - completion.error_type = Some(error_type); - } - } - let finished = self.shared.invocation_finished(py, self.invocation); - let _held = self.shared.hold(py)?; - if let Ok(mut completion) = self.completion.lock() { - completion.done = true; - } - self.shared.notify_all(py)?; - finished - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs index 64a98f3881..7b312e7408 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs @@ -13,13 +13,12 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::rules_config::load_rules_config; -/// Format settings and whether lint suppressions are allowed. +/// Format settings resolved for one run. pub(crate) struct FormatConfig { pub(crate) settings: FormatSettings, - pub(crate) allow_suppressions: bool, } -/// Resolve the settings, loading the rules configuration when the project file exists. +/// Resolve the settings, validating the rules configuration when the project file exists. pub(crate) fn format_config( py: Python<'_>, project_dir: &Bound<'_, PyAny>, @@ -27,23 +26,15 @@ pub(crate) fn format_config( let directory: std::path::PathBuf = project_dir.extract()?; let settings: FormatSettings = resolve_format_settings(Path::new(&directory)).map_err(PyValueError::new_err)?; - let allow_suppressions: bool = if settings.has_project_config { - load_rules_config( + if settings.has_project_config { + let _ = load_rules_config( py, project_dir .call_method0("resolve")? .extract::()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .getattr("allow_exceptions")? - .extract()? - } else { - true - }; - Ok(FormatConfig { - settings, - allow_suppressions, - }) + )?; + } + Ok(FormatConfig { settings }) } /// The Python `LintConfig` of `config`, linting as `dialect`. @@ -74,12 +65,6 @@ pub(crate) fn lint_config<'py>( "max_ranking_order_by", settings.max_ranking_order_by.into_pyobject(py)?.into_any(), ), - ( - "allow_suppressions", - pyo3::types::PyBool::new(py, config.allow_suppressions) - .to_owned() - .into_any(), - ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 9826bf58d9..bd12368073 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -25,8 +25,8 @@ use crate::bindings::_helpers::rules::dsl_headers::safe_format_files; use crate::bindings::_helpers::rules::dsl_headers::{header_violations, rules_config}; use crate::bindings::_helpers::rules::fixture_formatting::format_redundant_fixture_nulls; use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; -use crate::bindings::_helpers::rules::lint_text::apply_lint_suppressions; use crate::bindings::_helpers::rules::lint_text::format_lint_sql_bodies; +use crate::bindings::_helpers::rules::lint_text::reject_lint_suppressions; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderViolation}; @@ -303,9 +303,12 @@ pub(crate) fn run_format<'py>(request: &FormatRequest<'_, 'py>) -> PyResult
> = drop_resolved_description_faults(request.project_dir, violations)?; let final_contents: Bound<'py, PyDict> = overlaid(&files, &updated)?; - let allow_suppressions: bool = native_field(request.config, "allow_suppressions")?.extract()?; - let violations: Vec> = - apply_lint_suppressions(py, violations, final_contents.clone(), allow_suppressions)?; + let violations: Vec> = reject_lint_suppressions( + py, + violations, + final_contents.clone(), + request.project_dir.extract()?, + )?; Ok(FormatOutcome { files_checked, violations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index e7b86460f5..8007be6dac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -19,9 +19,9 @@ use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_target_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::call_with; @@ -300,16 +300,12 @@ fn prepare<'py>( ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? .extract()?; - let requested: Bound<'py, PyAny> = native_field(request, "concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &build_effective_settings_config(&discovered.clone())?, - "concurrency", - )? - .extract()? - } else { - requested.extract()? - }; + let concurrency: i64 = resolve_concurrency( + &native_field(request, "concurrency")?, + discovered, + &native_field(request, "selected_target")?, + )? + .limit; let typed: Bound<'py, PyAny> = type_cursor_overrides(&native_field(request, "cursor_overrides")?, &[])?; let cursor_overrides: Bound<'py, PyAny> = if typed.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 9d67a16ef4..bf5a563cdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -304,6 +304,7 @@ fn check_microbatch_concurrency<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], + concurrency: i64, ) -> PyResult>> { let py = project.py(); let skip = planner_type(py, RecordId::PlanAction, "SKIP")?; @@ -353,7 +354,7 @@ fn check_microbatch_concurrency<'py>( concurrent.push(entry); } } - if concurrent.is_empty() || !native_field(&settings, "concurrency")?.gt(1)? { + if concurrent.is_empty() || concurrency <= 1 { return Ok(None); } let count = concurrent.len(); @@ -404,6 +405,7 @@ pub(crate) fn plan_entries<'py>( cursor_overrides, full_refresh, build_inputs: inputs, + concurrency, } = request; let py = project.py(); let inputs = if inputs.is_none() { @@ -532,7 +534,8 @@ pub(crate) fn plan_entries<'py>( warnings.push(warning?); } } - if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries)? { + if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries, concurrency)? + { warnings.push(warning); } Ok(ModelEntryResults { entries, warnings }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index 6692769335..52e10ea560 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -29,6 +29,8 @@ pub(crate) struct PlanEntriesRequest<'py> { pub(crate) cursor_overrides: Bound<'py, PyAny>, pub(crate) full_refresh: bool, pub(crate) build_inputs: Bound<'py, PyAny>, + /// The effective worker limit execution will use. + pub(crate) concurrency: i64, } /// What assembling the plan output's fields reads. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 8adedd54f4..dae677b9a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,5 +1,6 @@ //! Write-hash reconciliation and plan entry construction. +use crate::bindings::_helpers::command_runtime::concurrency::resolve_configured_concurrency; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; @@ -65,6 +66,26 @@ fn config_or<'py>( Ok(value) } +/// The concurrency execution will use; without configs, the request or the compiled setting. +fn planned_concurrency( + requested: &Bound<'_, PyAny>, + (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + project: &Bound<'_, PyAny>, +) -> PyResult { + if project_config.is_none() || local_config.is_none() { + if !requested.is_none() { + return requested.extract(); + } + return native_field(&native_field(project, "settings")?, "concurrency")?.extract(); + } + Ok(resolve_configured_concurrency( + requested, + (project_config, local_config), + &native_field(project, "effective_target_name")?, + )? + .limit) +} + /// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. pub(crate) fn build_planner_entry_results<'py>( (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -133,6 +154,14 @@ pub(crate) fn build_planner_entry_results<'py>( }, )? .into_any(); + let concurrency: i64 = planned_concurrency( + &native_field(overrides, "concurrency")?, + ( + &config, + &runtime_field(runtime, |runtime| &runtime.local_config)?, + ), + &project, + )?; plan_entries(PlanEntriesRequest { project, adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, @@ -143,5 +172,6 @@ pub(crate) fn build_planner_entry_results<'py>( cursor_overrides: native_field(overrides, "cursor_overrides")?, full_refresh: native_field(overrides, "full_refresh")?.is_truthy()?, build_inputs, + concurrency, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 523d1e4eb2..d201e7b24d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -86,6 +86,7 @@ const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("concurrency", FieldDefault::None), ], }; const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index 183172518e..49dfea7d02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -12,7 +12,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; use crate::bindings::_helpers::records::fields::field as native_field; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { @@ -47,7 +49,7 @@ pub(crate) fn run_dag_command<'py>( project_dir, discovered, adapter, - } = project_adapter(py, project_dir)?; + } = project_adapter(py, project_dir, ProjectInputs::Discovered)?; let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver(project_dir, discovered.clone())?; let graph: Bound<'py, PyAny> = build_project_graph( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index eeaaa42ae5..9973b9619a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::discovery::configuration::discover_connection_configuration; use crate::bindings::_helpers::records::fields::field as native_field; /// The project directory, defaulting to the working directory, with its inputs and adapter. @@ -16,18 +17,31 @@ pub(crate) struct ProjectAdapter<'py> { pub(crate) adapter: Bound<'py, PyAny>, } +/// How much of the project a command reads before it resolves the adapter. +#[derive(Clone, Copy)] +pub(crate) enum ProjectInputs { + /// Every declaration file and the imported project Python. + Discovered, + /// Only the project and local configuration, with its connection settings validated. + Configuration, +} + pub(crate) fn project_adapter<'py>( py: Python<'py>, project_dir: Option>, + inputs: ProjectInputs, ) -> PyResult> { let project_dir: Bound<'py, PyAny> = match project_dir.filter(|path| !path.is_none()) { Some(path) => path, None => imported(py, "pathlib", "Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = discover_project_inputs( - py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), - )?; + let discovered: Bound<'py, PyAny> = match inputs { + ProjectInputs::Discovered => discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), + )?, + ProjectInputs::Configuration => discover_connection_configuration(py, &project_dir)?, + }; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, &native_field(&discovered, "local_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index 0adfe98ad0..4d3f947a2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -19,7 +19,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; use crate::bindings::_helpers::query_command::query_values::{ csv_text, display_text, json_cell, python_json_rows, strict_width, }; @@ -323,7 +325,11 @@ pub(crate) fn run_query_command( Some("pass a non-negative integer or use --no-limit"), )); } - let project: ProjectAdapter<'_> = project_adapter(py, Some(arguments.value("project_dir")?))?; + let project: ProjectAdapter<'_> = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; let result: Bound<'_, PyAny> = run_sql( &project, Some(arguments.value("selected_target")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 114976c1d5..a33d548588 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -9,6 +9,8 @@ pub(crate) const NOTICE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; pub(crate) const HOST_TOKEN_LENGTH: usize = 16; /// The shortest heartbeat interval the test environment variable may request. pub(crate) const MINIMUM_HEARTBEAT_SECONDS: f64 = 0.01; +/// What a Recall closing notice says is the only thing affected. +pub(crate) const RECALL_HISTORY: &str = "Recall history"; /// The compute-log stream index of stdout; every other output index is stderr. pub(crate) const STDOUT_STREAM_INDEX: usize = 0; pub(crate) const RECALL_DISABLED_CODE: &str = "C970"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 36b96eb21f..013617a07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cli::dispatch::main::closing_status::closing_status; use sqlbuild_cli::dispatch::models::CommandFacts; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; @@ -33,7 +34,7 @@ use crate::bindings::_helpers::recall::clickhouse_store::{ use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{ CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, - MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, + MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, RECALL_HISTORY, STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::recall::events::escaped_text; @@ -382,17 +383,6 @@ pub(crate) fn start_invocation_recall( }) } -/// The closing sentence after a Recall notice, so the command's own outcome stays the last word. -fn final_status(command: &str, exit_code: Option) -> String { - match exit_code { - Some(0) => { - format!("sqb {command} completed successfully; only its Recall history is affected.\n") - } - Some(code) => format!("sqb {command} failed with exit code {code}.\n"), - None => format!("sqb {command} failed.\n"), - } -} - /// What Recall must tell the user about this run once delivery has finished. fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec { let mut notices: Vec = session @@ -447,7 +437,7 @@ pub(crate) fn finish_invocation_recall( if !notices.is_empty() { notify( py, - Some(notices.concat() + &final_status(&recall.command, exit_code)), + Some(notices.concat() + &closing_status(&recall.command, exit_code, RECALL_HISTORY)), ); } Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 39f8e07f68..da4f43df58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -352,6 +352,7 @@ const DBT_SQLBUILD_WORK_CONTEXT: RecordSpec = RecordSpec { FieldSpec::required("use_color"), FieldSpec::required("snapshots_config"), FieldSpec::required("execution_limits"), + FieldSpec::required("concurrency"), ], }; const PLAN_SAFETY_GATES: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 94767d13ea..212db5027c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -19,6 +19,8 @@ const R_RULE_EXEMPTION: RecordSpec = RecordSpec { FieldSpec::required("rule"), FieldSpec::required("path"), FieldSpec::required("reason"), + FieldSpec::new("relation", FieldDefault::None), + FieldSpec::new("cte", FieldDefault::None), ], }; const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { @@ -266,7 +268,6 @@ const R_RULES_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("rule_exceptions", FieldDefault::EmptyTuple), FieldSpec::new("graph_edge_exceptions", FieldDefault::EmptyTuple), FieldSpec::new("rule_ignores", FieldDefault::EmptyTuple), - FieldSpec::new("allow_exceptions", FieldDefault::True), FieldSpec::new("select_star_allow", FieldDefault::EmptyTuple), FieldSpec::new("domains", FieldDefault::EmptyTuple), FieldSpec::new("approved_source_tokens", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index 2b27e8f645..bca5bbdfd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -33,7 +33,25 @@ fn option<'py>( let py: Python<'py> = option_class.py(); let (factory, default): (&str, Bound<'py, PyAny>) = match declared.kind { BuiltinOptionKind::Boolean => ("boolean", PyBool::new(py, false).to_owned().into_any()), - BuiltinOptionKind::StringList => ("string_list", PyTuple::empty(py).into_any()), + BuiltinOptionKind::StringList(defaults) => { + ("string_list", PyTuple::new(py, defaults)?.into_any()) + } + BuiltinOptionKind::TableList => { + let arguments = keywords( + py, + vec![ + ("name", PyString::new(py, declared.name).into_any()), + ("value_type", py.get_type::().into_any()), + ("default", PyTuple::empty(py).into_any()), + ( + "description", + PyString::new(py, declared.description).into_any(), + ), + ("item_type", py.get_type::().into_any()), + ], + )?; + return new_record(py, RecordId::RuleOption, (), Some(&arguments)); + } }; let arguments = keywords( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index f1bf047ba8..61fad4cdb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -34,7 +34,7 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; use crate::bindings::_helpers::rules::relation_keys::{ - RelationKeyRow, RelationKeys, relation_keys, source_key_input, + RelationKeyRow, RelationKeys, one_row_seed_names, relation_keys, source_key_input, }; use crate::bindings::_helpers::rules::rule_cache::stored_entries; use crate::bindings::_helpers::rules::rules_config::rules_config_of; @@ -57,7 +57,7 @@ pub(crate) fn sql_lint_settings( dialect: &str, codes: &[String], threshold: impl Fn(&str) -> PyResult>, - (relation_keys, allow_suppressions): (RelationKeys, bool), + relation_keys: RelationKeys, ) -> PyResult { let defaults: LintSettings = LintSettings::default(); Ok(LintSettings { @@ -71,7 +71,6 @@ pub(crate) fn sql_lint_settings( .unwrap_or(DEFAULT_MAX_RANKING_ORDER_BY), ..defaults.sql }, - allow_suppressions, relation_keys, ..defaults }) @@ -116,9 +115,13 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> = record_field(inputs, "seed_inputs")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&seed_inputs)?; let mut seeds: Vec> = Vec::new(); - for seed in record_field(inputs, "seed_inputs")?.try_iter()? { - seeds.push(record_field(&seed?, "schema_entry")?); + for seed in &seed_inputs { + seeds.push(record_field(seed, "schema_entry")?); } let mut sources: Vec> = Vec::new(); for source in record_field(inputs, "source_inputs")?.try_iter()? { @@ -131,12 +134,13 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult>)>, } @@ -89,7 +89,6 @@ impl Default for LintSettings { max_literal_length: 100, max_ranking_order_by: 5, }, - allow_suppressions: true, relation_keys: Vec::new(), } } @@ -119,7 +118,6 @@ impl LintSettings { max_literal_length: record_field(config, "max_literal_length")?.extract()?, max_ranking_order_by: record_field(config, "max_ranking_order_by")?.extract()?, }, - allow_suppressions: record_field(config, "allow_suppressions")?.is_truthy()?, relation_keys, }) } @@ -643,7 +641,7 @@ pub(crate) fn lint( check_stop(py, request.stop)?; } let violations: Vec = - suppressed(&files, rows, request.settings.allow_suppressions)?; + with_directive_findings(&files, request.project_dir, rows)?; Ok(LintOutcome { files, violations, @@ -1084,10 +1082,34 @@ fn lint_body( allows_dynamic_output_star: traits.dynamic_star, allows_empty_fixture_star: traits.ceremonial, relation_keys: Vec::new(), + relation_names: relation_names(&dependencies.relations), }; Ok((body, dependencies.relations)) } +/// `path` relative to `project_dir` with `/` separators, or as given outside the project. +pub(crate) fn project_relative_path(path: &Path, project_dir: &Path) -> String { + let relative: &Path = path.strip_prefix(project_dir).unwrap_or(path); + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") +} + +/// The relation name each dependency sentinel stands for, without its `kind:` prefix. +pub(crate) fn relation_names(relations: &[(String, String)]) -> Vec<(String, String)> { + relations + .iter() + .map(|(sentinel, identity)| { + let name: &str = identity + .split_once(':') + .map_or(identity.as_str(), |(_, name)| name); + (sentinel.clone(), name.to_owned()) + }) + .collect() +} + /// The declared keys of each dependency relation that declares any, by its sentinel. pub(crate) fn relation_keys( settings: &LintSettings, @@ -1119,33 +1141,31 @@ fn sentinel_spans(sites: &[InterpolationSite]) -> Vec { .collect() } -/// `rows` without suppressed violations, with suppression findings, sorted by path then place. -fn suppressed( +/// `rows` with file paths in their help and a finding per inline directive, sorted by place. +fn with_directive_findings( files: &[LintFile], - rows: Vec, - allow_suppressions: bool, + project_dir: &Path, + mut violations: Vec, ) -> PyResult> { - let keys: Vec = rows + let paths: Vec = files .iter() - .map(|row| SuppressibleViolation { - file: row.file, - line: usize::try_from(row.line).unwrap_or(usize::MAX), - code: row.code.clone(), - native: row.engine == NATIVE_ENGINE, - }) + .map(|file| project_relative_path(&file.path, project_dir)) .collect(); - let texts: Vec<&str> = files.iter().map(|file| file.contents.as_str()).collect(); - let retained: Vec = catch_compiler_panic(|| { - apply_suppressions(&texts, &keys, allow_suppressions).map_err(|error| error.to_string()) + let sources: Vec<(&str, &str)> = paths + .iter() + .zip(files) + .map(|(path, file)| (path.as_str(), file.contents.as_str())) + .collect(); + let findings: Vec = catch_compiler_panic(|| { + reject_inline_suppressions(&sources).map_err(|error| error.to_string()) }) .map_err(value_error)?; - let mut violations: Vec = retained - .into_iter() - .map(|item| match item { - RetainedViolation::Existing(index) => rows[index].clone(), - RetainedViolation::Added(finding) => LintViolationRow::suppression(finding), - }) - .collect(); + for violation in &mut violations { + if let Some(remediation) = &mut violation.remediation { + *remediation = fill_finding_path(remediation, &paths[violation.file]); + } + } + violations.extend(findings.into_iter().map(LintViolationRow::suppression)); violations.sort_by(|left, right| { ( files[left.file].path.to_string_lossy(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs index 1ee6aae410..599e08ee1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -11,7 +11,9 @@ use sqlbuild_rules::lint_text::models::{ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; -use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + LintViolationRow, ViolationClasses, relation_names as sentinel_relation_names, +}; use crate::bindings::types::CompilerDetach; fn spans(passes: &Bound<'_, PyAny>) -> PyResult>> { @@ -48,6 +50,15 @@ fn relation_keys( Ok(keys) } +/// The relation name each dependency sentinel of one body stands for. +fn relation_names(body: &Bound<'_, PyAny>) -> PyResult> { + let mut relations: Vec<(String, String)> = Vec::new(); + for relation in native_field(body, "dependency_relations")?.try_iter()? { + relations.push(relation?.extract()?); + } + Ok(sentinel_relation_names(&relations)) +} + fn lint_body( body: &Bound<'_, PyAny>, index_by_path: &Bound<'_, PyDict>, @@ -69,6 +80,7 @@ fn lint_body( allows_dynamic_output_star: native_field(body, "allows_dynamic_output_star")?.extract()?, allows_empty_fixture_star: native_field(body, "allows_empty_fixture_star")?.extract()?, relation_keys: relation_keys(body, declared)?, + relation_names: relation_names(body)?, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 9f6a9d83b6..8f447e874b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -1,4 +1,4 @@ -//! Interpolation sentinels, suppressions and fixture-null candidates over Python lint models. +//! Interpolation sentinels, inline directive findings and fixture-null candidates over lint models. use std::collections::BTreeSet; use std::path::PathBuf; @@ -7,18 +7,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::errors::LintTextError; -use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; use sqlbuild_rules::lint_text::main::interpolation_text_at::interpolation_text_at; use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind as role_kind; use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use sqlbuild_rules::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolation; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use sqlbuild_rules::lint_text::models::{ - FormatFile, FormattedBodies, InterpolationSite, Neutralized, RetainedViolation, - SuppressibleViolation, + FormatFile, FormattedBodies, InterpolationSite, Neutralized, SuppressionFinding, }; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; @@ -28,7 +27,7 @@ use crate::bindings::_helpers::records::construct::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::lint_pipeline::{ - LintViolationRow, NATIVE_ENGINE, ViolationClasses, + LintViolationRow, NATIVE_ENGINE, ViolationClasses, project_relative_path, }; use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; @@ -121,58 +120,39 @@ fn named_declaration_role_kind<'py>( } } -/// Remove violations matched by suppressions and add the suppression findings, as lint models. +/// `violations` plus a rejection finding for every inline suppression directive, as lint models. #[pyfunction] -pub(crate) fn apply_lint_suppressions<'py>( +pub(crate) fn reject_lint_suppressions<'py>( py: Python<'py>, violations: Vec>, contents_by_path: Bound<'py, PyDict>, - allow_suppressions: bool, + project_dir: PathBuf, ) -> PyResult>> { - let index_by_path: Bound<'py, PyDict> = PyDict::new(py); let mut paths: Vec> = Vec::new(); + let mut relative: Vec = Vec::new(); let mut contents: Vec = Vec::new(); for (path, text) in contents_by_path.iter() { - index_by_path.set_item(&path, paths.len())?; + relative.push(project_relative_path( + &path.extract::()?, + &project_dir, + )); paths.push(path); contents.push(text.extract()?); } - let keys: Vec = violations + let sources: Vec<(&str, &str)> = relative .iter() - .map(|violation| suppressible(violation, &index_by_path)) - .collect::>()?; - let texts: Vec<&str> = contents.iter().map(String::as_str).collect(); - let retained: Vec = compiler_guard(|| { - apply_suppressions(&texts, &keys, allow_suppressions).map_err(value_error) - })?; + .zip(&contents) + .map(|(path, text)| (path.as_str(), text.as_str())) + .collect(); + let findings: Vec = + compiler_guard(|| reject_inline_suppressions(&sources).map_err(value_error))?; let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; - retained - .into_iter() - .map(|item| match item { - RetainedViolation::Existing(index) => Ok(violations[index].clone()), - RetainedViolation::Added(finding) => { - let row: LintViolationRow = LintViolationRow::suppression(finding); - classes.violation(&paths[row.file], &row) - } - }) - .collect() -} - -/// The suppression identity of one Python lint violation. -fn suppressible( - violation: &Bound<'_, PyAny>, - index_by_path: &Bound<'_, PyDict>, -) -> PyResult { - let file: Option = match index_by_path.get_item(native_field(violation, "file_path")?)? { - Some(index) => Some(index.extract()?), - None => None, - }; - Ok(SuppressibleViolation { - file: file.unwrap_or(usize::MAX), - line: native_field(violation, "line")?.extract()?, - code: native_field(violation, "code")?.extract()?, - native: native_field(violation, "engine")?.extract::()? == NATIVE_ENGINE, - }) + let mut retained: Vec> = violations; + for finding in findings { + let row: LintViolationRow = LintViolationRow::suppression(finding); + retained.push(classes.violation(&paths[row.file], &row)?); + } + Ok(retained) } /// The parts of `path` below `project_dir`, or `None` outside the project. @@ -265,5 +245,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { )?)?; module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; module.add_function(wrap_pyfunction!(named_declaration_role_kind, module)?)?; - module.add_function(wrap_pyfunction!(apply_lint_suppressions, module)?) + module.add_function(wrap_pyfunction!(reject_lint_suppressions, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 406440f71f..70f684af2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -1,16 +1,20 @@ //! Declared unique keys of project relations, derived natively from their key facts. use std::collections::BTreeMap; +use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; +use sqlbuild_discovery::project_validation::models::CsvSettings; use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::compiled_project::source_views::{SourceKeyFacts, read_source_view}; +use crate::bindings::_helpers::discovery::project_facts::seed_csv_settings; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::lint_inputs::model_key_row; @@ -187,11 +191,36 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult>)>; -/// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. +/// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. +pub(crate) fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { + let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); + for seed in seeds { + let entry: Bound<'_, PyAny> = native_field(seed, "schema_entry")?; + candidates.push(( + native_field(&entry, "name")?.extract()?, + native_field(&native_field(seed, "seed_file")?, "file_path")?.extract()?, + seed_csv_settings(&entry)?, + )); + } + let py: Python<'_> = match seeds.first() { + Some(seed) => seed.py(), + None => return Ok(Vec::new()), + }; + Ok(py.detach(|| { + candidates + .into_iter() + .filter(|(_, path, settings)| at_most_one_csv_data_row(path, settings)) + .map(|(name, _, _)| name) + .collect() + })) +} + +/// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. pub(crate) fn relation_keys<'py>( py: Python<'py>, mut rows: Vec, (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + one_row_seeds: &[String], ) -> PyResult { for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { for entry in entries.try_iter()? { @@ -209,7 +238,11 @@ pub(crate) fn relation_keys<'py>( )?); } } - let relations: Vec = rows.into_iter().map(declared_relation).collect(); + let mut relations: Vec = rows.into_iter().map(declared_relation).collect(); + for relation in &mut relations { + relation.at_most_one_row = + relation.kind == RELATION_KIND_SEED && one_row_seeds.contains(&relation.name); + } let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); Ok(keys.into_iter().collect()) } @@ -236,7 +269,7 @@ fn declared_relation_keys<'py>( seeds: &Bound<'py, PyAny>, sources: &Bound<'py, PyAny>, ) -> PyResult> { - keys_dict(py, relation_keys(py, model_rows, (seeds, sources))?) + keys_dict(py, relation_keys(py, model_rows, (seeds, sources), &[])?) } /// Declared keys of a compiled project's models, seeds and sources. @@ -246,9 +279,13 @@ pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult> = native_field(project, "seeds")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&compiled_seeds)?; let mut seeds: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { - seeds.push(native_field(&seed?, "schema_entry")?); + for seed in &compiled_seeds { + seeds.push(native_field(seed, "schema_entry")?); } let mut sources: Vec> = Vec::new(); for source in native_field(project, "sources")?.try_iter()? { @@ -261,6 +298,7 @@ pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult DeclaredRelation { }) .collect(), audits, + at_most_one_row: false, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index f9e58d2580..19b848594c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -41,7 +41,6 @@ pub(crate) fn settings_lint_key(settings: &LintSettings) -> String { max_literal_length: settings.sql.max_literal_length, max_ranking_order_by: settings.sql.max_ranking_order_by, relation_keys: settings.relation_keys.clone(), - allow_suppressions: settings.allow_suppressions, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 83d3585991..65e094a21f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, Py, pyfunction, wrap_pyfunction}; use serde_json::Value; -use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::errors::RulesConfigLoadError; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; @@ -137,6 +137,8 @@ fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult(py: Python<'py>, config: &RulesConfig) -> PyResult, project_dir: PathBuf) -> PyResul pub(crate) enum InvalidRulesConfig { /// The configuration could not be read: a `RulesError`. Unreadable(String), - /// It allows exceptions it may not: a `RulesConfigError` with help. + /// It uses a setting SQLBuild rejects: a `RulesConfigError` with help. Forbidden { message: String, help: String }, } @@ -275,17 +273,16 @@ pub(crate) fn rules_config_of( let project_dir: PathBuf = project_dir .canonicalize() .or_else(|_| std::path::absolute(project_dir))?; - let config: RulesConfig = match compiler_guard(|| Ok(load_config(&project_dir)))? { - Ok(config) => config, - Err(message) => return Ok(Err(InvalidRulesConfig::Unreadable(message))), - }; - if let Some(error) = forbidden_exceptions(&config) { - return Ok(Err(InvalidRulesConfig::Forbidden { + match compiler_guard(|| Ok(load_config(&project_dir)))? { + Ok(config) => Ok(Ok(config)), + Err(RulesConfigLoadError::Invalid(message)) => { + Ok(Err(InvalidRulesConfig::Unreadable(message))) + } + Err(RulesConfigLoadError::Policy(error)) => Ok(Err(InvalidRulesConfig::Forbidden { message: error.message, help: error.help, - })); + })), } - Ok(Ok(config)) } /// The project's rules configuration, or `None` when it is invalid and the rules phase reports it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 6330b98eac..66ea5cdef2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -126,10 +126,7 @@ pub(crate) fn run_planned_sql_rules<'py>( Some(value) => value.extract(), None => Ok(None), }, - ( - project_relation_keys(&field(&graph, "project")?)?, - field(&effective_config, "allow_exceptions")?.is_truthy()?, - ), + project_relation_keys(&field(&graph, "project")?)?, )?; let early_lint: Bound<'py, PyAny> = options.get("early_lint")?; let run = SqlRun { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index adadca3f31..8ee77168f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -13,7 +13,7 @@ use sqlbuild_core::scenario_command::models::{ ScenarioNamespace, ScenarioTestFlags, SqlAnalysisCommand, }; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -98,16 +98,12 @@ fn prepare<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = request.getattr("concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &build_effective_settings_config(&project.discovered.clone())?, - "concurrency", - )? - .extract()? - } else { - requested.extract()? - }; + let concurrency: i64 = resolve_concurrency( + &request.getattr("concurrency")?, + &project.discovered, + &py.None().into_bound(py), + )? + .limit; let sys: Bound<'py, PyModule> = py.import("sys")?; let stream: Bound<'py, PyAny> = sys.getattr(if flag(request, "json_output")? { "stderr" diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index f43a61d2c9..80fc6e43fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -17,6 +17,7 @@ use sqlbuild_core::seed_command::models::SeedRow; use crate::bindings::_helpers::cli::progress::current_native_progress_projector; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_text::write_execution_header; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; @@ -437,16 +438,12 @@ pub(crate) fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py let invocation: Bound<'_, PyAny> = resolve_command_invocation(request, &record_class(py, RecordId::SeedInvocation)?)?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; - let requested: Bound<'_, PyAny> = native_field(request, "concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &native_field(&native_field(&pipeline, "project")?, "settings")?, - "concurrency", - )? - .extract::()? - } else { - requested.extract::()? - } + let concurrency: i64 = resolve_concurrency( + &native_field(request, "concurrency")?, + &native_field(&invocation, "discovered_inputs")?, + &native_field(request, "selected_target")?, + )? + .limit .max(1); write_header(&invocation, &pipeline, concurrency)?; let started: Instant = Instant::now(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 67b23cf31d..f0d0586175 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReport use crate::bindings::_helpers::cli::progress_text::{ format_expectation_detail, format_expectation_name, write_execution_header, }; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::python_calls::keywords; @@ -674,16 +675,12 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py if inspect_only { return inspect(&stream, &pipeline.getattr("plan_output")?); } - let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &native_field(&native_field(&pipeline, "project")?, "settings")?, - "concurrency", - )? - .extract()? - } else { - requested.extract()? - }; + let concurrency: i64 = resolve_concurrency( + &request.getattr("concurrency")?, + &native_field(&invocation, "discovered_inputs")?, + &request.getattr("selected_target")?, + )? + .limit; stream.call_method1("write", ("\n",))?; let header: Bound<'_, PyDict> = PyDict::new(py); header.set_item("stream", &stream)?; diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index ed6d6fa9af..f580959741 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,37 +1,48 @@ -use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; +use crate::configuration::_helpers::removed_settings::removed_settings; +use crate::configuration::errors::RulesConfigLoadError; +use crate::constants::{ + ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, + PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, RELATION_EXCEPTION_RULES, +}; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use std::path::Path; -pub(crate) fn load(project_dir: &Path) -> Result { +pub(crate) fn load(project_dir: &Path) -> Result { + let invalid = RulesConfigLoadError::Invalid; let path = project_dir.join(PROJECT_CONFIG_FILENAME); let source = std::fs::read_to_string(&path).map_err(|error| { - format!( + invalid(format!( "could not load rules config from {}: {error}", path.display() - ) + )) })?; let root: toml::Table = toml::from_str(&source).map_err(|error| { - format!( + invalid(format!( "could not load rules config from {}: {error}", path.display() - ) + )) })?; if root.contains_key("policy") || root.contains_key("lint") || root.contains_key("kata") { - return Err("unsupported legacy rules configuration; use [rules]".to_owned()); + return Err(invalid( + "unsupported legacy rules configuration; use [rules]".to_owned(), + )); } let config = match root.get("rules") { None => RulesConfig::default(), Some(value) => { - validate_raw(value)?; + if let Some(error) = value.as_table().and_then(removed_settings) { + return Err(RulesConfigLoadError::Policy(error)); + } + validate_raw(value).map_err(invalid)?; value .clone() .try_into::() - .map_err(|error| format!("invalid rules config: {error}"))? + .map_err(|error| invalid(format!("invalid rules config: {error}")))? } }; - validate(&config)?; + validate(&config).map_err(invalid)?; Ok(config) } @@ -48,7 +59,6 @@ fn validate_raw(value: &toml::Value) -> Result<(), String> { "rule_exceptions", "graph_edge_exceptions", "rule_ignores", - "allow_exceptions", "select_star_allow", "domains", "approved_source_tokens", @@ -199,6 +209,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { return Err(format!("invalid rule code: {code}")); } } + validate_published_layer_options(config)?; for exception in &config.rule_exceptions { if exception.rule.trim().is_empty() || exception.path.trim().is_empty() @@ -209,6 +220,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { if !grammar.rule_code_is_exact(&exception.rule) { return Err(format!("invalid rule code: {}", exception.rule)); } + validate_relation_scope(exception)?; } let mut graph_edges: std::collections::BTreeSet<(&str, &str)> = std::collections::BTreeSet::new(); @@ -259,6 +271,108 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { Ok(()) } +/// Published layers name model layers; allowed qualifiers are literal and carry a reason. +fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> { + for code in [PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE] { + let Some(layers) = config + .rule_options + .get(code) + .and_then(|options| options.get(PUBLISHED_LAYERS_OPTION)) + else { + continue; + }; + let label: String = format!("rules.rule_options.{code}.{PUBLISHED_LAYERS_OPTION}"); + let Some(layers) = layers.as_array() else { + return Err(format!("{label} must be a list of model name layers")); + }; + for layer in layers { + match layer.as_str() { + Some(name) if MODEL_NAME_LAYERS.contains(&name) => {} + _ => { + return Err(format!( + "{label} entries must be model name layers: {}", + MODEL_NAME_LAYERS.join(", ") + )); + } + } + } + } + let Some(entries) = config + .rule_options + .get(PUBLISHED_SHAPE_CODE) + .and_then(|options| options.get(ALLOWED_QUALIFIERS_OPTION)) + else { + return Ok(()); + }; + let label: String = + format!("rules.rule_options.{PUBLISHED_SHAPE_CODE}.{ALLOWED_QUALIFIERS_OPTION}"); + let shape: String = format!( + "{label} entries must be {{ qualifier = \"...\", reason = \"...\" }} tables with an \ + optional domain" + ); + let Some(entries) = entries.as_array() else { + return Err(shape); + }; + let mut seen: std::collections::BTreeSet<(Option<&str>, &str)> = + std::collections::BTreeSet::new(); + for entry in entries { + let Some(table) = entry.as_object() else { + return Err(shape); + }; + if table + .keys() + .any(|key| !["domain", "qualifier", "reason"].contains(&key.as_str())) + { + return Err(shape); + } + let text = |key: &str| table.get(key).map(|value| value.as_str().map(str::trim)); + let (Some(Some(qualifier)), Some(Some(reason))) = (text("qualifier"), text("reason")) + else { + return Err(shape); + }; + let domain: Option<&str> = match text("domain") { + None => None, + Some(Some(domain)) if !domain.is_empty() => Some(domain), + Some(_) => return Err(shape), + }; + if qualifier.is_empty() || reason.is_empty() { + return Err(format!( + "{label} entries need a non-empty qualifier and reason" + )); + } + if !seen.insert((domain, qualifier)) { + return Err(format!("duplicate {label} entry: {qualifier}")); + } + } + Ok(()) +} + +/// A relation-level exception names a non-empty relation, for a Rule that reports one. +fn validate_relation_scope(exception: &crate::models::RuleException) -> Result<(), String> { + let blank = |value: &Option| value.as_ref().is_some_and(|text| text.trim().is_empty()); + if blank(&exception.relation) || blank(&exception.cte) { + return Err("rules.rule_exceptions relation and cte must be non-empty strings".into()); + } + if exception.cte.is_some() && exception.relation.is_none() { + return Err(format!( + "rules.rule_exceptions entry for {} at {} sets cte without relation; cte narrows a \ + relation-level exception", + exception.rule, exception.path + )); + } + if exception.relation.is_some() && !RELATION_EXCEPTION_RULES.contains(&exception.rule.as_str()) + { + return Err(format!( + "rules.rule_exceptions entry for {} at {} sets relation, but only {} report a joined \ + relation; remove relation to cover the whole file", + exception.rule, + exception.path, + RELATION_EXCEPTION_RULES.join(", ") + )); + } + Ok(()) +} + fn validate_relative_path(value: &str, label: &str) -> Result<(), String> { let path = Path::new(value); if value.trim().is_empty() diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs b/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs index f33faa4b7b..c96e79b252 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs @@ -1 +1,2 @@ pub mod loading; +pub mod removed_settings; diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs b/crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs new file mode 100644 index 0000000000..cc2c9406d1 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs @@ -0,0 +1,26 @@ +//! `[rules]` settings SQLBuild no longer supports, with how to migrate. + +use sqlbuild_analysis::contracts::main::join_helps::join_helps; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; + +use crate::configuration::errors::ConfigPolicyError; +use crate::constants::REMOVED_ALLOW_EXCEPTIONS; + +/// The error for a `[rules]` table that still sets a removed setting. +pub(crate) fn removed_settings(rules: &toml::Table) -> Option { + if !rules.contains_key(REMOVED_ALLOW_EXCEPTIONS) { + return None; + } + let message: String = format!( + "{PROJECT_CONFIG_FILE} sets [rules] {REMOVED_ALLOW_EXCEPTIONS}, which SQLBuild no longer \ + supports: Rule exceptions are declared only in {PROJECT_CONFIG_FILE}, where review sees \ + them, and inline -- sqb: ignore suppressions were removed" + ); + let help: String = join_helps(&[ + format!("remove the {REMOVED_ALLOW_EXCEPTIONS} line from [rules] in {PROJECT_CONFIG_FILE}"), + "keep each reviewed deviation as a [[rules.rule_exceptions]] entry with rule, path and \ + reason, or turn a Rule off for the whole project with [rules] ignore" + .to_owned(), + ]); + Some(ConfigPolicyError { message, help }) +} diff --git a/crates/sqlbuild-rules/src/configuration/errors.rs b/crates/sqlbuild-rules/src/configuration/errors.rs index cd92448b71..14cdec8370 100644 --- a/crates/sqlbuild-rules/src/configuration/errors.rs +++ b/crates/sqlbuild-rules/src/configuration/errors.rs @@ -6,3 +6,12 @@ pub struct ConfigPolicyError { pub message: String, pub help: String, } + +/// Why a project's rules configuration cannot be loaded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RulesConfigLoadError { + /// The configuration is unreadable or malformed. + Invalid(String), + /// The configuration uses a setting SQLBuild rejects, with help showing the fix. + Policy(ConfigPolicyError), +} diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs deleted file mode 100644 index 690d165255..0000000000 --- a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! `forbidden_exceptions`: exceptions or scoped ignores in a project that disallows them. - -use sqlbuild_analysis::contracts::main::join_helps::join_helps; -use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; - -use crate::configuration::errors::ConfigPolicyError; -use crate::configuration::main::setting_help::setting_help; -use crate::models::RulesConfig; -use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; - -/// The policy error when `allow_exceptions = false` and exception or ignore entries exist. -#[must_use] -pub fn forbidden_exceptions(config: &RulesConfig) -> Option { - if config.allow_exceptions { - return None; - } - let mut entries: Vec = Vec::new(); - let mut tables: Vec = Vec::new(); - for (key, count) in [ - ("rule_exceptions", config.rule_exceptions.len()), - ("rule_ignores", config.rule_ignores.len()), - ] { - if count == 0 { - continue; - } - let table: String = format!("[[rules.{key}]]"); - let count: i64 = i64::try_from(count).unwrap_or(i64::MAX); - entries.push(format_count_noun( - count, - &format!("{table} entry"), - Some(&format!("{table} entries")), - )); - tables.push(table); - } - if entries.is_empty() { - return None; - } - let message: String = format!( - "{PROJECT_CONFIG_FILE} has {}, which this project does not allow; \ - {PROJECT_CONFIG_FILE} sets [rules] allow_exceptions = false", - entries.join(" and ") - ); - let help: String = join_helps(&[ - format!( - "remove every {} entry and fix the findings it hides; to turn a Rule off for the \ - whole project, add its code to [rules] ignore", - tables.join(" and ") - ), - setting_help( - "to allow Rule exceptions, scoped ignores and inline suppressions", - "rules", - "allow_exceptions", - "true", - ), - ]); - Some(ConfigPolicyError { message, help }) -} diff --git a/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs index 2c3ba017d0..b65f6aa60e 100644 --- a/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs +++ b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs @@ -2,9 +2,10 @@ use std::path::Path; +use crate::configuration::errors::RulesConfigLoadError; use crate::models::RulesConfig; /// The project's rules configuration, or the reason it cannot be loaded. -pub fn load_rules_config(project_dir: &Path) -> Result { +pub fn load_rules_config(project_dir: &Path) -> Result { crate::configuration::_helpers::loading::load(project_dir) } diff --git a/crates/sqlbuild-rules/src/configuration/main/mod.rs b/crates/sqlbuild-rules/src/configuration/main/mod.rs index 42488f2f4f..732d7c95f5 100644 --- a/crates/sqlbuild-rules/src/configuration/main/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/main/mod.rs @@ -1,4 +1,3 @@ -pub mod forbidden_exceptions; pub mod load_rules_config; pub mod setting_help; pub mod toml_string_array; diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index 73e3aa809f..ad865fb4e1 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -33,3 +33,27 @@ pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v2 pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; /// Threshold key setting the fewest test cases a custom rule must ship. pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; +/// Rules whose findings name a joined relation, so an exception may cover just that relation. +pub(crate) const RELATION_EXCEPTION_RULES: [&str; 1] = ["SQBRSQL038"]; +/// The layers the `____` model name grammar accepts. +pub(crate) const MODEL_NAME_LAYERS: [&str; 7] = [ + "stg", + "stg_v", + "int_clean", + "int_v", + "int_enriched", + "mart", + "mart_v", +]; +/// The removed exception switch, which used to allow or forbid every escape hatch at once. +pub(crate) const REMOVED_ALLOW_EXCEPTIONS: &str = "allow_exceptions"; +/// Published models must not read published models of their own domain. +pub(crate) const PUBLISHED_READ_CODE: &str = "SQBRGRAPH103"; +/// Each domain's published layer uses one model name shape. +pub(crate) const PUBLISHED_SHAPE_CODE: &str = "SQBRPROJECT107"; +/// The option naming the published layers. +pub(crate) const PUBLISHED_LAYERS_OPTION: &str = "published_layers"; +/// The option listing qualifiers that do not count towards a domain's name shape. +pub(crate) const ALLOWED_QUALIFIERS_OPTION: &str = "allowed_qualifiers"; +/// The published layers when a project does not configure them. +pub(crate) const DEFAULT_PUBLISHED_LAYERS: [&str; 2] = ["mart", "mart_v"]; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index f0f4e3c2ed..494165ea40 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -1,9 +1,9 @@ use crate::configuration::main::validate as config; -use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY, RELATION_EXCEPTION_RULES}; use crate::engine::_helpers::cache::{Cache, PendingBucketWrite, RuleCacheEntry}; use crate::models::{ - EvaluateRequest, EvaluateResponse, Fault, Model, RuleMetadata, RuleScope, RulesCodeGrammar, - RulesConfig, + EvaluateRequest, EvaluateResponse, Fault, Model, RuleException, RuleMetadata, RuleScope, + RulesCodeGrammar, RulesConfig, }; use crate::rules::main::{ assemble_catalogue, evaluate as rules, evaluate_project, fingerprint, @@ -12,6 +12,7 @@ use crate::rules::main::{ use crate::rules::models::{ ModelEvaluationRequest, ProjectEvaluationRequest, QueryShapes, ResolvedThresholdOverride, }; +use crate::sql_lint::main::cross_join_subject::cross_join_subject; use fensu_policy::apply_suppressions; use fensu_policy::lifecycle::errors::LifecycleError; use fensu_policy::lifecycle::models::{ @@ -580,18 +581,17 @@ fn apply_fault_policy( ) }) .collect(); + let exceptions: &[RuleException] = &request.config.rule_exceptions; let findings = faults .into_iter() - .map(fault_to_finding) + .map(|fault| fault_to_finding(fault, exceptions)) .collect::, _>>()?; - let suppressions = request - .config - .rule_exceptions + let suppressions = exceptions .iter() .map(|entry| ExactSuppression { code: entry.rule.clone(), path: entry.path.clone(), - symbol: None, + symbol: exception_symbol(entry), reason: entry.reason.clone(), }) .collect::>(); @@ -629,13 +629,71 @@ fn apply_fault_policy( .collect()) } -fn fault_to_finding(fault: Fault) -> Result { +/// The symbol a relation-level exception and the findings it covers share. +fn relation_symbol(relation: &str, cte: Option<&str>) -> String { + let relation: String = relation.to_lowercase(); + match cte { + Some(cte) => format!("relation {relation} in CTE {}", cte.to_lowercase()), + None => format!("relation {relation}"), + } +} + +/// The symbol a relation-level exception covers, `None` for a file-wide exception. +fn exception_symbol(entry: &RuleException) -> Option { + let relation: &str = entry.relation.as_deref()?; + Some(relation_symbol(relation, entry.cte.as_deref())) +} + +/// The relation-level exception symbol that covers `fault`: the CTE-scoped one first. +fn covering_symbol(fault: &Fault, exceptions: &[RuleException]) -> Option { + if !RELATION_EXCEPTION_RULES.contains(&fault.code.as_str()) { + return None; + } + let scoped: Vec<&RuleException> = exceptions + .iter() + .filter(|entry| { + entry.rule == fault.code && entry.path == fault.path && entry.relation.is_some() + }) + .collect(); + if scoped.is_empty() { + return None; + } + let (relation, cte) = cross_join_subject(&fault.message)?; + let matches = |entry: &&RuleException, cte: Option<&str>| { + entry + .relation + .as_deref() + .is_some_and(|named| named.eq_ignore_ascii_case(&relation)) + && match (entry.cte.as_deref(), cte) { + (Some(named), Some(found)) => named.eq_ignore_ascii_case(found), + (None, _) => true, + (Some(_), None) => false, + } + }; + scoped + .iter() + .find(|entry| entry.cte.is_some() && matches(entry, cte.as_deref())) + .or_else(|| { + scoped + .iter() + .find(|entry| entry.cte.is_none() && matches(entry, None)) + }) + .map(|entry| { + relation_symbol( + entry.relation.as_deref().unwrap_or_default(), + entry.cte.as_deref(), + ) + }) +} + +fn fault_to_finding(fault: Fault, exceptions: &[RuleException]) -> Result { + let symbol: Option = covering_symbol(&fault, exceptions); Ok(Finding { code: fault.code, path: fault.path, line: Some(u32::try_from(fault.line).map_err(|_| "rule finding line exceeds u32")?), column: Some(u32::try_from(fault.column).map_err(|_| "rule finding column exceeds u32")?), - symbol: None, + symbol, message: fault.message, remediation: Some(fault.remediation), severity: FindingSeverity::Blocking, @@ -656,6 +714,12 @@ fn finding_to_fault(finding: Finding) -> Fault { fn lifecycle_error(error: LifecycleError) -> String { match error { + LifecycleError::StaleSuppression { + code, + path, + symbol: Some(symbol), + .. + } => format!("stale rule exception suppresses no finding: {code} at {path} ({symbol})"), LifecycleError::StaleSuppression { code, path, .. } => { format!("stale rule exception suppresses no finding: {code} at {path}") } diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index 5195d6dd8a..3f069465db 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -8,8 +8,12 @@ mod evaluation; mod helpers; #[path = "tests/test_incremental_rules.rs"] mod incremental_rules; +#[path = "tests/test_published_layers.rs"] +mod published_layers; #[path = "tests/test_query_facts.rs"] mod query_facts; +#[path = "tests/test_relation_exceptions.rs"] +mod relation_exceptions; #[path = "tests/test_retained_rows.rs"] mod retained_rows; #[path = "tests/test_scope_facts.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index d68e0cc521..4bba084546 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -12,12 +12,14 @@ use crate::engine::main::discard_speculative_rows::discard_speculative_rows; use crate::engine::main::retained_request::retained_request; use crate::models::{ Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, Model, - Reference, RetainedRows, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, + Reference, RetainedRows, RowsEvaluation, RowsRequest, RulesConfig, SpeculativeAnswer, + SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; use crate::engine::_helpers::evaluation::parse_parts; +use crate::engine::main::finalize_rows::finalize_rows; use crate::engine::tests::test_types; pub(crate) fn request(project_dir: &TempDir, config: &Value) -> String { @@ -735,3 +737,98 @@ pub(crate) fn without_sql(models: Vec) -> Value { .collect(); serde_json::to_value(stripped).unwrap_or_default() } + +/// One published model JSON row, stored under its domain's `mart` folder. +fn published_model(name: &str, references: &[&str]) -> Value { + let domain: &str = name.split("__").next().unwrap_or_default(); + json!({ + "name": name, + "relative_path": format!("models/{domain}/mart/{name}.sql"), + "query_sql": "SELECT 1", + "authored_sql": "SELECT 1", + "references": references + .iter() + .map(|reference| json!({"ref_kind": "ref", "ref_name": reference})) + .collect::>(), + }) +} + +/// `(path, message and remediation)` of every fault the case's published models raise. +pub(crate) fn published_layer_faults( + test_case: &test_types::PublishedLayerTestCase, +) -> Result, String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let config = json!({ + "select": [test_case.code], + "rule_options": {test_case.code: test_case.options}, + "cache": {"enabled": false} + }); + let mut request: Value = + serde_json::from_str(&request(&project_dir, &config)).map_err(|error| error.to_string())?; + request["models"] = test_case + .models + .iter() + .map(|(name, references)| published_model(name, references)) + .collect(); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(result["faults"] + .as_array() + .ok_or("faults must be an array")? + .iter() + .map(|fault| { + ( + fault["path"].as_str().unwrap_or_default().to_owned(), + format!( + "{}\n{}", + fault["message"].as_str().unwrap_or_default(), + fault["remediation"].as_str().unwrap_or_default() + ), + ) + }) + .collect()) +} + +/// The model path the relation-exception cases use. +pub(crate) const CROSS_JOIN_MODEL_PATH: &str = "models/orders.sql"; +/// The Rule whose findings name a joined relation. +pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; + +fn cross_join_fault(message: &str, line: u64) -> Fault { + Fault { + unevaluated: false, + code: CROSS_JOIN_CODE.to_owned(), + path: CROSS_JOIN_MODEL_PATH.to_owned(), + line, + column: 1, + message: message.to_owned(), + remediation: String::new(), + } +} + +/// Finalize a top-level `order_policy` join and a `customers` join in CTE `checks`. +pub(crate) fn finalize_cross_joins(exceptions: Value) -> Result, String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + std::fs::create_dir_all(project_dir.path().join("models")) + .map_err(|error| error.to_string())?; + std::fs::write(project_dir.path().join(CROSS_JOIN_MODEL_PATH), "SELECT 1") + .map_err(|error| error.to_string())?; + let config: RulesConfig = serde_json::from_value(json!({"rule_exceptions": exceptions})) + .map_err(|error| error.to_string())?; + let findings: Vec = vec![ + cross_join_fault( + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + 3, + ), + cross_join_fault( + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + 7, + ), + ]; + finalize_rows( + project_dir.path().to_string_lossy().into_owned(), + config, + &[CROSS_JOIN_CODE.to_owned()], + findings, + ) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index 81853977f4..79678b5728 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -30,6 +30,7 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRDECLARATION306", RuleScope::Project), ("SQBRGRAPH101", RuleScope::Project), ("SQBRGRAPH102", RuleScope::Model), + ("SQBRGRAPH103", RuleScope::Project), ("SQBRMODEL101", RuleScope::Model), ("SQBRMODEL102", RuleScope::Model), ("SQBRMODEL103", RuleScope::Model), @@ -40,6 +41,7 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPROJECT104", RuleScope::Model), ("SQBRPROJECT105", RuleScope::Model), ("SQBRPROJECT106", RuleScope::Model), + ("SQBRPROJECT107", RuleScope::Project), ("SQBRPROJECT201", RuleScope::Project), ("SQBRPROJECT202", RuleScope::Project), ("SQBRPROJECT203", RuleScope::Project), diff --git a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs new file mode 100644 index 0000000000..80da53a289 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs @@ -0,0 +1,170 @@ +use serde_json::json; + +use crate::engine::tests::helpers::published_layer_faults; +use crate::engine::tests::test_types::PublishedLayerTestCase; + +#[test] +fn given_published_models_when_evaluating_layer_rules_then_reads_and_shapes_match() +-> Result<(), String> { + let test_cases = [ + PublishedLayerTestCase { + description: "a consumer variant reads its own domain's published model", + code: "SQBRGRAPH103", + models: &[ + ("sales__mart__orders", &["sales__int_clean__orders"]), + ("sales__mart__orders__finance", &["sales__mart__orders"]), + ], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__finance.sql", + "sales__mart__orders__finance reads sales__mart__orders; published models must not read published models of their own domain", + )], + }, + PublishedLayerTestCase { + description: "the finding gives the intermediate move command", + code: "SQBRGRAPH103", + models: &[("sales__mart__orders__finance", &["sales__mart__orders"])], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__finance.sql", + "sqb mv sales__mart__orders__finance models/sales/intermediate/enriched/sales__int_enriched__orders__finance.sql", + )], + }, + PublishedLayerTestCase { + description: "a view of its own entity may read the table", + code: "SQBRGRAPH103", + models: &[ + ("sales__mart_v__orders", &["sales__mart__orders"]), + ( + "sales__mart_v__orders__csv_export", + &["sales__mart__orders"], + ), + ], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualified view may read the table with its qualifier", + code: "SQBRGRAPH103", + models: &[("sales__mart_v__orders__fra", &["sales__mart__orders__fra"])], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualified view may not read the table with another qualifier", + code: "SQBRGRAPH103", + models: &[("sales__mart_v__orders__fra", &["sales__mart__orders__deu"])], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart_v__orders__fra.sql", + "sales__mart_v__orders__fra reads sales__mart__orders__deu", + )], + }, + PublishedLayerTestCase { + description: "a view of another entity may not", + code: "SQBRGRAPH103", + models: &[("sales__mart_v__customers", &["sales__mart__orders"])], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart_v__customers.sql", + "sales__mart_v__customers reads sales__mart__orders", + )], + }, + PublishedLayerTestCase { + description: "reading another domain's published model is allowed", + code: "SQBRGRAPH103", + models: &[("finance__mart__revenue", &["sales__mart__orders"])], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "names outside the grammar are not checked", + code: "SQBRGRAPH103", + models: &[("orders_report", &["orders_summary"])], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "configured published layers replace the defaults", + code: "SQBRGRAPH103", + models: &[("sales__mart__orders__finance", &["sales__mart__orders"])], + options: json!({"published_layers": ["mart_v"]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualified variant in an entity-shaped domain", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__customers", &[]), + ("sales__mart__orders__finance", &[]), + ], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__finance.sql", + "domain sales publishes 2 models as ____ and 1 as ______; sales__mart__orders__finance does not fit the majority shape ____", + )], + }, + PublishedLayerTestCase { + description: "an unqualified model in a qualified domain gets a rename command", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ("sales__mart__customers", &[]), + ], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__customers.sql", + "sqb rename sales__mart__customers sales__mart__customers__core", + )], + }, + PublishedLayerTestCase { + description: "an allowlisted qualifier does not count towards the shape", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart_v__orders__csv_export", &[]), + ("finance__mart__revenue", &[]), + ("finance__mart__revenue__fra", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"qualifier": "csv_export", "reason": "File export views of a published entity."}, + {"domain": "finance", "qualifier": "fra", "reason": "Published once per country."} + ]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a domain-scoped allowlist entry does not cover other domains", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__orders__fra", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"domain": "finance", "qualifier": "fra", "reason": "Published once per country."} + ]}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__fra.sql", + "allowed_qualifiers = [{ domain = \"sales\", qualifier = \"fra\"", + )], + }, + ]; + + for test_case in test_cases { + let found: Vec<(String, String)> = published_layer_faults(&test_case)?; + + assert_eq!( + found.len(), + test_case.expected_faults.len(), + "{}: {found:?}", + test_case.description + ); + for ((path, text), (expected_path, fragment)) in found.iter().zip(test_case.expected_faults) + { + assert_eq!(path, expected_path, "{}", test_case.description); + assert!(text.contains(fragment), "{}: {text}", test_case.description); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs b/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs new file mode 100644 index 0000000000..46ec32a515 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs @@ -0,0 +1,73 @@ +use serde_json::json; + +use crate::engine::tests::helpers::{CROSS_JOIN_CODE, CROSS_JOIN_MODEL_PATH, finalize_cross_joins}; +use crate::engine::tests::test_types::{RelationExceptionTestCase, StaleRelationExceptionTestCase}; + +#[test] +fn given_relation_level_exceptions_when_finalizing_then_exactly_the_named_joins_are_excepted() +-> Result<(), String> { + let test_cases = [ + RelationExceptionTestCase { + description: "a relation exception covers only that relation", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "order_policy", "reason": "Every order meets every policy."}]), + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + RelationExceptionTestCase { + description: "a CTE-scoped exception covers the join in that CTE only", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "cte": "checks", "reason": "Deliberate."}]), + expected_messages: &[ + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + ], + }, + RelationExceptionTestCase { + description: "relation names match case-insensitively", + exceptions: json!([ + {"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "Order_Policy", "reason": "Every order meets every policy."}, + {"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "reason": "Deliberate."} + ]), + expected_messages: &[], + }, + ]; + + for test_case in test_cases { + let kept = finalize_cross_joins(test_case.exceptions.clone())?; + + let messages: Vec<&str> = kept.iter().map(|fault| fault.message.as_str()).collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_relation_level_exceptions_matching_nothing_when_finalizing_then_they_are_stale() { + let test_cases = [ + StaleRelationExceptionTestCase { + description: "a relation exception that matches nothing is stale", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "shipments", "reason": "Gone."}]), + expected_error: "stale rule exception suppresses no finding: SQBRSQL038 at models/orders.sql (relation shipments)", + }, + StaleRelationExceptionTestCase { + description: "a CTE-scoped exception for another CTE is stale", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "cte": "other", "reason": "Gone."}]), + expected_error: "stale rule exception suppresses no finding: SQBRSQL038 at models/orders.sql (relation customers in CTE other)", + }, + ]; + + for test_case in test_cases { + let error: String = finalize_cross_joins(test_case.exceptions.clone()) + .err() + .unwrap_or_default(); + + assert!( + error.contains(test_case.expected_error), + "{}: {error}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index c22ae94525..e631ce4e0c 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -298,3 +298,23 @@ pub(crate) struct RowCodecTestCase { /// How many truncations of the rows still read back. pub(crate) expected_truncations_read: usize, } + +pub(crate) struct PublishedLayerTestCase { + pub(crate) description: &'static str, + pub(crate) code: &'static str, + pub(crate) models: &'static [(&'static str, &'static [&'static str])], + pub(crate) options: Value, + pub(crate) expected_faults: &'static [(&'static str, &'static str)], +} + +pub(crate) struct RelationExceptionTestCase { + pub(crate) description: &'static str, + pub(crate) exceptions: Value, + pub(crate) expected_messages: &'static [&'static str], +} + +pub(crate) struct StaleRelationExceptionTestCase { + pub(crate) description: &'static str, + pub(crate) exceptions: Value, + pub(crate) expected_error: &'static str, +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index b5909ebcde..9f61cde2ae 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -6,7 +6,8 @@ use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::lint_text::constants::{ - INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, + CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, + OMITTED_CEREMONIAL_SELECT_SQL, }; use crate::lint_text::models::{LintSqlBody, LintSqlOptions}; use crate::sql_lint::main::batch_engine::lint_batch_json; @@ -26,6 +27,13 @@ pub(crate) fn omitted_select(body: &LintSqlBody) -> Option { } } +/// Whether `text` may hold a `CROSS JOIN`; other bodies omit relation names to share requests. +fn mentions_cross_join(text: &str) -> bool { + text.as_bytes() + .windows(CROSS_KEYWORD.len()) + .any(|window| window.eq_ignore_ascii_case(CROSS_KEYWORD)) +} + fn strings(values: &[String]) -> Value { Value::from(values.to_vec()) } @@ -95,6 +103,14 @@ pub(crate) fn lint_payload( .collect(); let _ = payload.insert("relation_keys".to_string(), Value::Object(keys)); } + if !body.relation_names.is_empty() && mentions_cross_join(&body.lint_text) { + let names: Map = body + .relation_names + .iter() + .map(|(sentinel, name)| (sentinel.clone(), Value::from(name.clone()))) + .collect(); + let _ = payload.insert("relation_names".to_string(), Value::Object(names)); + } Value::Object(payload) } diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs index aa80d6d1ab..324573671d 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -1,153 +1,195 @@ -//! Suppression directive parsing and matching. +//! Inline suppression directives in SQL comments, rejected in favour of configured exceptions. use std::sync::LazyLock; use regex::Regex; -use sqlbuild_core::text::main::python_isspace::python_isspace; +use crate::constants::SETTING_SNIPPET_INDENT; use crate::lint_text::_helpers::lines::LineSpan; use crate::lint_text::_helpers::sites::compiled; use crate::lint_text::constants::{ - FORBIDDEN_FIX_REASON, FORBIDDEN_MESSAGE, FORBIDDEN_REMEDIATION, INVALID_MESSAGE, - INVALID_REMEDIATION, NATIVE_RULE_PREFIX, REASON_REQUIRED_FIX_REASON, SUPPRESSION_CODE, - SUPPRESSION_PATTERN, SUPPRESSION_PREFIX, UNUSED_REMEDIATION, + BLOCK_COMMENT_CLOSER, BLOCK_COMMENT_OPENER, COMMENT_OPENER_BYTES, CROSS_JOIN_CODE, + CROSS_JOIN_RELATION_PATTERN, DIRECTIVE_FIX_REASON, DIRECTIVE_MARKER, DIRECTIVE_MESSAGE, + DIRECTIVE_PATTERN, LINE_COMMENT_PREFIX, REASON_PLACEHOLDER, RULE_PLACEHOLDER, }; use crate::lint_text::errors::LintTextError; -use crate::lint_text::models::{ - RetainedViolation, SuppressibleViolation, SuppressionDirective, SuppressionFinding, -}; +use crate::lint_text::models::SuppressionFinding; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -static SUPPRESSION: LazyLock> = - LazyLock::new(|| Regex::new(SUPPRESSION_PATTERN)); +static DIRECTIVE: LazyLock> = + LazyLock::new(|| Regex::new(DIRECTIVE_PATTERN)); +static CROSS_JOIN_RELATION: LazyLock> = + LazyLock::new(|| Regex::new(CROSS_JOIN_RELATION_PATTERN)); -/// Valid directives, and the one-based lines of invalid ones. -pub(crate) fn parse_directives( - lines: &[LineSpan<'_>], -) -> Result<(Vec, Vec), LintTextError> { - let pattern: &Regex = compiled(&SUPPRESSION)?; - let mut directives: Vec = Vec::new(); - let mut invalid: Vec = Vec::new(); - for (index, line) in lines.iter().enumerate() { - if !line.content.to_lowercase().contains(SUPPRESSION_PREFIX) { - continue; - } - let Some(captures) = pattern.captures(line.content) else { - invalid.push(index + 1); - continue; - }; - let mut target: usize = index + 1; - while target < lines.len() && skippable(lines[target].content) { - target += 1; +/// One inline directive: what it names, and what the exception entry for it holds. +struct Directive { + line: usize, + code: Option, + reason: Option, + relation: Option, +} + +/// The rejection finding for every directive inside a SQL comment of one file. +pub(crate) fn directive_findings( + file: usize, + path: &str, + (contents, lines): (&str, &[LineSpan<'_>]), +) -> Result, LintTextError> { + if !mentions_directive(contents) { + return Ok(Vec::new()); + } + let pattern: &Regex = compiled(&DIRECTIVE)?; + let relation_pattern: &Regex = compiled(&CROSS_JOIN_RELATION)?; + let mut findings: Vec = Vec::new(); + for (start, end) in comment_ranges(contents) { + let body: &str = comment_body(&contents[start..end]); + let body_start: usize = start + COMMENT_OPENER_BYTES; + for captures in pattern.captures_iter(body) { + let Some(found) = captures.get(0) else { + continue; + }; + let Some(index) = line_index(contents, lines, body_start + found.start()) else { + continue; + }; + let text = |name: &str| -> Option { + captures + .name(name) + .map(|value| value.as_str().trim().to_owned()) + .filter(|value| !value.is_empty()) + }; + let code: Option = text("code"); + let relation: Option = if code.as_deref() == Some(CROSS_JOIN_CODE) { + target_line(contents, lines, (index, start)) + .and_then(|target| cross_joined_relation(relation_pattern, target)) + } else { + None + }; + let directive = Directive { + line: index + 1, + code, + reason: text("reason"), + relation, + }; + findings.push(SuppressionFinding { + file, + line: directive.line, + message: DIRECTIVE_MESSAGE.to_owned(), + remediation: remediation(&directive, path), + fault: true, + fix: None, + fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), + }); } - directives.push(SuppressionDirective { - code: captures["code"].to_string(), - line: index + 1, - target: target + 1, - }); } - Ok((directives, invalid)) + Ok(findings) } -/// Blank lines and comment lines between a directive and the line it suppresses. -fn skippable(line: &str) -> bool { - let stripped: &str = line.trim_start_matches(python_isspace); - stripped.is_empty() || stripped.starts_with("--") +/// A cheap check that `contents` may hold a directive at all. +fn mentions_directive(contents: &str) -> bool { + contents + .as_bytes() + .windows(DIRECTIVE_MARKER.len()) + .any(|window| window.eq_ignore_ascii_case(DIRECTIVE_MARKER)) } -/// Where a retained violation sits, its code and whether a native rule reported it. -struct SuppressedTarget<'a> { - file: usize, - line: usize, - code: &'a str, - native: bool, +/// Byte ranges of the `--` line comments and `/* */` block comments, skipping quoted text. +fn comment_ranges(contents: &str) -> Vec<(usize, usize)> { + let syntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT_PREFIX.to_owned()], + ..LexicalSyntax::default() + }; + dialect_non_code_ranges(contents, &syntax) + .into_iter() + .filter(|(start, _)| { + let rest: &str = &contents[*start..]; + rest.starts_with(LINE_COMMENT_PREFIX) || rest.starts_with(BLOCK_COMMENT_OPENER) + }) + .collect() } -/// Whether a retained violation is the native finding one directive suppresses. -pub(crate) fn matches_directive( - item: &RetainedViolation, - violations: &[SuppressibleViolation], - file: usize, - directive: &SuppressionDirective, -) -> bool { - let target: SuppressedTarget<'_> = match item { - RetainedViolation::Existing(index) => { - let violation: &SuppressibleViolation = &violations[*index]; - SuppressedTarget { - file: violation.file, - line: violation.line, - code: &violation.code, - native: violation.native, - } - } - RetainedViolation::Added(finding) => SuppressedTarget { - file: finding.file, - line: finding.line, - code: SUPPRESSION_CODE, - native: true, - }, - }; - target.file == file - && target.line == directive.target - && target.code == directive.code - && target.native - && target.code.starts_with(NATIVE_RULE_PREFIX) +/// A comment's text without its opener and, for a block comment, its closer. +fn comment_body(comment: &str) -> &str { + let body: &str = &comment[COMMENT_OPENER_BYTES..]; + if comment.starts_with(BLOCK_COMMENT_OPENER) { + body.strip_suffix(BLOCK_COMMENT_CLOSER).unwrap_or(body) + } else { + body + } } -/// Findings for every directive line when the project forbids suppressions. -pub(crate) fn forbidden_findings( - file: usize, - directives: &[SuppressionDirective], - invalid: &[usize], -) -> Vec { - let mut lines: Vec = invalid +/// The zero-based line holding byte `offset` of `contents`. +fn line_index(contents: &str, lines: &[LineSpan<'_>], offset: usize) -> Option { + let position: usize = contents[..offset].chars().count(); + lines .iter() - .copied() - .chain(directives.iter().map(|directive| directive.line)) - .collect(); - lines.sort_unstable(); - lines.dedup(); + .position(|line| line.start <= position && position < line.end) +} + +/// The code a directive covers: code before its comment, else the next non-comment line. +fn target_line<'a>( + contents: &'a str, + lines: &[LineSpan<'a>], + (directive, comment_start): (usize, usize), +) -> Option<&'a str> { + let line: &LineSpan<'a> = lines.get(directive)?; + let line_start: usize = contents + .char_indices() + .nth(line.start) + .map_or(contents.len(), |(byte, _)| byte); + let before: &str = contents.get(line_start..comment_start).unwrap_or_default(); + if !before.trim_matches(python_isspace).is_empty() { + return Some(before); + } lines - .into_iter() - .map(|line| { - RetainedViolation::Added(SuppressionFinding { - file, - line, - message: FORBIDDEN_MESSAGE.to_string(), - remediation: FORBIDDEN_REMEDIATION, - fault: true, - fix: None, - fix_unavailable_reason: Some(FORBIDDEN_FIX_REASON), - }) + .iter() + .skip(directive + 1) + .map(|line| line.content) + .find(|content| { + let stripped: &str = content.trim_start_matches(python_isspace); + !stripped.is_empty() && !stripped.starts_with(LINE_COMMENT_PREFIX) }) - .collect() } -/// The finding for an invalid directive line. -pub(crate) fn invalid_finding(file: usize, line: usize) -> RetainedViolation { - RetainedViolation::Added(SuppressionFinding { - file, - line, - message: INVALID_MESSAGE.to_string(), - remediation: INVALID_REMEDIATION, - fault: false, - fix: None, - fix_unavailable_reason: Some(REASON_REQUIRED_FIX_REASON), - }) +/// The relation a `CROSS JOIN` on `line` reads: a dependency's name or an authored identifier. +fn cross_joined_relation(pattern: &Regex, line: &str) -> Option { + let captures = pattern.captures(line)?; + captures + .name("dependency") + .or_else(|| captures.name("identifier")) + .map(|found| found.as_str().to_owned()) } -/// The finding for a directive that suppressed nothing, with an edit removing its line. -pub(crate) fn unused_finding( - file: usize, - directive: &SuppressionDirective, - span: &LineSpan<'_>, -) -> RetainedViolation { - RetainedViolation::Added(SuppressionFinding { - file, - line: directive.line, - message: format!("Unused suppression for {}", directive.code), - remediation: UNUSED_REMEDIATION, - fault: false, - fix: Some((span.start, span.end)), - fix_unavailable_reason: None, - }) +/// Remove-and-declare help, with the exact `[[rules.rule_exceptions]]` entry to add. +fn remediation(directive: &Directive, path: &str) -> String { + let mut entry: Vec = vec![ + "[[rules.rule_exceptions]]".to_owned(), + format!( + "rule = {}", + toml_string(directive.code.as_deref().unwrap_or(RULE_PLACEHOLDER)) + ), + format!("path = {}", toml_string(path)), + ]; + if let Some(relation) = &directive.relation { + entry.push(format!("relation = {}", toml_string(relation))); + } + entry.push(format!( + "reason = {}", + toml_string(directive.reason.as_deref().unwrap_or(REASON_PLACEHOLDER)) + )); + let rule: &str = directive.code.as_deref().unwrap_or("Rule"); + let snippet: String = entry + .iter() + .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) + .collect(); + format!( + "Remove this comment, then fix the {rule} finding, or record it as a reviewed exception in \ + {PROJECT_CONFIG_FILENAME}:{snippet}" + ) +} + +fn toml_string(value: &str) -> String { + toml::Value::String(value.to_owned()).to_string() } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index fd41451e50..099d33f95a 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -26,22 +26,24 @@ pub(crate) const LINT_SITE_PATTERN: &str = r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\()"; pub(crate) const FORMAT_SITE_PATTERN: &str = r"(?i)^(?:@@|\$\{|@|(?:__cursor_end|__cursor_start|__dbt_ref|__empty_fixture|__ref|__seed|__source|__table_fn|__udf)\s*\()"; -/// Diagnostic code of every suppression finding. +/// Diagnostic code of every inline suppression directive finding. pub const SUPPRESSION_CODE: &str = "SQBRSQL000"; -pub(crate) const SUPPRESSION_PREFIX: &str = "sqb: ignore"; -pub(crate) const SUPPRESSION_PATTERN: &str = - r"^\s*--\s*sqb:\s*ignore\s+(?P[A-Za-z0-9_-]+)\s+because\s+(?P\S.*)\s*$"; -pub(crate) const NATIVE_RULE_PREFIX: &str = "SQBRSQL"; -pub(crate) const INVALID_MESSAGE: &str = "Suppression directive is invalid"; -pub(crate) const INVALID_REMEDIATION: &str = "Use '-- sqb: ignore CODE because '."; -pub(crate) const UNUSED_REMEDIATION: &str = "Remove the stale suppression directive."; -pub(crate) const FORBIDDEN_MESSAGE: &str = - "Inline Rule suppressions are not allowed in this project"; -pub(crate) const FORBIDDEN_REMEDIATION: &str = "Remove the directive and fix the finding it suppresses; sqlbuild_project.toml sets [rules] allow_exceptions = false. To allow inline suppressions, set this in sqlbuild_project.toml:\n [rules]\n allow_exceptions = true"; -pub(crate) const FORBIDDEN_FIX_REASON: &str = - "removing the directive exposes the finding it suppresses"; -pub(crate) const REASON_REQUIRED_FIX_REASON: &str = - "a valid suppression reason requires user intent"; +/// Text every directive contains, checked before scanning a file's comments. +pub(crate) const DIRECTIVE_MARKER: &[u8] = b"sqb:"; +pub(crate) const BLOCK_COMMENT_OPENER: &str = "/*"; +pub(crate) const BLOCK_COMMENT_CLOSER: &str = "*/"; +/// Both comment openers are two bytes long. +pub(crate) const COMMENT_OPENER_BYTES: usize = 2; +pub(crate) const DIRECTIVE_PATTERN: &str = + r"(?i)sqb:\s*ignore\b\s*(?P[A-Za-z0-9_-]*)(?:\s+because\s+(?P.*\S))?"; +pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; +pub(crate) const CROSS_JOIN_RELATION_PATTERN: &str = r#"(?i)\bcross\s+join\s+(?:__(?:ref|seed|source)\s*\(\s*['"](?P[^'"]+)['"]\s*\)|(?P[A-Za-z_][A-Za-z0-9_$.]*))"#; +pub(crate) const DIRECTIVE_MESSAGE: &str = + "Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml"; +pub(crate) const RULE_PLACEHOLDER: &str = ""; +pub(crate) const REASON_PLACEHOLDER: &str = ""; +pub(crate) const DIRECTIVE_FIX_REASON: &str = + "moving an exception into sqlbuild_project.toml requires review"; pub(crate) const FIXTURE_TYPED_NULL_PATTERN: &str = r"(?i)(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)"; pub(crate) const MODEL_FIXTURE_PATTERN: &str = @@ -162,3 +164,5 @@ pub(crate) const GROUPED_NAMED_ROLE_KINDS: [(&[&str], &str); 9] = [ (&["hooks", "python"], "python_hook"), (&["_hooks", "python"], "python_hook"), ]; +/// The keyword whose presence makes a lint request carry relation names. +pub(crate) const CROSS_KEYWORD: &[u8] = b"cross"; diff --git a/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs b/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs deleted file mode 100644 index 590a0f4259..0000000000 --- a/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! `apply_suppressions`: reason-required next-line suppressions applied to lint violations. - -use crate::lint_text::_helpers::lines::{LineSpan, line_spans}; -use crate::lint_text::_helpers::suppressions::{ - forbidden_findings, invalid_finding, matches_directive, parse_directives, unused_finding, -}; -use crate::lint_text::errors::LintTextError; -use crate::lint_text::models::{RetainedViolation, SuppressibleViolation}; - -/// Remove matching next-line diagnostics, or reject every directive when suppressions are off. -pub fn apply_suppressions( - files: &[&str], - violations: &[SuppressibleViolation], - allow_suppressions: bool, -) -> Result, LintTextError> { - let mut retained: Vec = (0..violations.len()) - .map(RetainedViolation::Existing) - .collect(); - for (file, contents) in files.iter().enumerate() { - let lines: Vec> = line_spans(contents); - let (directives, invalid) = parse_directives(&lines)?; - if !allow_suppressions { - retained.extend(forbidden_findings(file, &directives, &invalid)); - continue; - } - retained.extend(invalid.into_iter().map(|line| invalid_finding(file, line))); - for directive in directives { - match retained - .iter() - .position(|item| matches_directive(item, violations, file, &directive)) - { - Some(index) => { - let _ = retained.remove(index); - } - None => retained.push(unused_finding(file, &directive, &lines[directive.line - 1])), - } - } - } - Ok(retained) -} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index d09920580a..2ff3fab536 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -1,4 +1,3 @@ -pub mod apply_suppressions; pub mod dependency_sites; pub mod first_header_only; pub mod format_fixture_blocks; @@ -9,6 +8,7 @@ pub mod named_declaration_role_kind; pub mod neutralize_audit_parameters; pub mod neutralize_context_interpolation; pub mod neutralize_interpolation; +pub mod reject_inline_suppressions; pub mod resolve_format_settings; pub mod restore_interpolation; pub mod scan_fixture_null_candidates; diff --git a/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs b/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs new file mode 100644 index 0000000000..23cf26ff3f --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs @@ -0,0 +1,18 @@ +//! `reject_inline_suppressions`: an error for every `sqb: ignore` directive in a SQL comment. + +use crate::lint_text::_helpers::lines::{LineSpan, line_spans}; +use crate::lint_text::_helpers::suppressions::directive_findings; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::SuppressionFinding; + +/// A finding per directive line of each `(relative path, contents)`, printing its exception entry. +pub fn reject_inline_suppressions( + files: &[(&str, &str)], +) -> Result, LintTextError> { + let mut findings: Vec = Vec::new(); + for (file, (path, contents)) in files.iter().enumerate() { + let lines: Vec> = line_spans(contents); + findings.extend(directive_findings(file, path, (contents, &lines))?); + } + Ok(findings) +} diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index fc0b21277b..85a527e749 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -40,42 +40,18 @@ pub struct Neutralized { pub sites: Vec, } -/// The identity of one existing violation that a suppression can remove. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SuppressibleViolation { - pub file: usize, - pub line: usize, - pub code: String, - pub native: bool, -} - -/// One valid directive: its code, its line and the line it suppresses. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SuppressionDirective { - pub code: String, - pub line: usize, - pub target: usize, -} - -/// A suppression finding on one directive line; offsets are character positions. +/// The rejection finding on one inline directive line; offsets are character positions. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SuppressionFinding { pub file: usize, pub line: usize, pub message: String, - pub remediation: &'static str, + pub remediation: String, pub fault: bool, pub fix: Option<(usize, usize)>, pub fix_unavailable_reason: Option<&'static str>, } -/// One retained violation: an input violation by index, or a new suppression finding. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RetainedViolation { - Existing(usize), - Added(SuppressionFinding), -} - /// SQL-test files that may hold removable typed nulls, and the models their fixtures stand in for. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct FixtureNullCandidates { @@ -139,6 +115,8 @@ pub struct LintSqlBody { pub allows_dynamic_output_star: bool, pub allows_empty_fixture_star: bool, pub relation_keys: Vec<(String, Vec>)>, + /// The relation name each dependency sentinel stands for. + pub relation_names: Vec<(String, String)>, } /// The rule selection and thresholds of one SQL lint run. diff --git a/crates/sqlbuild-rules/src/lint_text/tests.rs b/crates/sqlbuild-rules/src/lint_text/tests.rs index 57c38b8979..5fb19d6a2d 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests.rs @@ -2,6 +2,8 @@ mod fixture_formatting; #[path = "tests/test_format_settings.rs"] mod format_settings; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_lint_text.rs"] mod lint_text; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs new file mode 100644 index 0000000000..96a8aef7f0 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs @@ -0,0 +1,25 @@ +use crate::lint_text::constants::{DIRECTIVE_FIX_REASON, DIRECTIVE_MESSAGE}; +use crate::lint_text::models::SuppressionFinding; + +const INDENT: &str = "\n "; + +/// The rejection finding for a directive on `line`, printing `rest` after the entry header. +pub(crate) fn directive_finding(line: usize, rule: &str, rest: &[&str]) -> SuppressionFinding { + let mut remediation: String = format!( + "Remove this comment, then fix the {rule} finding, or record it as a reviewed exception \ + in sqlbuild_project.toml:{INDENT}[[rules.rule_exceptions]]" + ); + for line in rest { + remediation.push_str(INDENT); + remediation.push_str(line); + } + SuppressionFinding { + file: 0, + line, + message: DIRECTIVE_MESSAGE.to_owned(), + remediation, + fault: true, + fix: None, + fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index d5466360bc..7e0c27a01c 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -1,27 +1,24 @@ use crate::lint_text::_helpers::sites::site_pattern; -use crate::lint_text::constants::{ - FORBIDDEN_FIX_REASON, FORBIDDEN_MESSAGE, FORBIDDEN_REMEDIATION, INVALID_MESSAGE, - INVALID_REMEDIATION, REASON_REQUIRED_FIX_REASON, SITE_START_BYTES, UNUSED_REMEDIATION, -}; +use crate::lint_text::constants::SITE_START_BYTES; use crate::lint_text::errors::LintTextError; -use crate::lint_text::main::apply_suppressions::apply_suppressions; use crate::lint_text::main::format_sql_bodies::format_sql_bodies; use crate::lint_text::main::interpolation_text_at::interpolation_text_at; use crate::lint_text::main::lint_sql_bodies::lint_sql_bodies; use crate::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use crate::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use crate::lint_text::main::restore_interpolation::restore_interpolation; -use crate::lint_text::models::{ - ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions, RetainedViolation, - SuppressibleViolation, SuppressionFinding, -}; +use crate::lint_text::models::{ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions}; +use crate::lint_text::tests::helpers::directive_finding; use crate::lint_text::tests::test_types::{ Finding, FormatBodiesTestCase, LintBodiesTestCase, NeutralizeTestCase, ParameterTestCase, RestoreTestCase, SitePatternTestCase, SuppressionTestCase, TokenAtTestCase, }; -const SUPPRESSED_FILE: &str = "-- sqb: ignore SQBRSQL005 because kept\n\n-- note\nselect 1\n-- sqb: ignore X because y\n-- sqb: ignore nope\nselect 2\n"; +const SUPPRESSED_FILE: &str = + "-- sqb: ignore SQBRSQL005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; +const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQBRSQL038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; #[test] fn given_site_patterns_when_text_opens_with_another_byte_then_no_site_starts_there() @@ -227,72 +224,91 @@ fn given_character_offset_when_reading_token_then_only_site_starts_have_tokens() } #[test] -fn given_suppression_directives_when_applying_then_matching_findings_are_removed() +fn given_inline_directives_when_rejecting_then_each_prints_its_configured_exception() -> Result<(), LintTextError> { let test_cases = [ SuppressionTestCase { - description: "a used, an unused and an invalid directive", + description: "a reasoned directive and one without a reason", contents: SUPPRESSED_FILE, - violations: vec![(4, "SQBRSQL005"), (7, "SQBRSQL034")], - allow: true, - expected_retained: vec![ - RetainedViolation::Existing(1), - RetainedViolation::Added(SuppressionFinding { - file: 0, - line: 6, - message: INVALID_MESSAGE.to_string(), - remediation: INVALID_REMEDIATION, - fault: false, - fix: None, - fix_unavailable_reason: Some(REASON_REQUIRED_FIX_REASON), - }), - RetainedViolation::Added(SuppressionFinding { - file: 0, - line: 5, - message: "Unused suppression for X".to_string(), - remediation: UNUSED_REMEDIATION, - fault: false, - fix: Some((57, 84)), - fix_unavailable_reason: None, - }), + expected_findings: vec![ + directive_finding( + 1, + "SQBRSQL005", + &[ + "rule = \"SQBRSQL005\"", + "path = \"models/orders.sql\"", + "reason = \"kept\"", + ], + ), + directive_finding( + 5, + "nope", + &[ + "rule = \"nope\"", + "path = \"models/orders.sql\"", + "reason = \"\"", + ], + ), ], }, SuppressionTestCase { - description: "suppressions forbidden by the project", - contents: "select 1\n-- sqb: ignore nope\n", - violations: vec![(1, "SQBRSQL005")], - allow: false, - expected_retained: vec![ - RetainedViolation::Existing(0), - RetainedViolation::Added(SuppressionFinding { - file: 0, - line: 2, - message: FORBIDDEN_MESSAGE.to_string(), - remediation: FORBIDDEN_REMEDIATION, - fault: true, - fix: None, - fix_unavailable_reason: Some(FORBIDDEN_FIX_REASON), - }), - ], + description: "a cross join directive names the joined dependency", + contents: CROSS_JOIN_FILE, + expected_findings: vec![directive_finding( + 3, + "SQBRSQL038", + &[ + "rule = \"SQBRSQL038\"", + "path = \"models/orders.sql\"", + "relation = \"order_policy\"", + "reason = \"every order meets every policy\"", + ], + )], + }, + SuppressionTestCase { + description: "a string literal mentioning a directive is not one", + contents: "select '-- sqb: ignore SQBRSQL004 because text' as note, 'sqb: ignore' as other\n", + expected_findings: vec![], + }, + SuppressionTestCase { + description: "a trailing directive after code names the join on its line", + contents: "select o.order_id\nfrom orders as o\ncross join customers as c -- sqb: ignore SQBRSQL038 because deliberate\n", + expected_findings: vec![directive_finding( + 3, + "SQBRSQL038", + &[ + "rule = \"SQBRSQL038\"", + "path = \"models/orders.sql\"", + "relation = \"customers\"", + "reason = \"deliberate\"", + ], + )], + }, + SuppressionTestCase { + description: "a directive in a block comment", + contents: "select 1\n/* sqb: ignore SQBRSQL005 because kept */\nselect 2\n", + expected_findings: vec![directive_finding( + 2, + "SQBRSQL005", + &[ + "rule = \"SQBRSQL005\"", + "path = \"models/orders.sql\"", + "reason = \"kept\"", + ], + )], + }, + SuppressionTestCase { + description: "a file without directives", + contents: "select 1\n-- a plain comment\n", + expected_findings: vec![], }, ]; for test_case in test_cases { - let violations: Vec = test_case - .violations - .iter() - .map(|(line, code)| SuppressibleViolation { - file: 0, - line: *line, - code: (*code).to_string(), - native: true, - }) - .collect(); - - let retained = apply_suppressions(&[test_case.contents], &violations, test_case.allow)?; + let findings = reject_inline_suppressions(&[("models/orders.sql", test_case.contents)])?; assert_eq!( - retained, test_case.expected_retained, + findings, test_case.expected_findings, "{}", test_case.description ); diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index 3df6eae669..52aeb3db6f 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::lint_text::models::{Neutralized, RetainedViolation}; +use crate::lint_text::models::{Neutralized, SuppressionFinding}; pub(crate) struct NeutralizeTestCase { pub(crate) description: &'static str, @@ -32,9 +32,7 @@ pub(crate) struct TokenAtTestCase { pub(crate) struct SuppressionTestCase { pub(crate) description: &'static str, pub(crate) contents: &'static str, - pub(crate) violations: Vec<(usize, &'static str)>, - pub(crate) allow: bool, - pub(crate) expected_retained: Vec, + pub(crate) expected_findings: Vec, } pub(crate) struct FormatBodiesTestCase { diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 744677e574..d8a6b21248 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -65,6 +65,12 @@ pub struct RuleException { pub rule: String, pub path: String, pub reason: String, + /// The joined relation a relation-level exception covers, for Rules that report one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relation: Option, + /// The top-level CTE that contains the covered relation, when the exception is that narrow. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cte: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Eq, Ord, PartialEq, PartialOrd)] @@ -147,7 +153,6 @@ pub struct RulesConfig { pub rule_exceptions: Vec, pub graph_edge_exceptions: Vec, pub rule_ignores: Vec, - pub allow_exceptions: bool, pub select_star_allow: Vec, pub domains: Vec, pub approved_source_tokens: Vec, @@ -168,7 +173,6 @@ impl Default for RulesConfig { rule_exceptions: vec![], graph_edge_exceptions: vec![], rule_ignores: vec![], - allow_exceptions: true, select_star_allow: vec![], domains: vec![], approved_source_tokens: vec![], diff --git a/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs index 8f5b8bdc4d..2884da48cc 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs @@ -12,7 +12,7 @@ fn key_sets(keys: &[Vec]) -> String { sets.join("\u{1d}") } -/// A digest of the dialect, literal and ranking limits, sorted relation keys and suppressions. +/// A digest of the dialect, literal and ranking limits and sorted relation keys. #[must_use] pub fn lint_key(input: &LintKeyInput) -> String { let mut relation_keys: Vec<(String, Vec>)> = input.relation_keys.clone(); @@ -29,7 +29,6 @@ pub fn lint_key(input: &LintKeyInput) -> String { &input.max_literal_length.to_string(), &input.max_ranking_order_by.to_string(), &relations, - if input.allow_suppressions { "1" } else { "0" }, ], ) } diff --git a/crates/sqlbuild-rules/src/rule_cache/models.rs b/crates/sqlbuild-rules/src/rule_cache/models.rs index accfe26913..8a99a27b88 100644 --- a/crates/sqlbuild-rules/src/rule_cache/models.rs +++ b/crates/sqlbuild-rules/src/rule_cache/models.rs @@ -8,7 +8,6 @@ pub struct LintKeyInput { pub max_ranking_order_by: i64, /// Each relation and its key column sets, in any order. pub relation_keys: Vec<(String, Vec>)>, - pub allow_suppressions: bool, } /// One model's SQL rule inputs. diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 6f6a815944..1d2ae016fc 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -33,7 +33,7 @@ pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARA pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; -pub(crate) const RULE_EXAMPLES: [(&str, &str); 25] = [ +pub(crate) const RULE_EXAMPLES: [(&str, &str); 28] = [ ( "SQBRSQL040", "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", @@ -62,10 +62,22 @@ pub(crate) const RULE_EXAMPLES: [(&str, &str); 25] = [ "SQBRGRAPH101", "Internal flow is stg -> int_clean -> int_enriched; marts are published interfaces.", ), + ( + "SQBRGRAPH103", + "sales__mart__orders and sales__mart__orders__finance both read sales__int_enriched__orders; sales__mart_v__orders may read sales__mart__orders.", + ), ( "SQBRPROJECT101", "domain__int_clean__entity or domain__mart_v__entity.", ), + ( + "SQBRPROJECT107", + "sales publishes sales__mart__orders and sales__mart__customers; the finance variant lives in finance or sales intermediate.", + ), + ( + "SQBRSQL038", + "JOIN customers AS c ON o.customer_id = c.customer_id, or CROSS JOIN a provably one-row relation such as a one-row seed or an aggregate without GROUP BY.", + ), ( "SQBRPROJECT105", "A domain__int_enriched__entity model lives under intermediate/enriched.", diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs index 0c4b11601a..1cc8b483a7 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -5,3 +5,4 @@ pub mod render_skills_body; pub mod ruleset_fingerprint; pub mod skill_lock_name; pub mod sorted_catalogue_order; +pub mod string_tuple_repr; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs index 2a44c33eb3..9882a1a4d0 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs @@ -16,6 +16,7 @@ pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { "", "Rules enforce SQL model architecture and never modifies SQL files.", "Run `sqb rules run SQBR` before handing off model changes.", + "Fix findings in SQL. A deliberate deviation is a reviewed [[rules.rule_exceptions]] entry with a reason in sqlbuild_project.toml; inline `-- sqb: ignore` comments are errors.", "", "## Active Rules", ] @@ -48,8 +49,13 @@ pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { { body.extend([String::new(), "## Scoped Deviations".into(), String::new()]); for entry in &config.rule_exceptions { + let scope: String = match (&entry.relation, &entry.cte) { + (Some(relation), Some(cte)) => format!(" (relation `{relation}` in CTE `{cte}`)"), + (Some(relation), None) => format!(" (relation `{relation}`)"), + _ => String::new(), + }; body.push(format!( - "- Exception `{}` at `{}`: {}", + "- Exception `{}` at `{}`{scope}: {}", entry.rule, entry.path, entry.reason )); } diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs new file mode 100644 index 0000000000..b65087dcf0 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs @@ -0,0 +1,13 @@ +//! Python `repr` of a tuple of strings, as `rules show` prints a string-list option default. + +use crate::rule_catalogue::_helpers::python_repr::string_repr; + +/// `repr(tuple(items))`: `()`, `('a',)` or `('a', 'b')`. +#[must_use] +pub fn string_tuple_repr(items: &[&str]) -> String { + let parts: Vec = items.iter().map(|item| string_repr(item)).collect(); + match parts.as_slice() { + [only] => format!("({only},)"), + _ => format!("({})", parts.join(", ")), + } +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs index 2173f679a0..d1353c4aab 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs @@ -33,7 +33,9 @@ macro_rules! rule { | "SQBRPROJECT202" | "SQBRPROJECT203" | "SQBRPROJECT204" + | "SQBRPROJECT107" | "SQBRGRAPH101" + | "SQBRGRAPH103" | "SQBRTEST101" | "SQBRTEST102" | "SQBRTEST103" @@ -97,6 +99,22 @@ pub(crate) fn catalogue() -> Vec { "table dependencies must use __ref or __source", "Replace this qualified table with __ref(\"\") or __source(\"\") so it participates in the DAG.", ), + rule!( + "SQBRGRAPH103", + RuleScope::Project, + "graph", + "published-domain-isolation", + "published models must not read published models of their own domain", + "Move logic shared within a domain into its intermediate layer, or move a consumer-specific model into the consuming domain. A __mart_v__ view may read __mart__.", + ), + rule!( + "SQBRPROJECT107", + RuleScope::Project, + "repository", + "published-name-shape", + "each domain's published models must share one name shape", + "Name every published model of a domain either ____ or ______. Move consumer-specific variants into intermediate or the consuming domain, rename models into the family, or allowlist an intentional qualifier with a reason.", + ), rule!( "SQBRPROJECT101", RuleScope::Model, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index e2eb76d93f..d44680e3cf 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -1,10 +1,12 @@ use crate::configuration::main::setting_help::setting_help; use crate::configuration::main::toml_string_array::toml_string_array; -use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use crate::constants::{ DECLARATION_DOMAIN_COMPONENTS, ENFORCED_CONTRACT, REFERENCE_KIND, SOURCE_REFERENCE_KIND, VIEW_MATERIALIZATION, }; +use crate::constants::{ + MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, +}; use crate::models::{Declaration, EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; use crate::query_facts::main::dependency_texts::dependency_texts; use crate::query_facts::main::model_shape::model_shape; @@ -262,6 +264,12 @@ pub(crate) fn evaluate_project_rules(request: &ProjectEvaluationRequest<'_>) -> if let Some(rule) = request.selected.get("SQBRGRAPH101") { model_layers::internal_forward_refs(request, rule, &faults); } + if let Some(rule) = request.selected.get(PUBLISHED_READ_CODE) { + model_layers::same_domain_published_reads(request.request, rule, &faults); + } + if let Some(rule) = request.selected.get(PUBLISHED_SHAPE_CODE) { + model_layers::published_name_shapes(request.request, rule, &faults); + } if let Some(rule) = request.selected.get("SQBRDECLARATION201") { duplicate_enums(request.request, rule, &faults); } @@ -528,11 +536,11 @@ fn select_star( #[derive(Clone)] pub(super) struct NameParts { - domain: String, + pub(super) domain: String, pub(super) layer: String, - subject: String, - source: Option, - is_view: bool, + pub(super) subject: String, + pub(super) source: Option, + pub(super) is_view: bool, } pub(super) fn parse_name(name: &str) -> Option { @@ -540,17 +548,8 @@ pub(super) fn parse_name(name: &str) -> Option { if !(3..=4).contains(&parts.len()) { return None; } - let layers = [ - "stg", - "stg_v", - "int_clean", - "int_v", - "int_enriched", - "mart", - "mart_v", - ]; if !is_lowercase_identifier(parts[0]) - || !layers.contains(&parts[1]) + || !MODEL_NAME_LAYERS.contains(&parts[1]) || !is_lowercase_identifier(parts[2]) || parts .get(3) diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index ce66e8fcf6..8264cd1c62 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -1,15 +1,23 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; use crate::constants::{ - INTERMEDIATE_LAYER_DIRECTORY, MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, REFERENCE_KIND, + ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, INTERMEDIATE_LAYER_DIRECTORY, + MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; -use crate::models::{Fault, Model, RuleMetadata}; -use crate::rules::_helpers::evaluation::parse_name; +use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; +use crate::rules::_helpers::evaluation::{parse_name, path_fault}; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const NO_CANONICAL_LAYER_FOLDER: &str = "no canonical layer folder"; +/// Where an intermediate model that replaces a published one lives. +const INTERMEDIATE_FOLDER: &str = "intermediate/enriched"; +const ENTITY_SHAPE: &str = "____"; +const QUALIFIED_SHAPE: &str = "______"; +const MODEL_FILE_SUFFIX: &str = ".sql"; pub(crate) fn explicit_model_schema(model: &Model, rule: &RuleMetadata, faults: &FaultCollector) { if model @@ -204,3 +212,329 @@ fn model_fault( remediation: remediation.unwrap_or_else(|| rule.remediation.clone()), } } + +/// The published layers configured for `code`, or the defaults. +fn published_layers(config: &RulesConfig, code: &str) -> Vec { + match config + .rule_options + .get(code) + .and_then(|options| options.get(PUBLISHED_LAYERS_OPTION)) + .and_then(Value::as_array) + { + Some(layers) => layers + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + None => DEFAULT_PUBLISHED_LAYERS.map(str::to_owned).to_vec(), + } +} + +/// The parts of a `____[__]` model name. +struct PublishedName { + domain: String, + layer: String, + subject: String, + source: Option, + is_view: bool, +} + +fn published_name(name: &str) -> Option { + let parts = parse_name(name)?; + Some(PublishedName { + domain: parts.domain, + layer: parts.layer, + subject: parts.subject, + source: parts.source, + is_view: parts.is_view, + }) +} + +/// One published model and its parsed name. +struct Published<'a> { + model: &'a Model, + name: PublishedName, +} + +fn published<'a>(model: &'a Model, layers: &[String]) -> Option> { + let name: PublishedName = published_name(&model.name)?; + layers + .contains(&name.layer) + .then_some(Published { model, name }) +} + +/// Flag every read of a same-domain published model, except a view of its own entity. +pub(crate) fn same_domain_published_reads( + request: &EvaluateRequest, + rule: &RuleMetadata, + faults: &FaultCollector, +) { + let layers: Vec = published_layers(&request.config, &rule.code); + for model in &request.models { + let Some(consumer) = published(model, &layers) else { + continue; + }; + let mut reported: BTreeSet<&str> = BTreeSet::new(); + for reference in &model.references { + if reference.ref_kind != REFERENCE_KIND || !reported.insert(&reference.ref_name) { + continue; + } + let Some(producer) = published_name(&reference.ref_name) else { + continue; + }; + if producer.domain != consumer.name.domain + || !layers.contains(&producer.layer) + || is_view_of_own_entity(&consumer.name, &producer) + { + continue; + } + faults.push(path_fault( + rule, + &model.relative_path, + format!( + "{} reads {}; published models must not read published models of their own \ + domain", + model.name, reference.ref_name + ), + read_remediation(&consumer), + )); + } + } +} + +/// `X___v__[__]` reading `X____` or the same qualifier. +fn is_view_of_own_entity(consumer: &PublishedName, producer: &PublishedName) -> bool { + consumer.is_view + && !producer.is_view + && (producer.source.is_none() || producer.source == consumer.source) + && consumer.subject == producer.subject + && consumer.layer == format!("{}_v", producer.layer) +} + +fn read_remediation(consumer: &Published<'_>) -> String { + let domain: &str = &consumer.name.domain; + let mut remediation: String = format!( + "Move the shared logic into a {domain}__int_* model that both published models read, or \ + move this model into the domain that consumes it." + ); + if let Some(command) = intermediate_move(consumer) { + remediation.push_str(&format!(" To move it into intermediate: {command}")); + } + remediation +} + +/// `sqb mv` into the domain's intermediate layer, when the model lives in a `mart` folder. +fn intermediate_move(model: &Published<'_>) -> Option { + let layer: &str = if model.name.is_view { + "int_v" + } else { + "int_enriched" + }; + let mut name: String = format!("{}__{layer}__{}", model.name.domain, model.name.subject); + if let Some(qualifier) = &model.name.source { + name.push_str("__"); + name.push_str(qualifier); + } + let parts: Vec<&str> = model.model.relative_path.split('/').collect(); + let (_, folders) = parts.split_last()?; + let mart: usize = folders + .iter() + .position(|part| *part == MART_LAYER_DIRECTORY)?; + let mut target: Vec<&str> = folders[..mart].to_vec(); + target.push(INTERMEDIATE_FOLDER); + target.extend(&folders[mart + 1..]); + Some(format!( + "sqb mv {} {}/{name}{MODEL_FILE_SUFFIX}", + model.model.name, + target.join("/") + )) +} + +/// One literal qualifier that does not count towards a domain's name shape. +struct AllowedQualifier { + domain: Option, + qualifier: String, +} + +fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec { + let Some(entries) = config + .rule_options + .get(code) + .and_then(|options| options.get(ALLOWED_QUALIFIERS_OPTION)) + .and_then(Value::as_array) + else { + return Vec::new(); + }; + entries + .iter() + .filter_map(Value::as_object) + .filter_map(|entry| { + Some(AllowedQualifier { + domain: entry + .get("domain") + .and_then(Value::as_str) + .map(str::to_owned), + qualifier: entry.get("qualifier")?.as_str()?.to_owned(), + }) + }) + .collect() +} + +fn allowlisted(allowed: &[AllowedQualifier], name: &PublishedName) -> bool { + name.source.as_ref().is_some_and(|qualifier| { + allowed.iter().any(|entry| { + entry.qualifier == *qualifier + && entry + .domain + .as_ref() + .is_none_or(|domain| *domain == name.domain) + }) + }) +} + +/// Flag published models whose name shape differs from their domain's majority shape. +pub(crate) fn published_name_shapes( + request: &EvaluateRequest, + rule: &RuleMetadata, + faults: &FaultCollector, +) { + let layers: Vec = published_layers(&request.config, &rule.code); + let allowed: Vec = allowed_qualifiers(&request.config, &rule.code); + let mut domains: BTreeMap>> = BTreeMap::new(); + for model in &request.models { + let Some(entry) = published(model, &layers) else { + continue; + }; + if allowlisted(&allowed, &entry.name) { + continue; + } + domains + .entry(entry.name.domain.clone()) + .or_default() + .push(entry); + } + for (domain, models) in &domains { + let qualified: usize = models + .iter() + .filter(|entry| entry.name.source.is_some()) + .count(); + let entities: usize = models.len() - qualified; + if qualified == 0 || entities == 0 { + continue; + } + let majority_qualified: bool = qualified > entities; + let shape = DomainShape { + domain, + entities, + qualified, + family: models + .iter() + .filter(|entry| entry.name.source.is_some() == majority_qualified) + .collect(), + }; + for entry in models + .iter() + .filter(|entry| entry.name.source.is_some() != majority_qualified) + { + faults.push(shape_fault(rule, entry, &shape)); + } + } +} + +/// One domain's published name shapes: how many models use each, and the majority family. +struct DomainShape<'a> { + domain: &'a str, + entities: usize, + qualified: usize, + family: Vec<&'a Published<'a>>, +} + +fn shape_fault(rule: &RuleMetadata, entry: &Published<'_>, shape: &DomainShape<'_>) -> Fault { + let majority: &str = if entry.name.source.is_some() { + ENTITY_SHAPE + } else { + QUALIFIED_SHAPE + }; + let message: String = format!( + "domain {} publishes {} {} as {ENTITY_SHAPE} and {} as {QUALIFIED_SHAPE}; {} does not \ + fit the majority shape {majority}", + shape.domain, + shape.entities, + if shape.entities == 1 { + "model" + } else { + "models" + }, + shape.qualified, + entry.model.name + ); + path_fault( + rule, + &entry.model.relative_path, + message, + shape_remediation(rule, entry, &shape.family), + ) +} + +fn shape_remediation( + rule: &RuleMetadata, + entry: &Published<'_>, + family: &[&Published<'_>], +) -> String { + let domain: &str = &entry.name.domain; + let mut steps: Vec = Vec::new(); + match &entry.name.source { + Some(qualifier) => { + let mut adapt: String = format!( + "If {} adapts another published model, move that logic into {domain}__int_* or \ + into the consuming domain", + entry.model.name + ); + if let Some(command) = intermediate_move(entry) { + adapt.push_str(&format!(" ({command})")); + } + steps.push(adapt); + let mut split: String = format!( + "If {domain} really is published once per qualifier, rename its models without \ + one into the family" + ); + if let Some(model) = family.first() { + split.push_str(&format!( + " (for example: sqb rename {} {}__core)", + model.model.name, model.model.name + )); + } + steps.push(split); + steps.push(allowlist_step(rule, domain, qualifier)); + } + None => { + steps.push(format!( + "If the models with a qualifier adapt this one, move their logic into \ + {domain}__int_* or into the consuming domain" + )); + steps.push(format!( + "If {domain} really is published once per qualifier, rename this model into the \ + family (for example: sqb rename {} {}__core)", + entry.model.name, entry.model.name + )); + if let Some(qualifier) = family.first().and_then(|model| model.name.source.as_ref()) { + steps.push(allowlist_step(rule, domain, qualifier)); + } + } + } + steps + .iter() + .enumerate() + .map(|(index, step)| format!("{}. {step}.", index + 1)) + .collect::>() + .join(" ") +} + +fn allowlist_step(rule: &RuleMetadata, domain: &str, qualifier: &str) -> String { + format!( + "If {qualifier} is an intentional, structural qualifier, allowlist it with a reason in \ + {PROJECT_CONFIG_FILENAME}: [rules.rule_options.{}] {ALLOWED_QUALIFIERS_OPTION} = \ + [{{ domain = \"{domain}\", qualifier = \"{qualifier}\", reason = \"\" }}]", + rule.code + ) +} diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs index f2ad474e0c..aa1069b965 100644 --- a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -1,5 +1,8 @@ //! `builtin_options`: the options each built-in rule declares. +use crate::constants::{ + ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, PUBLISHED_LAYERS_OPTION, +}; use crate::rules::models::{BuiltinOption, BuiltinOptionKind}; const fn boolean(name: &'static str, description: &'static str) -> BuiltinOption { @@ -10,6 +13,14 @@ const fn boolean(name: &'static str, description: &'static str) -> BuiltinOption } } +const fn published_layers() -> BuiltinOption { + BuiltinOption { + kind: BuiltinOptionKind::StringList(&DEFAULT_PUBLISHED_LAYERS), + name: PUBLISHED_LAYERS_OPTION, + description: "Model name layers that form each domain's published interface.", + } +} + /// The declared options of the built-in rule `code`, in declaration order. #[must_use] pub fn builtin_options(code: &str) -> Vec { @@ -42,8 +53,18 @@ pub fn builtin_options(code: &str) -> Vec { "Accept string or variant encoded date values.", ), ], + "SQBRGRAPH103" => vec![published_layers()], + "SQBRPROJECT107" => vec![ + published_layers(), + BuiltinOption { + kind: BuiltinOptionKind::TableList, + name: ALLOWED_QUALIFIERS_OPTION, + description: "Literal qualifiers that do not count towards a domain's name \ + shape, as { qualifier, reason } tables with an optional domain.", + }, + ], "SQBRTEST203" => vec![BuiltinOption { - kind: BuiltinOptionKind::StringList, + kind: BuiltinOptionKind::StringList(&[]), name: "allowed_tests", description: "Reviewed SQL test names that may assert only that empty inputs produce \ no rows; listed tests count toward min_tests_per_model.", diff --git a/crates/sqlbuild-rules/src/rules/models.rs b/crates/sqlbuild-rules/src/rules/models.rs index 0158b2f7e5..cc882b4954 100644 --- a/crates/sqlbuild-rules/src/rules/models.rs +++ b/crates/sqlbuild-rules/src/rules/models.rs @@ -76,8 +76,10 @@ impl FaultCollector { pub enum BuiltinOptionKind { /// A boolean defaulting to false. Boolean, - /// A list of strings defaulting to empty. - StringList, + /// A list of strings with the given default. + StringList(&'static [&'static str]), + /// A list of tables, each checked by the rule's own configuration validation. + TableList, } /// One option a built-in rule declares. diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs index 8e851b00bf..567376e688 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs @@ -11,12 +11,13 @@ use polyglot_sql::{ use crate::sql_lint::constants::LINT_API_VERSION; use crate::sql_lint::models::{ - AdditionalFactOptions, LintDiagnostic, LintEdit, LintRequest, LintResponse, LintRuleMetadata, - QueryFacts, + AdditionalFactOptions, CrossJoinRequest, LintDiagnostic, LintEdit, LintRequest, LintResponse, + LintRuleMetadata, QueryFacts, }; use crate::sql_lint::_helpers::additional::collect_additional_facts; use crate::sql_lint::_helpers::inline_relations::collect_inline_query_relation_spans; +use crate::sql_lint::_helpers::join_predicates::cross_join_diagnostics; use crate::sql_lint::_helpers::terminal_shape::collect_terminal_shape_facts; const NULL_COMPARISON: LintRuleMetadata = LintRuleMetadata { @@ -192,7 +193,7 @@ const NESTED_ELSE_CASE: LintRuleMetadata = LintRuleMetadata { const UNUSED_JOINED_RELATION: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL032", message: "Joined relation contributes no referenced values", - remediation: "Use the joined relation, remove the join, or suppress this intentional cardinality filter.", + remediation: "Use the joined relation or remove the join. When the join intentionally filters rows, record it as a reviewed [[rules.rule_exceptions]] entry with a reason in sqlbuild_project.toml.", }; const COMMENT_ATTACHMENT: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL033", @@ -222,7 +223,7 @@ const RECURSIVE_CTE: LintRuleMetadata = LintRuleMetadata { const CROSS_JOIN: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL038", message: "CROSS JOIN creates an explicit cartesian product", - remediation: "Use a keyed JOIN, or suppress this rule with a reason when the cartesian product is intentional.", + remediation: "Use a keyed JOIN, or make one side provably at most one row (LIMIT 1, an aggregate without GROUP BY, or a one-row seed). When the cartesian product is intentional, record it as a reviewed [[rules.rule_exceptions]] entry with rule, path, relation and reason in sqlbuild_project.toml.", }; const INLINE_QUERY_RELATION: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL039", @@ -420,6 +421,16 @@ pub(crate) fn lint(request: LintRequest) -> Result { enabled: &enabled, }; let mut diagnostics = diagnostics(&context); + if enabled.contains(CROSS_JOIN.code) { + let cross_joins = CrossJoinRequest { + statements: &statements, + tokens: &tokens, + cross_spans: &facts.cross_joins, + relation_keys: &request.relation_keys, + relation_names: &request.relation_names, + }; + diagnostics.extend(cross_join_diagnostics(&cross_joins, &CROSS_JOIN)); + } let depths = token_depths(&tokens); if enabled.contains(JOIN_EXPRESSION.code) { diagnostics.extend(diagnostics_for_spans( @@ -1713,13 +1724,6 @@ fn rule_migration_diagnostics(context: &DiagnosticContext<'_>) -> Vec bool { _ => false, } } + +/// A `rule` diagnostic for every cross join that can multiply rows, naming its relation and CTE. +pub(super) fn cross_join_diagnostics( + request: &CrossJoinRequest<'_>, + rule: &LintRuleMetadata, +) -> Vec { + let one_row_relations: HashSet = request + .relation_keys + .iter() + .filter(|(_, keys)| keys.iter().any(Vec::is_empty)) + .map(|(name, _)| name.to_ascii_lowercase()) + .collect(); + let relation_names: HashMap = request + .relation_names + .iter() + .map(|(sentinel, name)| (sentinel.to_ascii_lowercase(), name.clone())) + .collect(); + let inputs = OneRowInputs { + one_row_relations: &one_row_relations, + relation_names: &relation_names, + }; + multiplying_cross_joins( + request.statements, + request.tokens, + request.cross_spans, + &inputs, + ) + .into_iter() + .map(|finding| LintDiagnostic { + code: rule.code, + message: finding + .relation + .as_deref() + .map_or(Cow::Borrowed(CROSS_JOIN_UNNAMED_MESSAGE), |relation| { + Cow::Owned(cross_join_message(relation, finding.cte.as_deref())) + }), + remediation: if finding.relation.is_some() { + Cow::Borrowed(rule.remediation) + } else { + Cow::Owned(file_wide_remediation(rule.code)) + }, + start: finding.span.start, + end: finding.span.end, + fix_unavailable_reason: Some(CROSS_JOIN_FIX_REASON), + fix: None, + }) + .collect() +} + +/// Help for a join no relation names: fix it, or record the file-wide exception it allows. +fn file_wide_remediation(code: &str) -> String { + let entry: String = [ + "[[rules.rule_exceptions]]".to_owned(), + format!("rule = \"{code}\""), + format!("path = \"{CROSS_JOIN_PATH_PLACEHOLDER}\""), + format!("reason = \"{CROSS_JOIN_REASON_PLACEHOLDER}\""), + ] + .iter() + .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) + .collect(); + format!( + "Use a keyed JOIN, or make one side provably at most one row. This join reads no named \ + relation, so a relation-level exception cannot cover it; when the cartesian product is \ + intentional, record a reviewed file-wide exception in sqlbuild_project.toml:{entry}" + ) +} + +/// The finding message naming the joined relation and, inside a CTE, that CTE. +fn cross_join_message(relation: &str, cte: Option<&str>) -> String { + match cte { + Some(cte) => { + format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_CTE_MARKER}{cte}{CROSS_JOIN_SUFFIX}") + } + None => format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_SUFFIX}"), + } +} + +/// One explicit cross join found in the query tree. +struct CrossJoinTarget { + /// Character offsets where the joined relation may start in the token stream. + anchors: Vec, + /// Whether both joined sides may hold more than one row. + multiplies: bool, + /// The joined relation's name as authored, with dependency sentinels resolved. + relation: Option, + /// The top-level CTE whose query contains the join. + cte: Option, +} + +/// The cross join reported at one `CROSS` keyword. +struct CrossJoinFinding { + span: Span, + relation: Option, + cte: Option, +} + +/// What the analysis knows about relations named in the query. +struct OneRowInputs<'a> { + /// Lowercased dependency sentinels whose relation holds at most one row. + one_row_relations: &'a HashSet, + /// Lowercased dependency sentinels mapped to the relation names they stand for. + relation_names: &'a HashMap, +} + +struct Analysis<'a> { + inputs: &'a OneRowInputs<'a>, + ctes: HashMap, + targets: Vec, +} + +/// Cross joins at `cross_spans` that can multiply rows, each with its relation and CTE. +fn multiplying_cross_joins( + statements: &[Expression], + tokens: &[Token], + cross_spans: &[Span], + inputs: &OneRowInputs<'_>, +) -> Vec { + if cross_spans.is_empty() { + return Vec::new(); + } + let mut analysis = Analysis { + inputs, + ctes: HashMap::new(), + targets: Vec::new(), + }; + for statement in statements { + analysis.record_ctes(statement); + } + for statement in statements { + analysis.visit(statement, None, true); + } + let significant: Vec = (0..tokens.len()) + .filter(|&index| { + !matches!( + tokens[index].token_type, + TokenType::Space + | TokenType::Break + | TokenType::LineComment + | TokenType::BlockComment + ) + }) + .collect(); + let mut findings: Vec = Vec::new(); + for span in cross_spans { + let candidates: Vec = relation_anchor_candidates(tokens, &significant, *span); + let target: Option<&CrossJoinTarget> = analysis + .targets + .iter() + .find(|target| anchored_at(target, &candidates)); + match target { + Some(target) if !target.multiplies => {} + Some(target) => findings.push(CrossJoinFinding { + span: *span, + relation: target.relation.clone(), + cte: target.cte.clone(), + }), + None => findings.push(CrossJoinFinding { + span: *span, + relation: None, + cte: None, + }), + } + } + findings +} + +impl Analysis<'_> { + /// Record whether every CTE holds at most one row, in definition order. + fn record_ctes(&mut self, expression: &Expression) { + for node in expression.dfs() { + let Some(with) = with_clause(node) else { + continue; + }; + for cte in &with.ctes { + let one_row: bool = !with.recursive && self.at_most_one_row(&cte.this); + let name: String = cte.alias.name.to_ascii_lowercase(); + let known: bool = self.ctes.get(&name).copied().unwrap_or(one_row); + self.ctes.insert(name, known && one_row); + } + } + } + + /// Record joins under `expression`, labelled with the top-level CTE containing them. + fn visit(&mut self, expression: &Expression, cte: Option<&str>, top: bool) { + let mut bodies: Vec<&Expression> = Vec::new(); + if let Some(with) = with_clause(expression) { + for definition in &with.ctes { + bodies.push(&definition.this); + let label: Option<&str> = if top { + Some(&definition.alias.name) + } else { + cte + }; + self.visit(&definition.this, label, false); + } + } + if let Expression::Select(select) = expression { + self.record_joins(select, cte); + } + let branches_are_top: bool = top && is_set_operation(expression); + for child in expression.children() { + if bodies.iter().any(|body| std::ptr::eq(*body, child)) { + continue; + } + self.visit(child, cte, branches_are_top); + } + } + + fn record_joins(&mut self, select: &Select, cte: Option<&str>) { + let single_from: Option<&Expression> = select + .from + .as_ref() + .filter(|from| from.expressions.len() == 1) + .and_then(|from| from.expressions.first()); + for (index, join) in select.joins.iter().enumerate() { + if join.kind != JoinKind::Cross { + continue; + } + let left_one_row: bool = + index == 0 && single_from.is_some_and(|from| self.relation_one_row(from)); + let multiplies: bool = !(left_one_row || self.relation_one_row(&join.this)); + self.targets.push(CrossJoinTarget { + anchors: relation_anchors(&join.this), + multiplies, + relation: self.relation_name(&join.this), + cte: cte.map(str::to_owned), + }); + } + } + + fn relation_name(&self, relation: &Expression) -> Option { + match relation { + Expression::Table(table) => Some(self.table_name(table)), + Expression::Subquery(subquery) => { + subquery.alias.as_ref().map(|alias| alias.name.clone()) + } + Expression::Alias(alias) => Some(alias.alias.name.clone()), + _ => None, + } + } + + fn table_name(&self, table: &TableRef) -> String { + if table.schema.is_none() && table.catalog.is_none() { + let lowered: String = table.name.name.to_ascii_lowercase(); + if let Some(name) = self.inputs.relation_names.get(&lowered) { + return name.clone(); + } + } + [&table.catalog, &table.schema] + .into_iter() + .flatten() + .map(|part| part.name.as_str()) + .chain([table.name.name.as_str()]) + .collect::>() + .join(".") + } + + /// Whether a query expression returns at most one row. + fn at_most_one_row(&self, expression: &Expression) -> bool { + match expression { + Expression::Select(select) => self.select_one_row(select), + Expression::Subquery(subquery) => self.subquery_one_row(subquery), + Expression::Paren(paren) => self.at_most_one_row(&paren.this), + Expression::Values(values) => values.expressions.len() <= 1, + _ => false, + } + } + + fn subquery_one_row(&self, subquery: &Subquery) -> bool { + !subquery.lateral + && (subquery + .limit + .as_ref() + .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)) + || self.at_most_one_row(&subquery.this)) + } + + fn select_one_row(&self, select: &Select) -> bool { + if select.limit_by.is_some() || select.connect.is_some() || !select.lateral_views.is_empty() + { + return false; + } + if limited_to_one_row(select) { + return true; + } + if select.expressions.iter().any(generates_rows) { + return false; + } + if select.group_by.is_none() && select.expressions.iter().any(aggregates) { + return true; + } + if select + .qualify + .as_ref() + .is_some_and(|qualify| first_unpartitioned_row(&qualify.this, select)) + { + return true; + } + if select.group_by.is_some() || !select.joins.is_empty() { + return false; + } + match &select.from { + None => true, + Some(from) => match from.expressions.as_slice() { + [relation] => self.relation_one_row(relation), + _ => false, + }, + } + } + + /// Whether a relation in FROM or JOIN holds at most one row. + fn relation_one_row(&self, relation: &Expression) -> bool { + match relation { + Expression::Table(table) => { + if table.schema.is_some() + || table.catalog.is_some() + || table.identifier_func.is_some() + || table.changes.is_some() + { + return false; + } + let name: String = table.name.name.to_ascii_lowercase(); + match self.ctes.get(&name) { + Some(one_row) => *one_row, + None => self.inputs.one_row_relations.contains(&name), + } + } + Expression::Subquery(subquery) => self.subquery_one_row(subquery), + Expression::Paren(paren) => self.relation_one_row(&paren.this), + Expression::Values(values) => values.expressions.len() <= 1, + _ => false, + } + } +} + +/// The WITH clause a query expression owns. +fn with_clause(expression: &Expression) -> Option<&With> { + match expression { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(union) => union.with.as_ref(), + Expression::Intersect(intersect) => intersect.with.as_ref(), + Expression::Except(except) => except.with.as_ref(), + _ => None, + } +} + +fn is_set_operation(expression: &Expression) -> bool { + matches!( + expression, + Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_) + ) +} + +fn limited_to_one_row(select: &Select) -> bool { + let limited: bool = select + .limit + .as_ref() + .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)); + let fetched: bool = select.fetch.as_ref().is_some_and(|fetch| { + !fetch.percent && !fetch.with_ties && fetch.count.as_ref().is_some_and(at_most_one) + }); + let top: bool = select + .top + .as_ref() + .is_some_and(|top| !top.percent && !top.with_ties && at_most_one(&top.this)); + limited || fetched || top +} + +fn at_most_one(expression: &Expression) -> bool { + match expression { + Expression::Literal(literal) => match literal.as_ref() { + Literal::Number(text) => text.parse::().is_ok_and(|count| count <= 1), + _ => false, + }, + Expression::Paren(paren) => at_most_one(&paren.this), + _ => false, + } +} + +/// Whether a projected expression can return several rows per input row. +fn generates_rows(expression: &Expression) -> bool { + expression.dfs().any(|node| match node { + Expression::Unnest(_) + | Expression::Explode(_) + | Expression::ExplodeOuter(_) + | Expression::GenerateSeries(_) => true, + Expression::Function(function) => is_row_generator(&function.name), + _ => false, + }) +} + +fn is_row_generator(name: &str) -> bool { + let name: &str = name.rsplit('.').next().unwrap_or_default(); + ROW_GENERATOR_FUNCTIONS + .iter() + .any(|generator| generator.eq_ignore_ascii_case(name)) +} + +fn anchored_at(target: &CrossJoinTarget, candidates: &[usize]) -> bool { + target + .anchors + .iter() + .any(|anchor| candidates.contains(anchor)) +} + +/// Whether an expression aggregates outside any window function or subquery. +fn aggregates(expression: &Expression) -> bool { + if polyglot_sql::traversal::is_aggregate(expression) { + return true; + } + match expression { + Expression::WindowFunction(_) | Expression::Subquery(_) | Expression::Select(_) => false, + _ => expression.children().into_iter().any(aggregates), + } +} + +/// `QUALIFY ROW_NUMBER() OVER (ORDER BY ...) = 1` without a partition, directly or by alias. +fn first_unpartitioned_row(condition: &Expression, select: &Select) -> bool { + match condition { + Expression::And(operation) => { + first_unpartitioned_row(&operation.left, select) + || first_unpartitioned_row(&operation.right, select) + } + Expression::Paren(paren) => first_unpartitioned_row(&paren.this, select), + Expression::Eq(operation) => { + let compared: &Expression = match (&operation.left, &operation.right) { + (other, Expression::Literal(literal)) | (Expression::Literal(literal), other) + if matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) => + { + other + } + _ => return false, + }; + match compared { + Expression::WindowFunction(_) => unpartitioned_row_number(compared), + Expression::Column(column) if column.table.is_none() => select + .expressions + .iter() + .any(|projection| match projection { + Expression::Alias(alias) => { + alias.alias.name.eq_ignore_ascii_case(&column.name.name) + && unpartitioned_row_number(&alias.this) + } + _ => false, + }), + _ => false, + } + } + _ => false, + } +} + +fn unpartitioned_row_number(expression: &Expression) -> bool { + matches!( + expression, + Expression::WindowFunction(window) + if matches!(window.this, Expression::RowNumber(_)) + && window.over.window_name.is_none() + && window.over.partition_by.is_empty() + ) +} + +/// Where a joined relation can start: its first name part, or a derived table's alias. +fn relation_anchors(relation: &Expression) -> Vec { + match relation { + Expression::Table(table) => { + let mut anchors: Vec = Vec::new(); + for identifier in [&table.catalog, &table.schema].into_iter().flatten() { + anchors.extend(identifier.span.map(|span| span.start)); + } + anchors.extend(table.name.span.map(|span| span.start)); + anchors.extend(table.span.map(|span| span.start)); + anchors + } + Expression::Subquery(subquery) => subquery + .alias + .as_ref() + .and_then(|alias| alias.span) + .map(|span| span.start) + .into_iter() + .collect(), + Expression::Alias(alias) => alias + .alias + .span + .map(|span| span.start) + .into_iter() + .collect(), + _ => Vec::new(), + } +} + +/// Where the relation after `CROSS JOIN` at `cross` starts, and a derived table's alias. +fn relation_anchor_candidates(tokens: &[Token], significant: &[usize], cross: Span) -> Vec { + let Some(position) = significant + .iter() + .position(|&index| tokens[index].span.start == cross.start) + else { + return Vec::new(); + }; + let Some(&first) = significant.get(position + 2) else { + return Vec::new(); + }; + let mut candidates: Vec = vec![tokens[first].span.start]; + if tokens[first].token_type == TokenType::LParen { + let mut depth: usize = 0; + let mut cursor: usize = position + 2; + while let Some(&index) = significant.get(cursor) { + match tokens[index].token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth = depth.saturating_sub(1); + if depth == 0 { + break; + } + } + _ => {} + } + cursor += 1; + } + let mut alias: usize = cursor + 1; + if significant + .get(alias) + .is_some_and(|&index| tokens[index].token_type == TokenType::As) + { + alias += 1; + } + if let Some(&index) = significant.get(alias) { + candidates.push(tokens[index].span.start); + } + } + candidates +} diff --git a/crates/sqlbuild-rules/src/sql_lint/constants.rs b/crates/sqlbuild-rules/src/sql_lint/constants.rs index ebb457fe57..8bec6a6bd0 100644 --- a/crates/sqlbuild-rules/src/sql_lint/constants.rs +++ b/crates/sqlbuild-rules/src/sql_lint/constants.rs @@ -50,3 +50,12 @@ pub(crate) const RELATION_MODIFIER_KEYWORDS: [&str; 43] = [ "WHERE", "WINDOW", ]; +/// The parts of a `SQBRSQL038` message that names its joined relation and CTE. +pub(crate) const CROSS_JOIN_PREFIX: &str = "CROSS JOIN with `"; +pub(crate) const CROSS_JOIN_CTE_MARKER: &str = "` in CTE `"; +pub(crate) const CROSS_JOIN_SUFFIX: &str = "` creates an explicit cartesian product"; +/// The `SQBRSQL038` message for a join whose relation cannot be named. +pub(crate) const CROSS_JOIN_UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; +/// Stands for the finding's project-relative path until the caller that knows it fills it in. +pub(crate) const CROSS_JOIN_PATH_PLACEHOLDER: &str = ""; +pub(crate) const CROSS_JOIN_REASON_PLACEHOLDER: &str = ""; diff --git a/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs b/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs new file mode 100644 index 0000000000..c9a03220dd --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs @@ -0,0 +1,15 @@ +//! The relation and CTE a `SQBRSQL038` finding names, for relation-level Rule exceptions. + +use crate::sql_lint::constants::{CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PREFIX, CROSS_JOIN_SUFFIX}; + +/// The `(relation, cte)` a cross join finding message names, `None` for the generic message. +#[must_use] +pub fn cross_join_subject(message: &str) -> Option<(String, Option)> { + let named: &str = message + .strip_prefix(CROSS_JOIN_PREFIX)? + .strip_suffix(CROSS_JOIN_SUFFIX)?; + match named.split_once(CROSS_JOIN_CTE_MARKER) { + Some((relation, cte)) => Some((relation.to_owned(), Some(cte.to_owned()))), + None => Some((named.to_owned(), None)), + } +} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs b/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs new file mode 100644 index 0000000000..d5a65801ce --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs @@ -0,0 +1,9 @@ +//! `fill_finding_path`: a SQL finding's help with its file's path written into the entry it prints. + +use crate::sql_lint::constants::CROSS_JOIN_PATH_PLACEHOLDER; + +/// `remediation` with the path placeholder of a printed exception entry replaced by `path`. +#[must_use] +pub fn fill_finding_path(remediation: &str, path: &str) -> String { + remediation.replace(CROSS_JOIN_PATH_PLACEHOLDER, path) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs index f14a92656f..6fdc141f76 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs @@ -2,7 +2,9 @@ pub mod backtick_identifiers; pub mod batch_engine; pub mod batch_formatter; pub mod catalogue; +pub mod cross_join_subject; pub mod engine; +pub mod fill_finding_path; pub mod format_requests; pub mod formatter; pub mod preparation; diff --git a/crates/sqlbuild-rules/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs index 9371ecd79c..ea4fbc58b2 100644 --- a/crates/sqlbuild-rules/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules/src/sql_lint/models.rs @@ -1,7 +1,8 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; -use polyglot_sql::tokens::Span; +use polyglot_sql::Expression; +use polyglot_sql::tokens::{Span, Token}; use serde::{Deserialize, Serialize}; #[derive(Debug, Default)] @@ -80,6 +81,8 @@ pub(crate) struct LintRequest { pub max_ranking_order_by: Option, #[serde(default)] pub relation_keys: HashMap>>, + #[serde(default)] + pub relation_names: HashMap, } #[derive(Debug)] @@ -100,7 +103,7 @@ pub(crate) struct LintResponse { #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub(crate) struct LintDiagnostic { pub code: &'static str, - pub message: &'static str, + pub message: Cow<'static, str>, pub remediation: Cow<'static, str>, pub start: usize, pub end: usize, @@ -147,3 +150,15 @@ pub(crate) struct FormatResponse { #[serde(skip_serializing_if = "Option::is_none")] pub reason: Option, } + +/// One lint request's explicit cross joins, with what is known about the relations they read. +#[derive(Debug)] +pub(crate) struct CrossJoinRequest<'a> { + pub(crate) statements: &'a [Expression], + pub(crate) tokens: &'a [Token], + pub(crate) cross_spans: &'a [Span], + /// Declared keys by dependency sentinel; the empty key proves at most one row. + pub(crate) relation_keys: &'a HashMap>>, + /// The relation name each dependency sentinel stands for. + pub(crate) relation_names: &'a HashMap, +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests.rs b/crates/sqlbuild-rules/src/sql_lint/tests.rs index c46d727733..5e5d936598 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests.rs @@ -4,6 +4,8 @@ mod authored_tokens; mod batch_lint; #[path = "tests/test_comment_formatter.rs"] mod comment_formatter; +#[path = "tests/test_cross_joins.rs"] +mod cross_joins; #[path = "tests/test_empty_fixture_lint.rs"] mod empty_fixture_lint; #[path = "tests/test_format_corpus.rs"] diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs index 57aabf1ed2..830abb4d14 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs @@ -1,16 +1,20 @@ use std::str::FromStr; use polyglot_sql::{Dialect, DialectType}; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; use crate::sql_lint::main::batch_formatter::format_batch_json; use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::main::formatter::format_json; +use crate::sql_lint::tests::test_types::CrossJoinTestCase; use sqlbuild_analysis::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; use sqlbuild_analysis::sql_tokens::main::builtin_function_names::builtin_function_names; use sqlbuild_analysis::sql_tokens::main::canonical_tokens::canonical_tokens; use sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint; +/// The dependency sentinel the cross join cases name `order_policy`. +pub(crate) const POLICY_SENTINEL: &str = "__sqb_lint_7__"; + pub(crate) fn nested_function_sql(depth: usize) -> String { format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) } @@ -253,3 +257,40 @@ fn is_tight_call(name: &str, sql: &str, arguments: &str) -> bool { .to_ascii_uppercase() .contains(&format!("{} (", name.to_ascii_uppercase())) } + +/// The `SQBRSQL038` messages for one cross join case. +pub(crate) fn cross_join_messages(test_case: &CrossJoinTestCase) -> Result, String> { + cross_join_texts(test_case, "message") +} + +/// The `SQBRSQL038` remediations for one cross join case. +pub(crate) fn cross_join_remediations( + test_case: &CrossJoinTestCase, +) -> Result, String> { + cross_join_texts(test_case, "remediation") +} + +fn cross_join_texts(test_case: &CrossJoinTestCase, field: &str) -> Result, String> { + let keys: Map = test_case + .one_row_relations + .iter() + .map(|name| ((*name).to_owned(), json!([[]]))) + .collect(); + let request = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": test_case.dialect, + "enabled_rules": ["SQBRSQL038"], + "relation_keys": keys, + "relation_names": { POLICY_SENTINEL: "order_policy" }, + }); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response["diagnostics"] + .as_array() + .cloned() + .unwrap_or_default() + .iter() + .map(|diagnostic| diagnostic[field].as_str().unwrap_or_default().to_owned()) + .collect()) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs new file mode 100644 index 0000000000..2817d87427 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs @@ -0,0 +1,286 @@ +use crate::sql_lint::main::cross_join_subject::cross_join_subject; +use crate::sql_lint::tests::helpers::{ + POLICY_SENTINEL, cross_join_messages, cross_join_remediations, +}; +use crate::sql_lint::tests::test_types::{CrossJoinSubjectTestCase, CrossJoinTestCase}; + +const GENERIC_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product"; +const UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; + +#[test] +fn given_cross_joins_when_one_side_is_provably_one_row_then_only_multiplying_joins_are_reported() +-> Result<(), String> { + let test_cases = [ + CrossJoinTestCase { + description: "two unknown relations multiply rows", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a one-row seed on the right", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqb_lint_7__ AS p", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "a seed with several rows names its relation", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqb_lint_7__ AS p", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a one-row seed on the left of the first join", + sql: "SELECT o.order_id FROM __sqb_lint_7__ AS p CROSS JOIN orders AS o", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "an aggregate without GROUP BY carried through a filtering CTE", + sql: "WITH totals AS (SELECT COUNT(*) AS order_count FROM orders), \ + kept AS (SELECT order_count FROM totals WHERE order_count > 0) \ + SELECT o.order_id, k.order_count FROM orders AS o CROSS JOIN kept AS k", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "an aggregate with GROUP BY can return many rows", + sql: "WITH totals AS (SELECT customer_id, COUNT(*) AS order_count FROM orders GROUP BY customer_id) \ + SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a windowed count is not an aggregate", + sql: "WITH totals AS (SELECT COUNT(*) OVER () AS order_count FROM orders) \ + SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "LIMIT 1", + sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "LIMIT 2 can return two rows", + sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 2) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `latest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "one-row VALUES", + sql: "WITH settings AS (SELECT * FROM (VALUES (1, 'EUR')) AS v(version, currency)) \ + SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "two-row VALUES", + sql: "WITH settings AS (SELECT * FROM (VALUES (1), (2)) AS v(version)) \ + SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `settings` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "QUALIFY ROW_NUMBER() = 1 without a partition", + sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (ORDER BY order_id DESC) = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "snowflake", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "QUALIFY ROW_NUMBER() = 1 with a partition keeps one row per partition", + sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY order_id DESC) = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "snowflake", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a join inside the carrying CTE stops the proof", + sql: "WITH totals AS (SELECT COUNT(*) AS n FROM orders), \ + joined AS (SELECT t.n FROM totals AS t INNER JOIN customers AS c ON c.n = t.n) \ + SELECT o.order_id FROM orders AS o CROSS JOIN joined AS j", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `joined` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "UNNEST in a carrying CTE stops the proof", + sql: "WITH totals AS (SELECT LIST(order_id) AS ids FROM orders), \ + expanded AS (SELECT UNNEST(ids) AS order_id FROM totals) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN expanded AS e", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `expanded` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a set operation stops the proof", + sql: "WITH both_totals AS (SELECT COUNT(*) AS n FROM orders UNION ALL SELECT COUNT(*) AS n FROM customers) \ + SELECT o.order_id FROM orders AS o CROSS JOIN both_totals AS b", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `both_totals` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "the finding names the CTE that contains the join", + sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ + SELECT order_id FROM checks", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a later join still multiplies the rows before it", + sql: "SELECT o.order_id FROM __sqb_lint_7__ AS p CROSS JOIN orders AS o CROSS JOIN customers AS c", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "an unnamed relation says only a file-wide exception covers it", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[UNNAMED_MESSAGE], + }, + CrossJoinTestCase { + description: "a named window is never treated as unpartitioned", + sql: "WITH newest AS (SELECT order_id FROM orders WINDOW w AS (PARTITION BY customer_id ORDER BY order_id DESC) QUALIFY ROW_NUMBER() OVER w = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a join under a nested WITH names the top-level CTE", + sql: "WITH outer_checks AS (WITH inner_checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) SELECT order_id FROM inner_checks) \ + SELECT order_id FROM outer_checks", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `outer_checks` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a WITH inside a subquery of the final select names no CTE", + sql: "SELECT s.order_id FROM (WITH inner_checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) SELECT order_id FROM inner_checks) AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a WITH owned by a set operation names its CTE", + sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ + SELECT order_id FROM checks UNION ALL SELECT order_id FROM orders", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + ]; + + for test_case in test_cases { + let messages: Vec = cross_join_messages(&test_case)?; + + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_unnamed_cross_join_when_linting_then_help_prints_a_file_wide_exception() +-> Result<(), String> { + let test_cases = [CrossJoinTestCase { + description: "CROSS JOIN UNNEST", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "[[rules.rule_exceptions]]\n rule = \"SQBRSQL038\"\n path = \"\"\n reason = \"\"", + ], + }]; + + for test_case in test_cases { + let remediations: Vec = cross_join_remediations(&test_case)?; + + assert_eq!(remediations.len(), 1, "{}", test_case.description); + assert!( + remediations[0].contains(test_case.expected_messages[0]), + "{}: {remediations:?}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_cross_join_messages_when_reading_their_subject_then_relation_and_cte_are_named() { + let test_cases = [ + CrossJoinSubjectTestCase { + description: "a top-level join", + message: "CROSS JOIN with `order_policy` creates an explicit cartesian product", + expected_subject: Some(("order_policy", None)), + }, + CrossJoinSubjectTestCase { + description: "a join inside a CTE", + message: "CROSS JOIN with `order_policy` in CTE `order_checks` creates an explicit cartesian product", + expected_subject: Some(("order_policy", Some("order_checks"))), + }, + CrossJoinSubjectTestCase { + description: "the generic message names nothing", + message: GENERIC_MESSAGE, + expected_subject: None, + }, + ]; + + for test_case in test_cases { + let subject = cross_join_subject(test_case.message); + + assert_eq!( + subject + .as_ref() + .map(|(relation, cte)| (relation.as_str(), cte.as_deref())), + test_case.expected_subject, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs index 0235c87b20..d036ebdecc 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs @@ -150,3 +150,17 @@ pub(crate) struct FunctionDepthFailureTestCase { pub expected_code: &'static str, pub expected_limit: &'static str, } + +pub(crate) struct CrossJoinTestCase { + pub description: &'static str, + pub sql: &'static str, + pub dialect: &'static str, + pub one_row_relations: &'static [&'static str], + pub expected_messages: &'static [&'static str], +} + +pub(crate) struct CrossJoinSubjectTestCase { + pub description: &'static str, + pub message: &'static str, + pub expected_subject: Option<(&'static str, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs index c60f04a74a..c029f0779c 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs @@ -421,7 +421,7 @@ fn unused_outputs( }); found.push(LintDiagnostic { code: UNUSED_CTE_OUTPUT.code, - message: UNUSED_CTE_OUTPUT.message, + message: Cow::Borrowed(UNUSED_CTE_OUTPUT.message), remediation: Cow::Borrowed(if select.distinct { DISTINCT_UNUSED_REMEDIATION } else if select diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs index 38228dad45..6e95858613 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs @@ -95,7 +95,7 @@ fn scope_diagnostics( let (start, end) = function_span(tokens, second); found.push(LintDiagnostic { code: REPEATED_JSON_PARSE.code, - message: REPEATED_JSON_PARSE.message, + message: Cow::Borrowed(REPEATED_JSON_PARSE.message), remediation: Cow::Owned(remediation(second, group.len(), quote)), start, end, diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs index 667db34102..d052eb7085 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs @@ -23,7 +23,7 @@ pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { let length: usize = token.text.chars().count(); (length > request.max_literal_length).then(|| LintDiagnostic { code: LONG_LITERAL.code, - message: LONG_LITERAL.message, + message: Cow::Borrowed(LONG_LITERAL.message), remediation: Cow::Owned(remediation(length, request.max_literal_length)), start: token.span.start, end: token.span.end, diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs index eceee81cfa..fb4e463d6a 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs @@ -84,7 +84,7 @@ pub(crate) fn diagnostics( if exceeds { found.push(LintDiagnostic { code: RANKING_SORT_CAP.code, - message: RANKING_SORT_CAP.message, + message: Cow::Borrowed(RANKING_SORT_CAP.message), remediation: Cow::Owned(sort_cap_remediation( order.len(), request.max_ranking_order_by, @@ -98,7 +98,7 @@ pub(crate) fn diagnostics( if unproven { found.push(LintDiagnostic { code: UNSTABLE_ROW_NUMBER_CODE, - message: UNPROVEN_RANKING_MESSAGE, + message: Cow::Borrowed(UNPROVEN_RANKING_MESSAGE), remediation: Cow::Borrowed(UNPROVEN_RANKING_REMEDIATION), start, end, diff --git a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs b/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs index 8527f39658..f99921268b 100644 --- a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs +++ b/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs @@ -4,7 +4,7 @@ use crate::sql_quality::constants::{RELATION_KIND_REF, RELATION_KIND_SEED}; use crate::sql_quality::declared_keys::{non_null_unique_columns, trusted_unique_key}; use crate::sql_quality::models::DeclaredRelation; -/// Keys that prove one row per key, by `kind:name` identity; seeds are also keyed as refs. +/// Keys proving one row per key by `kind:name`; seeds are also refs, one-row relations key `[]`. pub fn declared_relation_keys( relations: &[DeclaredRelation], ) -> BTreeMap>> { @@ -14,6 +14,9 @@ pub fn declared_relation_keys( for column in non_null_unique_columns(relation) { found.push(vec![column]); } + if relation.at_most_one_row { + found.push(Vec::new()); + } if found.is_empty() { continue; } diff --git a/crates/sqlbuild-rules/src/sql_quality/models.rs b/crates/sqlbuild-rules/src/sql_quality/models.rs index d57f1aee24..7b91480b64 100644 --- a/crates/sqlbuild-rules/src/sql_quality/models.rs +++ b/crates/sqlbuild-rules/src/sql_quality/models.rs @@ -32,6 +32,8 @@ pub struct DeclaredRelation { pub columns: Vec, /// Table-level audits as `(definition name, its string column argument)`. pub audits: Vec<(String, Option)>, + /// Whether the relation is known to hold at most one row, which the empty key proves. + pub at_most_one_row: bool, } /// One declared column: its name, its audit definition names and whether it is declared non-null. diff --git a/fensu.toml b/fensu.toml index d99ff59940..dad712116c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1602,6 +1602,12 @@ symbols = [ ] reason = "Old-name steps re-read facts and physical state per move so a retry never repeats a completed step, and compatibility views are re-created in dependency order, so a chained view must read the columns of the view it selects from after that view is re-created. Each view's existence is checked right before it is redefined, and a model has only its own short chain of old names." +[[targets.python.rule_exceptions]] +rule = "FFS110" +path = "src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" +symbols = ["invoke_lifecycle_event_sink"] +reason = "The delivery thread claims each call in the dict it shares with the native dispatcher; the claim is the mutation, atomic under the GIL, so a cancelled call is never both run and counted dropped." + [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/old_name_views/_helpers/janitor.py" @@ -1628,7 +1634,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=25", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1640,7 +1646,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", @@ -1662,7 +1668,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs=6", @@ -1823,7 +1829,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", @@ -1858,12 +1864,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=29", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", @@ -1875,7 +1881,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs=17", @@ -2331,9 +2337,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", diff --git a/python-boundary.toml b/python-boundary.toml index 8fbb3795d5..532ec56b91 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -295,7 +295,7 @@ note = "Rule 3 facade: reads the decorator's definition off a user function; re- [[stays]] path = "runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" rule = 2 -note = "Rule 2 shim: the daemon delivery thread's target that calls one user lifecycle sink and reports how it ended to the native dispatcher, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." +note = "Rule 2 shim: target of each lifecycle sink's long-lived daemon delivery thread; claims each queued call in a shared dict (atomic under the GIL, so the dispatcher counts a cancellation as dropped only when the call can never run), calls the user sink in order and reports how each call ended through a Python queue, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." [[stays]] path = "runtime/event_exporting/main/_run_command_output_sinks.py" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index c659d00a80..0b5f43355a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -66,6 +66,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Overview](concepts/rules.md) (`concepts/rules`) - Turn repeated SQL and project review decisions into compiler-enforced requirements. - [Configuration and selection](concepts/rules/configuration-and-selection.md) (`concepts/rules/configuration-and-selection`) - Select built-in and custom Rules by exact code or family prefix. - [Findings and exceptions](concepts/rules/findings-and-exceptions.md) (`concepts/rules/findings-and-exceptions`) - Understand Rule diagnostics and record intentional exceptions safely. +- [Published-layer Rules](concepts/rules/published-layer-rules.md) (`concepts/rules/published-layer-rules`) - Keep each domain's published layer free of consumer-specific variants. - [Execution and caching](concepts/rules/execution-and-caching.md) (`concepts/rules/execution-and-caching`) - Understand Rule enforcement order, focused runs, and dependency-aware reuse. - [Testing](concepts/testing.md) (`concepts/testing`) - SQL unit tests and multi-model tests with macro support, assertions, and model chaining. - [Scenarios](concepts/scenarios.md) (`concepts/scenarios`) - End-to-end tests that build real project graphs against coherent fixture data. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md index 6b3b907b0d..c2dbb5f2fa 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md @@ -30,6 +30,8 @@ sqb --project-dir query --file my_query.sql File paths are resolved from the current working directory. +`sqb query` reads only `sqlbuild_project.toml` and `sqlbuild_local.toml` to connect. It does not parse models or import project Python, so it starts quickly in large projects and still runs when a model or Python file is broken. The SQL runs as written: SQLBuild does not resolve model names in it. + ## Flags | Flag | Description | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index ea93ec6ce1..39c68946f3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -132,10 +132,20 @@ contract rate. Use `sqb cost`, `sqb cost latest`, `sqb cost `, or provide a versioned, decimal-safe output contract. Pending detail records are refreshed from Snowflake when inspected again. +SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` for the whole run in one query when the run +finishes. Each read has a 10-second statement timeout; a window that reaches it, or that returns +more rows than one read allows, is split in half and read again. Collection at the end of a command +has a 30-second budget so the command exits promptly. If history cannot be read within it, the +build is unaffected and its cost is recorded as `pending`, with a message naming the timeout. +`sqb cost ` collects it later with a 5-minute budget, which completes long runs. Any other +failure is recorded as `collection_failed` with the Snowflake error that caused it. + These values are attributed compute credits and estimated cost, not Snowflake-billed credits or invoice reconciliation. The estimate uses only query history visible to the executing role and does not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60-second minimum, -cloud-services credits, contract adjustments, or multi-cluster billing. Run metadata and query IDs +cloud-services credits, contract adjustments, or multi-cluster billing. A statement run with a `QUERY_TAG` +of your own is not recognised as a metadata-only `CREATE`, so it may report missing warehouse +metadata. Run metadata and query IDs are stored under `target/executions//`; that statement ledger stores only an SQL digest, not SQL text. Executed SQL artifacts are stored separately under the sensitive `target/run/` tree. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md index aa0c9d1058..c66d46e273 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -87,7 +87,10 @@ min_severity = "warning" `shutdown_timeout` sets how long a command waits at exit for lifecycle sinks to drain queued events. It is a fixed duration from `"0s"` to `"10m"`, defaults to `"2s"`, and is accepted only under `[sinks.lifecycle]`, not per named sink. `"0s"` drops anything still queued at exit. The -setting does not change the one-second timeout for each sink call. +final quarter of the budget is reserved for run and invocation outcomes: when it begins, other +queued events are dropped and counted so the outcomes still arrive. The setting does not change the one-second timeout for each sink call. A sink call still running when +the budget ends gets up to that one second to return before providers are torn down; teardown +always runs on the command's own thread. ```toml [sinks.lifecycle] @@ -100,15 +103,21 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. ## Delivery behavior -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority. +- Each sink receives events in order on its own long-lived delivery thread. +- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; + `invocation_completed` and `invocation_failed` come after every other queued event. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. + Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, + `invocation_completed`, `invocation_failed`) are always admitted and never displaced. - A failing or timed-out sink is isolated from command correctness. - `audit_completed` facts are published as each audit finishes (model audits in `sqb build` when their model finishes), not in one burst at the end of the run. - When lifecycle events are dropped or fail, the command prints one warning to stderr with the counts and affected sinks, for example `Warning: command completed successfully, but lifecycle event export was incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). Increase sinks.lifecycle.shutdown_timeout or check sink health.` - The exit code and stdout, including `--json` output, are unchanged. + A successful command then ends with + `sqb audit completed successfully; only its lifecycle event export is affected.` The exit code + and stdout, including `--json` output, are unchanged. - Provider setup and declaration errors fail before execution because the project configuration is invalid. - Providers shared by multiple sinks are set up and torn down once per command. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b4a40fe677..c64f5cdca5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -186,6 +186,7 @@ A target is a named build context - the database and schema you build into, plus | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | | `warehouses` | Default Snowflake warehouse per command group, `build` and `query` (see below) | +| `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | ```toml [connections.warehouse] @@ -253,6 +254,33 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Target concurrency + +Targets often run on very different warehouses. A target can set its own worker limit, so local +and CI runs use many workers while production stays within what its warehouse runs without +queuing: + +```toml +[settings] +concurrency = 16 + +[targets.prod] +concurrency = 8 +``` + +`sqb build`, `sqb test`, `sqb audit`, `sqb seed`, `sqb load`, and `sqb scenario test` use the first +value set in this order: + +1. `--concurrency` on the command line, for that invocation only +2. The selected target's `concurrency`, from `[targets.]` in either file +3. `[settings] concurrency` in `sqlbuild_local.toml` +4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) + +The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build +--verbose` shows the effective limit and the setting it came from, for example +`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and +uses the active target's value. + ### Command-group warehouses Builds and ad-hoc inspection usually need different compute. A Snowflake target can choose a @@ -661,7 +689,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](models/configuration.md#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; `--concurrency` overrides it | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](python-nodes/loaders.md). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | @@ -713,11 +741,16 @@ select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See -[Compiler-integrated Rules](rules.md) for configuration, authoring, and suppressions. +[Compiler-integrated Rules](rules.md) for configuration, authoring, and exceptions. + +| Key | Description | +| --- | --- | +| `ignore` | Rule codes or families turned off for the whole project. | +| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQBRSQL038`. Stale entries fail compilation. See [Findings and exceptions](rules/findings-and-exceptions.md). | +| `[[rules.rule_ignores]]` | Rules turned off for `paths` or `selectors`, with a `reason`. | -| Key | Default | Description | -| --- | --- | --- | -| `allow_exceptions` | `true` | When `false`, `[[rules.rule_exceptions]]` and `[[rules.rule_ignores]]` entries are configuration errors and inline `-- sqb: ignore` directives are `SQBRSQL000` errors. The project-wide `ignore` list stays allowed. See [Forbidding exceptions](rules/findings-and-exceptions.md#forbidding-exceptions). | +Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSQL000` errors +that print the entry to add, and the former `allow_exceptions` setting is a configuration error. ## Scopes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index a512a2610d..b788e472c2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -209,10 +209,10 @@ need to execute SQL against controlled data. Use an audit when the answer depend A finding contains a stable code, project-relative path, line, column, explanation, and remediation. This makes the same requirement usable in a terminal, JSON output, CI annotation, or agent workflow. -Intentional departures remain explicit. Exact exceptions and path- or resource-scoped ignores -require a reason; exact exceptions are stale-checked so obsolete suppressions do not silently -accumulate. Mandatory compiler correctness cannot be suppressed. A project can forbid exceptions, -scoped ignores and inline suppressions entirely with `[rules] allow_exceptions = false`. +Intentional departures remain explicit and live only in `sqlbuild_project.toml`. Exact and +relation-level exceptions and path- or resource-scoped ignores require a reason; exceptions are +stale-checked so obsolete entries do not silently accumulate. There are no inline suppression +comments, and mandatory compiler correctness cannot be excepted at all. See [Findings and exceptions](rules/findings-and-exceptions.md) for configuration examples. @@ -234,5 +234,6 @@ and [Execution and caching](rules/execution-and-caching.md) for the Rules lifecy Select exact Rules or families and configure project-owned options. Understand diagnostics, exact exceptions, path ignores, and stale checks. + Keep each domain's published layer free of consumer-specific variants. Learn evaluation order, focused execution, and dependency-aware reuse. List, inspect, run, and generate guidance for configured Rules. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 5f3cc3c8ea..6a835eb92f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -71,6 +71,10 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE `min_tests_per_model`. Entries that name no existing test are reported as stale. See [Empty-input tests](../testing.md#empty-input-tests). +`SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts +`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +[Published-layer Rules](published-layer-rules.md). + Unknown Rule codes, option names, and invalid option values fail configuration. ## Focused selection diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md index b833e7b2a6..07110f3292 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md @@ -6,9 +6,23 @@ Online: https://sqlbuild.com/docs/concepts/rules/findings-and-exceptions/ +## Contents + +- Exact exceptions +- Relation-level exceptions +- Path- and resource-scoped ignores +- No inline suppressions +- Removed settings +- Mandatory correctness +- Machine-readable findings + A Rule finding identifies an exact code, project-relative path, line, column, message, and remediation. Error findings make compilation fail and prevent artifact completion. +Every deliberate deviation from a selected Rule lives in `sqlbuild_project.toml`, never in SQL +comments. A change to the project file is something a reviewer stops to look at; a comment in a +large SQL diff is not. + ## Exact exceptions Use an exact exception for one known finding. Exceptions require a reason and are stale-checked: @@ -23,6 +37,38 @@ reason = "This example intentionally demonstrates one sampled row." An exception refers to one exact Rule code and path. If the finding disappears, SQLBuild reports the stale exception so obsolete configuration does not accumulate silently. +## Relation-level exceptions + +A file-wide exception for a deliberate cartesian product would also hide any accidental cross join +added to the same file later. `SQBRSQL038` findings name the joined relation and, inside a CTE, +that CTE, so an exception can cover exactly one join: + +```toml +[[rules.rule_exceptions]] +rule = "SQBRSQL038" +path = "models/orders/order_checks.sql" +relation = "order_policy" +cte = "policy_checks" # optional: only the join inside this top-level CTE +reason = "Every order is evaluated against every policy check." +``` + +`relation` is the joined relation as written, with `__ref`, `__seed` and `__source` calls resolved +to the referenced name. Without `cte`, the exception covers that relation anywhere in the file. +Relation-level exceptions are stale-checked like exact exceptions: an entry that matches no finding +fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be +combined with relation-level exceptions for the same code and path. + +A join that reads no named relation, such as `CROSS JOIN UNNEST(...)` or `CROSS JOIN LATERAL`, has +no relation to name. Its finding says so and prints the file-wide exception (rule and path) that is +the only way to cover it. + +`SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most +intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a +declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an +aggregate without `GROUP BY`, a one-row `VALUES`, `LIMIT 1`, or `QUALIFY ROW_NUMBER() OVER (...) = 1` +without a partition, carried through CTEs and subqueries that only filter or project. A join, +`UNNEST`/`FLATTEN`, set operation or `GROUP BY` in between stops the proof, and the join is reported. + ## Path- and resource-scoped ignores Use a path ignore when a documented project area intentionally follows a different convention: @@ -59,36 +105,35 @@ reason = "These intermediate SQL files intentionally preserve upstream columns." `paths` and `selectors` may be combined in one scoped ignore. Both forms require a reason. -## Inline suppressions - -Suppress one SQL Rule finding on the next SQL line with a reasoned comment: +To turn a Rule off for the whole project, add its code to `[rules] ignore`. -```sql --- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row -SELECT customer_id FROM customers LIMIT 1 -``` +## No inline suppressions -A directive without a reason, or one that no longer matches a finding, is reported as -`SQBRSQL000`. +SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` or `/* sqb: ignore ... */` +comment is a `SQBRSQL000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +exact entry to add instead: -## Forbidding exceptions +```text +error[SQBRSQL000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml + --> models/orders/order_checks.sql:4:1 + = help: Remove this comment, then fix the SQBRSQL038 finding, or record it as a reviewed exception in sqlbuild_project.toml: + [[rules.rule_exceptions]] + rule = "SQBRSQL038" + path = "models/orders/order_checks.sql" + relation = "order_policy" + reason = "every order meets every policy" +``` -Set `allow_exceptions = false` to require that every selected Rule passes everywhere: +Copy the entry into `sqlbuild_project.toml`, adjust the reason for a reviewer, and delete the +comment. -```toml -[rules] -select = ["SQBRSQL", "SQBRMODEL"] -ignore = ["SQBRSQL004"] -allow_exceptions = false # default true -``` +## Removed settings -With exceptions forbidden, any `[[rules.rule_exceptions]]` or `[[rules.rule_ignores]]` entry fails -configuration loading with an error that counts the entries and shows the setting to change, so -`sqb compile` and `sqb rules run` stop before evaluating Rules. Each inline `-- sqb: ignore` -directive becomes a `SQBRSQL000` error at the directive and suppresses nothing, so the finding it -was hiding is reported too. The project-wide `ignore` list stays allowed: it is the adoption switch -for a Rule across the whole project, not a scoped escape hatch. `[[rules.graph_edge_exceptions]]`, -`[[rules.select_star_allow]]` and `[[rules.threshold_overrides]]` are not affected. +`[rules] allow_exceptions` was removed. Configured exceptions are always allowed because the project +file is the review gate, and inline suppressions no longer exist. A project that still sets +`allow_exceptions` fails configuration loading with an error that explains the removal: delete the +line, keep reviewed deviations as `[[rules.rule_exceptions]]` entries, and use `[rules] ignore` to +turn a Rule off for the whole project. ## Mandatory correctness diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md new file mode 100644 index 0000000000..248639b4a9 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md @@ -0,0 +1,81 @@ + + +# Published-layer Rules + +> Keep each domain's published layer free of consumer-specific variants. + +Online: https://sqlbuild.com/docs/concepts/rules/published-layer-rules/ + +A domain's published layer is its interface to the rest of the project. Under pressure to make +something work, it is tempting to add a consumer-specific variant next to an existing published +model: + +```text +sales__int_clean__orders -> sales__mart__orders -> sales__mart__orders__finance -> finance__stg_v__orders +``` + +`sales__mart__orders__finance` reshapes a published model for one consumer in another domain. That +work belongs in the consumer's domain, or in the producing domain's intermediate layer. Two +built-in Rules catch it: `SQBRPROJECT107` by its name as soon as it is created, and `SQBRGRAPH103` +by its dependency even when the name looks uniform. Together they leave one fix: logic shared within +a domain belongs in its intermediate layer. + +Both Rules read model names of the form `____[__]`. Models whose +names do not follow that grammar are not checked. Both are errors and are selected with their +families (`SQBRGRAPH`, `SQBRPROJECT`) or `SQBR`. + +## SQBRGRAPH103: no published reads within a domain + +A model in a published layer must not read another published model of its own domain. The one +built-in allowance is a view of its own table: `__mart_v__[__]` may read +`__mart__`, or `__mart____` with the same qualifier. Reading another domain's published models is the point of publishing +them and is allowed. + +```text +error[SQBRGRAPH103]: sales__mart__orders__finance reads sales__mart__orders; published models must not read published models of their own domain + --> models/sales/mart/sales__mart__orders__finance.sql:1:1 + = help: Move the shared logic into a sales__int_* model that both published models read, or move this model into the domain that consumes it. To move it into intermediate: sqb mv sales__mart__orders__finance models/sales/intermediate/enriched/sales__int_enriched__orders__finance.sql +``` + +## SQBRPROJECT107: one name shape per domain + +Within one domain's published layer, every model name has the same number of segments: either all +`____` or all `______`. The models that do +not fit the domain's majority shape are reported, with the remediation: + +1. If a model with a qualifier adapts a model without one, move that logic into intermediate or into + the consumer's domain. +2. If the domain really is split by qualifier, rename the models without one into the family, for + example `sqb rename sales__mart__customers sales__mart__customers__core`. +3. If the qualifier is intentional and structural, allowlist it with a reason. + +Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left +out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain +only: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { qualifier = "csv_export", reason = "File export views of a published entity." }, + { domain = "sales", qualifier = "fra", reason = "Published once per country." }, +] +``` + +## Published layers + +Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule +to change that; each Rule reads its own option: + +```toml +[rules.rule_options.SQBRGRAPH103] +published_layers = ["mart", "mart_v"] + +[rules.rule_options.SQBRPROJECT107] +published_layers = ["mart", "mart_v"] +``` + +## Deviating + +There are no inline suppressions. Turn a Rule off for the whole project with `[rules] ignore`, or +record one reviewed model as a `[[rules.rule_exceptions]]` entry with its path and a reason. See +[Findings and exceptions](findings-and-exceptions.md). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md index 361ffc5e31..c1503e8ff4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md @@ -102,7 +102,7 @@ precedence; SQLBuild does not silently combine it with Dagster's implicit asset SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster asset checks: - **Unit tests** become checks attached to the models they test -- **Audits** become checks attached to the model or source they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN` +- **Audits** become checks attached to the model, source, or seed they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN`. A singular audit that runs at the end of the build becomes a check on each model it reads - **Scenarios** become checks attached to the models they exercise - **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 19bf31f222..b80648b9e4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3212,9 +3212,6 @@ class EventExporterDispatcher: class DispatcherWorker: def __call__(self) -> None: ... -class InvocationCompletion: - def __call__(self, error: BaseException | None, /) -> None: ... - def stop_event_exporter_threads() -> None: ... class EventExporterCommandScope: @@ -3966,10 +3963,10 @@ def format_lint_sql_bodies( def lint_native_sql_bodies( bodies: list[LintBody], contents_by_path: dict[Path, str], config: LintConfig ) -> list[LintViolation]: ... -def apply_lint_suppressions( +def reject_lint_suppressions( violations: list[LintViolation], contents_by_path: dict[Path, str], - allow_suppressions: bool, + project_dir: Path, ) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... def valid_rules_config(*, project_dir: Path) -> RulesConfig | None: ... @@ -7581,6 +7578,7 @@ def collect_snowflake_cost( attempts: int = 3, retry_delay_seconds: float = 1.0, result_limit: int = 10000, + budget_seconds: float | None = None, ) -> RunCostSummary: ... def snowflake_query_observations( *, @@ -7592,14 +7590,15 @@ def snowflake_query_observations( deadline: float, database: str | None = None, result_limit: int | None = None, + run_id: str | None = None, ) -> tuple[tuple[QueryCostObservation, ...], frozenset[str], dict[str, str], bool]: ... def render_snowflake_query_history_sql( *, started_at: datetime, - classification_query_ids: Iterable[str], database: str | None = None, completed_at: datetime | None = None, result_limit: int = 10000, + run_id: str | None = None, ) -> str: ... def snowflake_statement_is_metadata_only(*, query_type: object, query_text: object) -> bool: ... def parse_sqlbuild_query_tag(value: object) -> dict[str, str | None] | None: ... @@ -7875,6 +7874,7 @@ class DbtSqlbuildWorkContext(NativeRecord): use_color: bool snapshots_config: SnapshotsConfig execution_limits: ExecutionLimitsConfig + concurrency: int def __init__( self, plan_output: PlanOutput, @@ -7885,6 +7885,7 @@ class DbtSqlbuildWorkContext(NativeRecord): use_color: bool, snapshots_config: SnapshotsConfig, execution_limits: ExecutionLimitsConfig, + concurrency: int, ) -> None: ... class PlanSafetyGates(NativeRecord): @@ -10456,7 +10457,6 @@ class LintConfig(NativeRecord): header_rules_enabled: bool max_literal_length: int max_ranking_order_by: int - allow_suppressions: bool relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] def __init__( self, @@ -10469,7 +10469,6 @@ class LintConfig(NativeRecord): header_rules_enabled: bool = ..., max_literal_length: int = ..., max_ranking_order_by: int = ..., - allow_suppressions: bool = ..., relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] = ..., ) -> None: ... @@ -11899,6 +11898,7 @@ class CompilePipelineOptions(NativeRecord): resolve_python_run_selectors: bool no_cache: bool max_microbatches: int | None + concurrency: int | None selection_diagnostics: bool plan_sql_tests: bool record_migration_fingerprints: bool @@ -11923,6 +11923,7 @@ class CompilePipelineOptions(NativeRecord): resolve_python_run_selectors: bool = ..., no_cache: bool = ..., max_microbatches: int | None = ..., + concurrency: int | None = ..., selection_diagnostics: bool = ..., plan_sql_tests: bool = ..., record_migration_fingerprints: bool = ..., @@ -12188,6 +12189,7 @@ class PlannerOverrides(NativeRecord): forced_stale_model_names: tuple[str, ...] external_blocked_model_names: tuple[str, ...] max_microbatches: int | None + concurrency: int | None def __init__( self, cursor_overrides: CursorOverrides | None = ..., @@ -12196,6 +12198,7 @@ class PlannerOverrides(NativeRecord): forced_stale_model_names: tuple[str, ...] = ..., external_blocked_model_names: tuple[str, ...] = ..., max_microbatches: int | None = ..., + concurrency: int | None = ..., ) -> None: ... class DeferralInputs(NativeRecord): @@ -12572,11 +12575,15 @@ class RuleExemption(NativeRecord): rule: str path: str reason: str + relation: str | None + cte: str | None def __init__( self, rule: str, path: str, reason: str, + relation: str | None = ..., + cte: str | None = ..., ) -> None: ... class GraphEdgeExclusion(NativeRecord): @@ -13698,6 +13705,7 @@ class TargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig diff: TargetDiffConfig warehouses: TargetWarehousesConfig + concurrency: int | None def __init__( self, connection: dict[str, object] = ..., @@ -13720,6 +13728,7 @@ class TargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig = ..., diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., + concurrency: int | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -13745,6 +13754,7 @@ class LocalTargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig diff: TargetDiffConfig warehouses: TargetWarehousesConfig + concurrency: int | None def __init__( self, connection: dict[str, object] = ..., @@ -13768,6 +13778,7 @@ class LocalTargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig = ..., diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., + concurrency: int | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): @@ -17878,7 +17889,6 @@ class RulesConfig(NativeRecord): rule_exceptions: tuple[RuleExemption, ...] graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] rule_ignores: tuple[RuleIgnore, ...] - allow_exceptions: bool select_star_allow: tuple[SelectStarAllow, ...] domains: tuple[str, ...] approved_source_tokens: tuple[str, ...] @@ -17896,7 +17906,6 @@ class RulesConfig(NativeRecord): rule_exceptions: tuple[RuleExemption, ...] = ..., graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] = ..., rule_ignores: tuple[RuleIgnore, ...] = ..., - allow_exceptions: bool = ..., select_star_allow: tuple[SelectStarAllow, ...] = ..., domains: tuple[str, ...] = ..., approved_source_tokens: tuple[str, ...] = ..., diff --git a/src/sqlbuild/adapters/snowflake/_helpers/cost.py b/src/sqlbuild/adapters/snowflake/_helpers/cost.py new file mode 100644 index 0000000000..149fa8f000 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/cost.py @@ -0,0 +1,16 @@ +"""Describe Snowflake cost-collection failures.""" + +from __future__ import annotations + +_FAILURE_DETAIL_LIMIT: int = 200 + + +def cost_failure_cause(error: Exception) -> str: + """Name a cost-collection failure by type and its message on one bounded line.""" + + detail: str = " ".join(str(error).split()) + if not detail: + return type(error).__name__ + if len(detail) > _FAILURE_DETAIL_LIMIT: + detail = f"{detail[: _FAILURE_DETAIL_LIMIT - 3]}..." + return f"{type(error).__name__}: {detail}" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 1ea747e1c7..55e7ddfdc9 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -118,6 +118,7 @@ from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.snowflake._helpers.cost import cost_failure_cause from sqlbuild.adapters.snowflake._helpers.grants import ( render_snowflake_view_grants, render_snowflake_view_revokes, @@ -168,6 +169,8 @@ _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 +_COST_LOGIN_TIMEOUT_SECONDS: int = 5 +_COST_NETWORK_TIMEOUT_SECONDS: int = 15 _SHOW_COLUMN_KIND: str = "COLUMN" _SHOW_COLUMN_KINDS: frozenset[str] = frozenset({_SHOW_COLUMN_KIND, "VIRTUAL_COLUMN"}) _UNRESOLVED: object = object() @@ -355,10 +358,11 @@ def collect_run_cost( statement_ledger_path: Path, usd_per_credit: Decimal, rate_source: str, + budget_seconds: float | None = None, ) -> RunCostSummary: telemetry_connection_config: dict[str, object] = dict(connection_config) - telemetry_connection_config["login_timeout"] = 5 - telemetry_connection_config["network_timeout"] = 5 + telemetry_connection_config["login_timeout"] = _COST_LOGIN_TIMEOUT_SECONDS + telemetry_connection_config["network_timeout"] = _COST_NETWORK_TIMEOUT_SECONDS try: connection: Any = self.connect(telemetry_connection_config) except Exception as error: @@ -378,13 +382,14 @@ def collect_run_cost( statement_ledger_path=statement_ledger_path, usd_per_credit=usd_per_credit, rate_source=rate_source, + budget_seconds=budget_seconds, ) except Exception as error: summary = RunCostSummary( status=CostStatus.COLLECTION_FAILED, usd_per_credit=usd_per_credit, rate_source=rate_source, - message=f"Snowflake cost collection failed ({type(error).__name__}).", + message=f"Snowflake cost collection failed ({cost_failure_cause(error)}).", ) finally: try: diff --git a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py index 844e0e5382..b73ee3a6c0 100644 --- a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py +++ b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py @@ -1,22 +1,28 @@ -"""Call one lifecycle-event sink on its delivery thread.""" +"""Call one lifecycle-event sink on its long-lived delivery thread.""" from collections.abc import Callable, Mapping +from queue import SimpleQueue from sqlbuild.runtime.observability.models import LifecycleEvent def invoke_lifecycle_event_sink( *, - function: Callable[..., object], - provider_arguments: Mapping[str, object], - event: LifecycleEvent, - completion: Callable[[BaseException | None], None], + calls: SimpleQueue[ + tuple[Callable[..., object], Mapping[str, object], LifecycleEvent, int] | None + ], + results: SimpleQueue[tuple[int, BaseException | None]], + claims: dict[int, bool], ) -> None: - """Call a user sink with one event, then report how the call ended.""" + """Claim and call the sink with each queued event in order and report how each call ended.""" - try: - _ = function(event=event, **provider_arguments) - except BaseException as error: - _ = completion(error) - return - _ = completion(None) + for function, provider_arguments, event, token in iter(calls.get, None): + if not claims.setdefault(token, True): + del claims[token] + continue + try: + _ = function(event=event, **provider_arguments) + except BaseException as error: + results.put((token, error)) + else: + results.put((token, None)) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py index 309b8d22e1..f1f3f57f1b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py @@ -35,6 +35,7 @@ class SlowSinkDropE2ETestCase: description: str shutdown_timeout: str expected_accepted: int + expected_last_line: str @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py index 7c213b7fe7..1e7c4fc1c6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py @@ -111,7 +111,14 @@ def test_given_many_audits_when_running_then_audit_completed_events_arrive_durin @pytest.mark.parametrize( "test_case", - (SlowSinkDropE2ETestCase("zero drain drops queued events", "0s", 13),), + ( + SlowSinkDropE2ETestCase( + "zero drain drops queued events", + "0s", + 13, + "sqb audit completed successfully; only its lifecycle event export is affected.", + ), + ), ids=lambda case: case.description, ) def test_given_slow_sink_and_zero_drain_when_auditing_then_warns_with_counts_and_succeeds( @@ -161,6 +168,7 @@ def test_given_slow_sink_and_zero_drain_when_auditing_then_warns_with_counts_and assert test_case.expected_accepted - dropped - failed <= delivered_lines assert delivered_lines <= test_case.expected_accepted - dropped assert result.stderr.count("event export was incomplete") == 1 + assert result.stderr.splitlines()[-1] == test_case.expected_last_line @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 4067389ef9..19826f9db5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1372,3 +1372,19 @@ class MaterializationTypeChangeTestCase: description: str materializations: tuple[str, ...] expected_table_types: tuple[str, ...] + + +@dataclass(frozen=True) +class TargetConcurrencyE2ETestCase: + description: str + target_concurrency: str + local_settings: str + command: tuple[str, ...] + expected_fragment: str + + +@dataclass(frozen=True) +class TargetMicrobatchWarningE2ETestCase: + description: str + command: tuple[str, ...] + expected_warning_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 6a99667565..9e6c1db9e3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1151,3 +1151,85 @@ def order_totals_model_sql(*, materialized: str) -> str: f"MODEL (description 'Order totals.', materialized {materialized});\n\n" "SELECT 1 AS order_id, 250 AS total_cents\n" ) + + +def target_concurrency_project_files( + *, target_concurrency: str, local_settings: str +) -> dict[str, str]: + """A DuckDB project with `[settings] concurrency = 2` and a `ci` target with extra lines.""" + + return { + "sqlbuild_project.toml": ( + 'name = "target_concurrency"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[settings]\n" + "concurrency = 2\n\n" + "[targets.dev]\n" + 'schema = "main"\n\n' + "[targets.ci]\n" + 'schema = "main"\n' + f"{target_concurrency}" + ), + "sqlbuild_local.toml": local_settings, + "seeds/products.csv": "product_id\n1\n2\n", + "seeds/products.yml": ( + "seeds:\n" + " - name: products\n" + " description: Products.\n" + " columns:\n" + " - name: product_id\n" + " type: INTEGER\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\nSELECT 1 AS order_id\n" + ), + } + + +def target_concurrency_microbatch_files() -> dict[str, str]: + """A project whose `ci` target raises concurrency above `[settings]` for a microbatch model.""" + + return { + "sqlbuild_project.toml": ( + 'name = "target_microbatch"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[settings]\n" + "concurrency = 1\n" + "microbatch_concurrency = true\n\n" + "[targets.dev]\n" + 'schema = "main"\n\n' + "[targets.ci]\n" + 'schema = "main"\n' + "concurrency = 3\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_events\n" + " description: Raw events.\n" + " schema: main\n" + " table: raw_events\n" + ), + "models/orders.sql": ( + 'MODEL (description "Orders.",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n" + " cursor event_time,\n" + " cursor_type timestamp,\n" + " cursor_grain hour,\n" + " cursor_inputs (raw_events (column event_time, roles [filter, watermark])),\n" + " batch_size 1h,\n" + " batch_concurrency 3,\n" + ");\n\n" + 'SELECT id, event_time FROM __source("raw_events")\n' + "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py new file mode 100644 index 0000000000..871c5bbeca --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py @@ -0,0 +1,208 @@ +"""E2E coverage for per-target concurrency and its precedence.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import duckdb +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + TargetConcurrencyE2ETestCase, + TargetMicrobatchWarningE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + target_concurrency_microbatch_files, + target_concurrency_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, + table_exists, +) + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyE2ETestCase( + description="the selected target's concurrency replaces the project setting", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="3 configured limit (targets.ci.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="--concurrency overrides the target", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "build", "--verbose", "--target", "ci", "--concurrency", "1"), + expected_fragment="1 configured limit (--concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="the target overrides a local settings override", + target_concurrency="concurrency = 3\n", + local_settings="[settings]\nconcurrency = 5\n", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="3 configured limit (targets.ci.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="a target without concurrency keeps the local settings override", + target_concurrency="", + local_settings="[settings]\nconcurrency = 5\n", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="5 configured limit (sqlbuild_local.toml settings.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="a local target concurrency overrides the project target", + target_concurrency="concurrency = 3\n", + local_settings="[targets.ci]\nconcurrency = 4\n", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="4 configured limit (targets.ci.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="other targets keep the project setting", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "build", "--verbose"), + expected_fragment="2 configured limit (settings.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="seed uses the target's concurrency", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "seed", "--target", "ci"), + expected_fragment="concurrency: 3", + ), + ], + ids=lambda case: case.description, +) +def test_given_target_concurrency_when_running_then_documented_precedence_applies( + tmp_path: Path, + test_case: TargetConcurrencyE2ETestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONCURRENCY", raising=False) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="target_concurrency", + repo_files=target_concurrency_project_files( + target_concurrency=test_case.target_concurrency, + local_settings=test_case.local_settings, + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_fragment in result.stdout + result.stderr + assert table_exists(db_path=project_dir / "warehouse.duckdb", table_name="products") + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyE2ETestCase( + description="zero is rejected with a coded configuration error", + target_concurrency="concurrency = 0\n", + local_settings="", + command=("--no-color", "build", "--target", "ci"), + expected_fragment="targets.ci.concurrency must be an integer >= 1", + ), + TargetConcurrencyE2ETestCase( + description="a non-integer is rejected with a coded configuration error", + target_concurrency='concurrency = "8"\n', + local_settings="", + command=("--no-color", "build", "--target", "ci"), + expected_fragment="targets.ci.concurrency must be an integer >= 1", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_target_concurrency_when_running_then_config_error_is_coded( + tmp_path: Path, + test_case: TargetConcurrencyE2ETestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONCURRENCY", raising=False) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="target_concurrency", + repo_files=target_concurrency_project_files( + target_concurrency=test_case.target_concurrency, + local_settings=test_case.local_settings, + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + output: str = result.stdout + result.stderr + + assert result.returncode != 0 + assert "D001" in output + assert test_case.expected_fragment in output + assert not (project_dir / "warehouse.duckdb").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + TargetMicrobatchWarningE2ETestCase( + description="the target's concurrency enables the concurrent microbatch warning", + command=("--no-color", "build", "--target", "ci"), + expected_warning_count=1, + ), + TargetMicrobatchWarningE2ETestCase( + description="--concurrency 1 overrides the target and suppresses the warning", + command=("--no-color", "build", "--target", "ci", "--concurrency", "1"), + expected_warning_count=0, + ), + TargetMicrobatchWarningE2ETestCase( + description="a target without concurrency keeps the serial project setting", + command=("--no-color", "build", "--target", "dev"), + expected_warning_count=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_target_concurrency_when_planning_microbatches_then_warning_uses_it( + test_case: TargetMicrobatchWarningE2ETestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SQLBUILD_CONCURRENCY", raising=False) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="target_microbatch", + repo_files=target_concurrency_microbatch_files(), + ) + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / "warehouse.duckdb")) + connection.execute("CREATE TABLE raw_events (id INTEGER, event_time TIMESTAMP)") + connection.execute( + "INSERT INTO raw_events VALUES (1, '2026-01-01 00:30:00'), (2, '2026-01-01 01:30:00')" + ) + connection.close() + initial: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + connection = duckdb.connect(str(project_dir / "warehouse.duckdb")) + connection.execute( + "INSERT INTO raw_events VALUES (3, '2026-01-01 02:30:00'), (4, '2026-01-01 03:30:00')" + ) + connection.close() + + incremental: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = incremental.stdout + incremental.stderr + assert initial.returncode == 0, initial.stdout + initial.stderr + assert incremental.returncode == 0, output + assert ( + output.count("Concurrent microbatch execution is enabled") + == test_case.expected_warning_count + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py index dee4c8baf4..2ff3651c63 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py @@ -122,3 +122,11 @@ class DbtPlanSafetyE2ETestCase: command: tuple[str, ...] expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DbtTargetConcurrencyE2ETestCase: + description: str + target_settings: str + command: tuple[str, ...] + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py index c364a9b16d..ed4273cb4f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py @@ -12,10 +12,12 @@ DbtExecutionFailureCliTestCase, DbtExecutionQueryAssertion, DbtMissingOriginE2ETestCase, + DbtTargetConcurrencyE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.dbt.helpers import ( break_dbt_interop_fact_orders_model, declare_missing_origin_requiring_confirmation, + declare_sqlbuild_main_target, load_json_stdout, prepare_dbt_interop_project, skip_unless_dbt_is_runnable, @@ -579,3 +581,31 @@ def test_given_missing_origin_requiring_confirmation_when_running_dbt_then_refus assert test_case.expected_output in result.stdout + result.stderr assert "Pass --allow-missing-migration-origin" not in result.stdout + result.stderr assert not table_exists(db_path=project_dir / "dbt_interop.duckdb", table_name="local_only") + + +@pytest.mark.parametrize( + "test_case", + [ + DbtTargetConcurrencyE2ETestCase( + description="SQLBuild work in dbt interop uses the target's concurrency", + target_settings="concurrency = 3\n", + command=("--no-color", "dbt", "run", "--select", "local_only"), + expected_fragment="concurrency: 3", + ), + ], + ids=lambda case: case.description, +) +def test_given_target_concurrency_when_running_dbt_interop_then_sqlbuild_work_uses_it( + test_case: DbtTargetConcurrencyE2ETestCase, tmp_path: Path +) -> None: + skip_unless_dbt_is_runnable() + project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) + declare_sqlbuild_main_target(project_dir=project_dir, target_settings=test_case.target_settings) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + assert test_case.expected_fragment in output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py index 009abe88ae..a9430b328c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py @@ -124,3 +124,10 @@ class FilteredSinkFailureE2ETestCase: expected_runs_only: tuple[str, ...] expected_errors_only: tuple[str, ...] expected_warning: str + + +@dataclass(frozen=True) +class SlowSinkExitE2ETestCase: + description: str + command: tuple[str, ...] + expected_marker_entries: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index b676e69f0d..79d1d9a0ea 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -15,6 +15,7 @@ FilteredSinkFailureE2ETestCase, NoExporterCommandE2ETestCase, ReadOnlyProviderE2ETestCase, + SlowSinkExitE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -421,3 +422,115 @@ def failing(event: LifecycleEvent) -> None: assert errors_only == test_case.expected_errors_only assert test_case.expected_warning.format(accepted=accepted) in result.stderr assert "destination detail" not in result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + SlowSinkExitE2ETestCase( + description="sink call in flight when the shutdown budget ends", + command=("audit",), + expected_marker_entries=("teardown started", "teardown finished"), + ), + ), + ids=lambda case: case.description, +) +def test_given_sink_call_in_flight_at_shutdown_when_process_exits_then_teardown_completes_without_abort( + test_case: SlowSinkExitE2ETestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + marker_path: Path = tmp_path / "teardown.txt" + monkeypatch.setenv("TEARDOWN_MARKER_PATH", str(marker_path)) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="slow_sink_exit_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "slow_sink_exit_project" + adapter = "duckdb" + + [connection] + database = "slow_sink_exit_project.duckdb" + + [sinks.lifecycle] + shutdown_timeout = "0s" + """ + ).lstrip(), + "providers/destination.py": dedent( + """ + import os + import sys + import time + import types + from pathlib import Path + + from sqlbuild.providers import Provider + + + class _SlowInterpreterExit: + def __del__(self, _time=time): + _time.sleep(1.0) + + + _holder = types.ModuleType("slow_interpreter_exit") + _holder.finalizer = _SlowInterpreterExit() + sys.modules["slow_interpreter_exit"] = _holder + + + class Destination(Provider): + '''Destination whose publishes and flush are slow.''' + + def publish(self) -> None: + time.sleep(0.05) + + def teardown(self) -> None: + path = Path(os.environ["TEARDOWN_MARKER_PATH"]) + with path.open("a", encoding="utf-8") as stream: + stream.write("teardown started\\n") + time.sleep(0.5) + with path.open("a", encoding="utf-8") as stream: + stream.write("teardown finished\\n") + """ + ).lstrip(), + "sinks/publish.py": dedent( + """ + from providers.destination import Destination + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def publish_lifecycle(event: LifecycleEvent, destination: Destination) -> None: + del event + destination.publish() + """ + ).lstrip(), + "models/orders.sql": dedent( + """ + MODEL ( + description "Test model orders.", + materialized table, + columns (order_id (audits [not_null, unique])) + ); + + SELECT range AS order_id FROM range(10) + """ + ).lstrip(), + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + marker_path.unlink() + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert ( + tuple(marker_path.read_text(encoding="utf-8").splitlines()) + == test_case.expected_marker_entries + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py index 42d64cf8f6..6bfcfc0b91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py @@ -32,3 +32,33 @@ class QueryExecutionErrorCliTestCase: command: tuple[str, ...] expected_stderr_fragment: str expected_returncode: int = 2 + + +@dataclass(frozen=True) +class QueryUnrelatedProjectFaultTestCase: + description: str + repo_files: dict[str, str] + expected_rows: tuple[dict[str, int], ...] = ({"order_id": 7},) + + +@dataclass(frozen=True) +class QueryProjectDiscoveryTestCase: + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_project_discoveries: int + + +@dataclass(frozen=True) +class QueryTargetConnectionTestCase: + description: str + command: tuple[str, ...] + expected_stdout_fragment: str + + +@dataclass(frozen=True) +class QueryConnectionConfigErrorTestCase: + description: str + project_toml: str + expected_stderr_fragment: str + expected_returncode: int = 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py new file mode 100644 index 0000000000..dcac52db7c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py @@ -0,0 +1,243 @@ +"""`sqb query` reads only the project configuration it needs to connect.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.query._test_types import ( + QueryConnectionConfigErrorTestCase, + QueryProjectDiscoveryTestCase, + QueryTargetConnectionTestCase, + QueryUnrelatedProjectFaultTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, +) + +PROJECT_TOML: str = ( + 'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +VALID_MODEL_SQL: str = "MODEL (materialized table);\n\nSELECT 1 AS order_id\n" +TARGETS_PROJECT_TOML: str = dedent( + """ + name = "orders_targets" + adapter = "duckdb" + default_target = "dev" + + [connections.dev] + database = "dev.duckdb" + + [connections.prod] + database = "prod.duckdb" + + [targets.dev] + connection = "dev" + + [targets.prod] + connection = "prod" + """ +).lstrip() +STAGE_WORK_SCRIPT: str = """ +import contextlib, io, json, sys +import sqlbuild._native as native +from sqlbuild.cli.entry.main.entry import main + +_ = native.compile_stage_work_info(reset=True) +with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + code = main(json.loads(sys.argv[1])) +work = native.compile_stage_work_info() +print(json.dumps({"code": code, "discoveries": work.get("discover_project_inputs", 0)})) +""" + + +@pytest.mark.parametrize( + "test_case", + ( + QueryUnrelatedProjectFaultTestCase( + description="model without a MODEL header", + repo_files={"models/orders.sql": "SELECT FROM WHERE (\n"}, + ), + QueryUnrelatedProjectFaultTestCase( + description="project Python that fails to import", + repo_files={"python/broken.py": 'raise RuntimeError("broken project python")\n'}, + ), + QueryUnrelatedProjectFaultTestCase( + description="local SQL analysis opt-out the project forbids", + repo_files={ + "sqlbuild_project.toml": PROJECT_TOML + + "\n[settings]\nrequire_sql_analysis = true\n", + "sqlbuild_local.toml": "[settings]\nsql_analysis = false\n", + }, + ), + ), + ids=lambda case: case.description, +) +def test_given_unrelated_broken_project_input_when_running_ad_hoc_query_then_sql_runs( + tmp_path: Path, + test_case: QueryUnrelatedProjectFaultTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={"sqlbuild_project.toml": PROJECT_TOML, **test_case.repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("query", "SELECT 7 AS order_id", "--format", "json"), + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == list(test_case.expected_rows) + assert "Declaration parsing" not in result.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + QueryProjectDiscoveryTestCase( + description="ad hoc query discovers no project files", + command=("query", "SELECT 1 AS order_id"), + expected_exit_code=0, + expected_project_discoveries=0, + ), + QueryProjectDiscoveryTestCase( + description="dag still discovers the project", + command=("dag", "--json"), + expected_exit_code=0, + expected_project_discoveries=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_project_when_running_command_then_project_discovery_runs_only_when_needed( + tmp_path: Path, + test_case: QueryProjectDiscoveryTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={"sqlbuild_project.toml": PROJECT_TOML, "models/orders.sql": VALID_MODEL_SQL}, + ) + argv: list[str] = ["--project-dir", str(project_dir), *test_case.command] + + result: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", STAGE_WORK_SCRIPT, json.dumps(argv)], + check=True, + capture_output=True, + text=True, + cwd=tmp_path, + ) + outcome: dict[str, int] = json.loads(result.stdout.strip().splitlines()[-1]) + + assert outcome == { + "code": test_case.expected_exit_code, + "discoveries": test_case.expected_project_discoveries, + } + + +@pytest.mark.parametrize( + "test_case", + ( + QueryTargetConnectionTestCase( + description="default target connection", + command=("query", "SELECT label FROM destination", "--format", "csv"), + expected_stdout_fragment="label\ndev\n", + ), + QueryTargetConnectionTestCase( + description="explicit target connection", + command=( + "query", + "SELECT label FROM destination", + "--format", + "csv", + "--target", + "prod", + ), + expected_stdout_fragment="label\nprod\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_target_connections_when_running_query_then_selected_target_database_is_read( + tmp_path: Path, + test_case: QueryTargetConnectionTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_targets", + repo_files={"sqlbuild_project.toml": TARGETS_PROJECT_TOML}, + ) + for label in ("dev", "prod"): + execute_duckdb( + db_path=project_dir / f"{label}.duckdb", + sql=f"CREATE TABLE destination AS SELECT '{label}' AS label", + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == test_case.expected_stdout_fragment + + +@pytest.mark.parametrize( + "test_case", + ( + QueryConnectionConfigErrorTestCase( + description="target warehouse under an adapter without warehouses", + project_toml=TARGETS_PROJECT_TOML + '\n[targets.dev.warehouses]\nquery = "ORDERS_WH"\n', + expected_stderr_fragment=( + "[targets.dev.warehouses] selects a query warehouse, but the 'duckdb' adapter " + "has no warehouse to select" + ), + ), + QueryConnectionConfigErrorTestCase( + description="target selecting an unknown connection", + project_toml=TARGETS_PROJECT_TOML.replace( + 'connection = "dev"\n', 'connection = "missing"\n' + ), + expected_stderr_fragment="error[P001]: Unknown connection 'missing' selected by target.", + ), + QueryConnectionConfigErrorTestCase( + description="no connection configured", + project_toml='name = "orders_project"\nadapter = "duckdb"\n', + expected_stderr_fragment=( + "no target is selected and no top-level [connection] is configured" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_connection_configuration_when_running_query_then_reports_config_error( + tmp_path: Path, + test_case: QueryConnectionConfigErrorTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_config_error", + repo_files={"sqlbuild_project.toml": test_case.project_toml}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("query", "SELECT 1 AS order_id"), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_returncode + assert test_case.expected_stderr_fragment in result.stderr + assert "RECORD" not in result.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 40e2f72c12..4510bfa822 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -245,3 +245,37 @@ class OneModelCustomRuleEditCase: expected_warm_counts: tuple[int, int] expected_edit_counts: tuple[int, int] expected_edit_exit_code: int + + +@dataclass(frozen=True) +class OneRowCrossJoinCase: + """One model whose CROSS JOIN reads a relation that may hold at most one row.""" + + description: str + seed_rows: tuple[str, ...] + model_sql: str + expected_messages: tuple[str, ...] + expected_exit: int + + +@dataclass(frozen=True) +class RuleExceptionConfigCase: + """One `sqlbuild_project.toml` exception setup compiled against two cross joins.""" + + description: str + rules_toml: str + model_sql: str + expected_exit: int + expected_messages: tuple[str, ...] = () + expected_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PublishedLayerCase: + """One set of published models checked by the published-layer Rules.""" + + description: str + files: tuple[tuple[str, str], ...] + rules_toml: str + expected_findings: tuple[tuple[str, str], ...] + expected_exit: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 8c7a47a4be..5e480c12ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -626,3 +626,52 @@ def write_bulk_rules_cache(*, root: Path, file_name: str, text: str) -> None: _ = rules_bulk_cache_path(project_dir=root, file_name=file_name).write_text( text, encoding="utf-8" ) + + +def compile_diagnostics(result: subprocess.CompletedProcess[str]) -> list[dict[str, Any]]: + """Return every diagnostic of one `sqb compile --json` result.""" + + return list(json.loads(result.stdout)["diagnostics"]) + + +def diagnostic_messages(result: subprocess.CompletedProcess[str], code: str) -> tuple[str, ...]: + """Return the messages of one code's diagnostics, in report order.""" + + return tuple( + str(item["message"]) + for item in filter(lambda item: item["code"] == code, compile_diagnostics(result)) + ) + + +def diagnostic_text(result: subprocess.CompletedProcess[str]) -> str: + """Return every diagnostic message and help, followed by stderr.""" + + lines: list[str] = [ + f"{item['message']}\n{item.get('help', '')}" for item in compile_diagnostics(result) + ] + return "\n".join((*lines, result.stderr)) + + +def write_one_row_cross_join_project( + *, project_dir: Path, toml: str, seed_rows: tuple[str, ...], files: tuple[tuple[str, str], ...] +) -> Path: + """Write a project with the `order_policy` seed and model files; return the seed CSV.""" + + (project_dir / "sqlbuild_project.toml").write_text(toml, encoding="utf-8") + for relative, text in files: + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + seed: Path = project_dir / "seeds" / "order_policy.csv" + seed.write_text("max_amount\n" + "".join(f"{row}\n" for row in seed_rows), encoding="utf-8") + return seed + + +def write_model_files(*, project_dir: Path, toml: str, files: tuple[tuple[str, str], ...]) -> None: + """Write `sqlbuild_project.toml` and each `(relative path, text)` file.""" + + (project_dir / "sqlbuild_project.toml").write_text(toml, encoding="utf-8") + for relative, text in files: + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py new file mode 100644 index 0000000000..7b9d8ca95f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py @@ -0,0 +1,191 @@ +"""E2E coverage of SQBRSQL038 inferring that one side of a CROSS JOIN holds at most one row.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import OneRowCrossJoinCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + diagnostic_messages, + run_compile_cli, + write_one_row_cross_join_project, +) + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL038"]\n' +_SEED_YAML: str = ( + "seeds:\n" + " - name: order_policy\n" + " description: Order limits applied to every order.\n" + " columns:\n" + " - name: max_amount\n" + " type: INTEGER\n" +) +_RAW_ORDERS_SQL: str = "MODEL (description 'Raw orders.');\nSELECT 1 AS order_id, 3 AS amount\n" +_HEADER: str = "MODEL (description 'Orders checked against limits.');\n" +_DIRECT_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "final AS (\n" + " SELECT o.order_id, p.max_amount\n" + " FROM orders AS o\n" + ' CROSS JOIN __seed("order_policy") AS p\n' + ")\n" + "SELECT f.order_id, f.max_amount FROM final AS f\n" +) +_CARRIED_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "policy AS (\n" + ' SELECT p.max_amount FROM __seed("order_policy") AS p WHERE p.max_amount > 0\n' + "),\n" + "final AS (\n" + " SELECT o.order_id, p.max_amount FROM orders AS o CROSS JOIN policy AS p\n" + ")\n" + "SELECT f.order_id, f.max_amount FROM final AS f\n" +) +_AGGREGATE_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "totals AS (\n" + " SELECT SUM(o.amount) AS total_amount FROM orders AS o\n" + "),\n" + "final AS (\n" + " SELECT o.order_id, t.total_amount FROM orders AS o CROSS JOIN totals AS t\n" + ")\n" + "SELECT f.order_id, f.total_amount FROM final AS f\n" +) +_LIMIT_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "largest AS (\n" + " SELECT o.amount AS largest_amount FROM orders AS o\n" + " ORDER BY o.amount DESC, o.order_id ASC\n" + " LIMIT 1\n" + "),\n" + "final AS (\n" + " SELECT o.order_id, l.largest_amount FROM orders AS o CROSS JOIN largest AS l\n" + ")\n" + "SELECT f.order_id, f.largest_amount FROM final AS f\n" +) +_MULTIPLYING: str = ( + "CROSS JOIN with `order_policy` in CTE `final` creates an explicit cartesian product" +) + + +_SEED_YAML_FILE: tuple[str, str] = ("seeds/order_policy.yml", _SEED_YAML) +_RAW_ORDERS_FILE: tuple[str, str] = ("models/raw_orders.sql", _RAW_ORDERS_SQL) + + +@pytest.mark.parametrize( + "test_case", + [ + OneRowCrossJoinCase( + description="one-row seed joined directly", + seed_rows=("100",), + model_sql=_DIRECT_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="one-row seed carried through a filtering CTE", + seed_rows=("100",), + model_sql=_CARRIED_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="aggregate without GROUP BY", + seed_rows=("100", "200"), + model_sql=_AGGREGATE_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="LIMIT 1", + seed_rows=("100", "200"), + model_sql=_LIMIT_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="two-row seed multiplies rows", + seed_rows=("100", "200"), + model_sql=_DIRECT_SQL, + expected_messages=(_MULTIPLYING,), + expected_exit=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_cross_join_when_compiling_then_only_multiplying_joins_are_reported( + test_case: OneRowCrossJoinCase, + tmp_path: Path, +) -> None: + _ = write_one_row_cross_join_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + seed_rows=test_case.seed_rows, + files=( + _SEED_YAML_FILE, + _RAW_ORDERS_FILE, + ("models/checked_orders.sql", test_case.model_sql), + ), + ) + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + assert diagnostic_messages(result, "SQBRSQL038") == test_case.expected_messages, ( + result.stdout + result.stderr + ) + assert result.returncode == test_case.expected_exit, result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + OneRowCrossJoinCase( + description="adding a seed row brings the finding back", + seed_rows=("100",), + model_sql=_DIRECT_SQL, + expected_messages=(_MULTIPLYING,), + expected_exit=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_one_row_seed_when_a_row_is_added_then_the_cached_compile_reports_the_join( + test_case: OneRowCrossJoinCase, + tmp_path: Path, +) -> None: + seed: Path = write_one_row_cross_join_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + seed_rows=test_case.seed_rows, + files=( + _SEED_YAML_FILE, + _RAW_ORDERS_FILE, + ("models/checked_orders.sql", test_case.model_sql), + ), + ) + first: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + seed.write_text(seed.read_text(encoding="utf-8") + "200\n", encoding="utf-8") + + second: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + assert first.returncode == 0, first.stdout + first.stderr + assert diagnostic_messages(first, "SQBRSQL038") == () + assert diagnostic_messages(second, "SQBRSQL038") == test_case.expected_messages, ( + second.stdout + second.stderr + ) + assert second.returncode == test_case.expected_exit + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py new file mode 100644 index 0000000000..72112b7c1e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py @@ -0,0 +1,114 @@ +"""E2E coverage of the published-layer Rules on the orders and finance example.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import PublishedLayerCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + compile_diagnostics, + run_compile_cli, + write_model_files, +) + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRGRAPH103", "SQBRPROJECT107"]\n' +) +_ALLOWLIST: str = ( + "\n[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ domain = "sales", qualifier = "finance", ' + 'reason = "Finance reads a reviewed published variant." }]\n' +) +_FINANCE_PATH: str = "models/sales/mart/sales__mart__orders__finance.sql" +_INTERMEDIATE: tuple[str, str] = ( + "models/sales/intermediate/clean/sales__int_clean__orders.sql", + "MODEL (description 'Clean orders.');\nSELECT 1 AS order_id\n", +) +_ORDERS: tuple[str, str] = ( + "models/sales/mart/sales__mart__orders.sql", + "MODEL (description 'Published orders.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', +) +_CUSTOMERS: tuple[str, str] = ( + "models/sales/mart/sales__mart__customers.sql", + "MODEL (description 'Published customers.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', +) +_FINANCE_VARIANT: tuple[str, str] = ( + _FINANCE_PATH, + "MODEL (description 'Orders reshaped for finance.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', +) +_FINANCE_STAGING: tuple[str, str] = ( + "models/finance/staging/finance__stg_v__orders.sql", + "MODEL (description 'Finance view of orders.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders__finance") AS o\n', +) +_ORDERS_VIEW: tuple[str, str] = ( + "models/sales/mart/sales__mart_v__orders.sql", + "MODEL (description 'Orders view.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', +) +_FINANCE_FROM_PUBLISHED: tuple[str, str] = ( + "models/finance/staging/finance__stg_v__orders.sql", + "MODEL (description 'Finance view of orders.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', +) +_ORDERS_FINANCE: tuple[tuple[str, str], ...] = ( + _INTERMEDIATE, + _ORDERS, + _CUSTOMERS, + _FINANCE_VARIANT, + _FINANCE_STAGING, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PublishedLayerCase( + description="a consumer variant in the producing domain's published layer", + files=_ORDERS_FINANCE, + rules_toml="", + expected_findings=(("SQBRGRAPH103", _FINANCE_PATH), ("SQBRPROJECT107", _FINANCE_PATH)), + expected_exit=1, + ), + PublishedLayerCase( + description="an allowlisted qualifier passes the shape rule but not the read rule", + files=_ORDERS_FINANCE, + rules_toml=_ALLOWLIST, + expected_findings=(("SQBRGRAPH103", _FINANCE_PATH),), + expected_exit=1, + ), + PublishedLayerCase( + description="shared logic in intermediate and a view of its own entity", + files=(_INTERMEDIATE, _ORDERS, _CUSTOMERS, _ORDERS_VIEW, _FINANCE_FROM_PUBLISHED), + rules_toml="", + expected_findings=(), + expected_exit=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_published_models_when_compiling_then_layer_rules_report_variants( + test_case: PublishedLayerCase, + tmp_path: Path, +) -> None: + write_model_files( + project_dir=tmp_path, toml=_PROJECT_TOML + test_case.rules_toml, files=test_case.files + ) + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + findings: tuple[tuple[str, str], ...] = tuple( + sorted((str(item["code"]), str(item["path"])) for item in compile_diagnostics(result)) + ) + assert findings == test_case.expected_findings, result.stdout + result.stderr + assert result.returncode == test_case.expected_exit, result.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py new file mode 100644 index 0000000000..2d3d84d997 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py @@ -0,0 +1,182 @@ +"""E2E coverage of configuration-only Rule exceptions, including relation-level exceptions.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RuleExceptionConfigCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + diagnostic_messages, + diagnostic_text, + run_compile_cli, + write_model_files, +) + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL038"]\n' +_MODEL_PATH: str = "models/order_checks.sql" +_CUSTOMERS: str = ( + "CROSS JOIN with `customers` in CTE `order_checks` creates an explicit cartesian product" +) +_REGIONS: str = "CROSS JOIN with `regions` in CTE `final` creates an explicit cartesian product" +_EXCEPTION_HEAD: str = f'\n[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "{_MODEL_PATH}"\n' +_REASON: str = 'reason = "Every order is checked for every customer and region."\n' +_CUSTOMERS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "customers"\n{_REASON}' +_REGIONS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "regions"\n{_REASON}' +_SHIPMENTS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "shipments"\n{_REASON}' +_CTE_EXCEPTIONS: str = ( + f'{_EXCEPTION_HEAD}relation = "customers"\ncte = "order_checks"\n{_REASON}' + f'{_EXCEPTION_HEAD}relation = "regions"\ncte = "final"\n{_REASON}' +) +_MODEL_SQL: str = ( + "MODEL (description 'Order checks.');\n" + "WITH order_checks AS (\n" + ' SELECT o.order_id, c.customer_id FROM __ref("orders") AS o\n' + "{directive}" + ' CROSS JOIN __ref("customers") AS c\n' + "),\n" + "final AS (\n" + " SELECT k.order_id, k.customer_id, r.region_id FROM order_checks AS k\n" + ' CROSS JOIN __ref("regions") AS r\n' + ")\n" + "SELECT f.order_id, f.customer_id, f.region_id FROM final AS f\n" +) +_PLAIN_SQL: str = _MODEL_SQL.format(directive="") +_DIRECTIVE_SQL: str = _MODEL_SQL.format( + directive=" -- sqb: ignore SQBRSQL038 because every order meets every customer\n" +) +_LITERAL_SQL: str = _PLAIN_SQL.replace( + "SELECT f.order_id, f.customer_id, f.region_id FROM final AS f", + "SELECT f.order_id, f.customer_id, f.region_id, 'sqb: ignore SQBRSQL038' AS note FROM final AS f", +) +_TRAILING_SQL: str = _PLAIN_SQL.replace( + ' CROSS JOIN __ref("customers") AS c\n', + ' CROSS JOIN __ref("customers") AS c -- sqb: ignore SQBRSQL038 because deliberate\n', +) +_UNNAMED_SQL: str = ( + "MODEL (description 'Order checks.');\n" + 'SELECT o.order_id, u.n FROM __ref("orders") AS o CROSS JOIN UNNEST([1, 2]) AS u(n)\n' +) +_UPSTREAM_FILES: tuple[tuple[str, str], ...] = ( + ("models/orders.sql", "MODEL (description 'Sample orders.');\nSELECT 1 AS order_id\n"), + ( + "models/customers.sql", + "MODEL (description 'Sample customers.');\nSELECT 1 AS customer_id\n", + ), + ("models/regions.sql", "MODEL (description 'Sample regions.');\nSELECT 1 AS region_id\n"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + RuleExceptionConfigCase( + description="no exception reports both joins", + rules_toml="", + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_messages=(_CUSTOMERS, _REGIONS), + ), + RuleExceptionConfigCase( + description="relation-level exception covers exactly one join", + rules_toml=_CUSTOMERS_EXCEPTION, + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_messages=(_REGIONS,), + ), + RuleExceptionConfigCase( + description="CTE-scoped exceptions cover both joins", + rules_toml=_CTE_EXCEPTIONS, + model_sql=_PLAIN_SQL, + expected_exit=0, + ), + RuleExceptionConfigCase( + description="stale relation-level exception fails", + rules_toml=_CUSTOMERS_EXCEPTION + _REGIONS_EXCEPTION + _SHIPMENTS_EXCEPTION, + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_fragments=( + "stale rule exception suppresses no finding: SQBRSQL038 at " + "models/order_checks.sql (relation shipments)", + ), + ), + RuleExceptionConfigCase( + description="inline directive is an error that prints its exception", + rules_toml="", + model_sql=_DIRECTIVE_SQL, + expected_exit=1, + expected_messages=(_CUSTOMERS, _REGIONS), + expected_fragments=( + "Inline Rule suppressions are not supported", + '[[rules.rule_exceptions]]\n rule = "SQBRSQL038"\n' + ' path = "models/order_checks.sql"\n' + ' relation = "customers"\n' + ' reason = "every order meets every customer"', + ), + ), + RuleExceptionConfigCase( + description="a string literal mentioning a directive compiles cleanly", + rules_toml=_CTE_EXCEPTIONS, + model_sql=_LITERAL_SQL, + expected_exit=0, + ), + RuleExceptionConfigCase( + description="a trailing directive after code is an error", + rules_toml=_CTE_EXCEPTIONS, + model_sql=_TRAILING_SQL, + expected_exit=1, + expected_fragments=( + "Inline Rule suppressions are not supported", + ' relation = "customers"\n reason = "deliberate"', + ), + ), + RuleExceptionConfigCase( + description="an unnamed join prints a file-wide exception for its file", + rules_toml="", + model_sql=_UNNAMED_SQL, + expected_exit=1, + expected_messages=( + "CROSS JOIN creates an explicit cartesian product with an unnamed relation; " + "only a file-wide exception (rule and path) can cover it", + ), + expected_fragments=( + '[[rules.rule_exceptions]]\n rule = "SQBRSQL038"\n' + ' path = "models/order_checks.sql"\n', + ), + ), + RuleExceptionConfigCase( + description="removed allow_exceptions setting fails with migration help", + rules_toml="allow_exceptions = false\n", + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_fragments=( + "sets [rules] allow_exceptions, which SQLBuild no longer supports", + "remove the allow_exceptions line from [rules]", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_exception_config_when_compiling_then_only_named_findings_are_excepted( + test_case: RuleExceptionConfigCase, + tmp_path: Path, +) -> None: + write_model_files( + project_dir=tmp_path, + toml=_PROJECT_TOML + test_case.rules_toml, + files=(*_UPSTREAM_FILES, (_MODEL_PATH, test_case.model_sql)), + ) + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + text: str = diagnostic_text(result) + assert result.returncode == test_case.expected_exit, text + assert diagnostic_messages(result, "SQBRSQL038") == test_case.expected_messages, text + for fragment in test_case.expected_fragments: + assert fragment in text + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py b/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py index dbef4d15bb..a9ac76f605 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py @@ -145,3 +145,10 @@ class DagsterEmptyCursorInputsCheckE2ETestCase: model_sql: str asset_key: tuple[str, ...] expected_check_names: tuple[str, ...] + + +@dataclass(frozen=True) +class DagsterAuditAttachmentChecksE2ETestCase: + description: str + command: tuple[str, ...] + expected_checked_assets: tuple[tuple[str, ...], ...] diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index f6eacfa60a..36917cf737 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -5,6 +5,7 @@ import time from collections import defaultdict from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path from threading import Event from types import MappingProxyType @@ -391,3 +392,94 @@ def sqlbuild_assets(context): defs = dg.Definitions(assets=[sqlbuild_assets], jobs=[dg.define_asset_job("stdout_capture_job")]) """ + + +@dataclass(frozen=True) +class AuditCheckOutcome: + """The asset checks one Dagster run declared and evaluated.""" + + success: bool + declared: frozenset[tuple[AssetKey, str]] + declared_assets: frozenset[AssetKey] + evaluated: frozenset[tuple[AssetKey, str]] + all_passed: bool + + +ATTACHMENT_CHECK_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "attachment_checks"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "attachment_checks.duckdb"\n' + ), + "seeds/products.csv": "product_id,product_name\n1,widget\n2,gadget\n", + "seeds/products.yml": ( + "seeds:\n" + " - name: products\n" + " description: Product catalogue.\n" + " columns:\n" + " - name: product_id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - name: product_name\n" + " type: VARCHAR\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n" + " description: Raw orders.\n" + " expression: |\n" + " SELECT 1 AS order_id, 1 AS product_id\n" + " UNION ALL SELECT 2 AS order_id, 2 AS product_id\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - name: product_id\n" + " type: INTEGER\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.',\n" + " columns (order_id (), product_id ()),\n" + " audits [not_null (column order_id)],\n" + ");\n\n" + 'SELECT order_id, product_id FROM __source("raw_orders")\n' + ), + "models/customers.sql": ( + "MODEL (description 'Customers.');\n\nSELECT 1 AS customer_id, 1 AS order_id\n" + ), + "audits/singular/orders_have_customers.sql": ( + 'AUDIT (name "orders_have_customers", severity error);\n\n' + "SELECT o.order_id\n" + 'FROM __ref("orders") o\n' + 'LEFT JOIN __ref("customers") c ON o.order_id = c.order_id\n' + "WHERE c.order_id IS NULL AND o.order_id = 1\n" + ), + "audits/singular/orders_match_raw_orders.sql": ( + 'AUDIT (name "orders_match_raw_orders", severity error);\n\n' + "SELECT o.order_id\n" + 'FROM __ref("orders") o\n' + 'LEFT JOIN __source("raw_orders") r ON o.order_id = r.order_id\n' + "WHERE r.order_id IS NULL\n" + ), +} + + +def audit_check_outcome(*, result: ExecuteInProcessResult, assets: Any) -> AuditCheckOutcome: + """The asset checks a Dagster run of `assets` declared and evaluated.""" + + declared: frozenset[tuple[AssetKey, str]] = frozenset( + (spec.asset_key, spec.name) for spec in assets.check_specs + ) + evaluations: list[Any] = list(result.get_asset_check_evaluations()) + return AuditCheckOutcome( + success=result.success, + declared=declared, + declared_assets=frozenset(asset_key for asset_key, _ in declared), + evaluated=frozenset( + (evaluation.asset_key, evaluation.check_name) for evaluation in evaluations + ), + all_passed=all(evaluation.passed for evaluation in evaluations), + ) diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py new file mode 100644 index 0000000000..4b3cc97628 --- /dev/null +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py @@ -0,0 +1,139 @@ +"""E2E coverage of Dagster asset checks for every audit attachment kind. + +The project has seed, source and model audits, a singular audit inferred to attach to a model, +and a singular audit that runs at the end of the build. +""" + +import subprocess +from collections.abc import Iterator +from pathlib import Path + +import pytest +from dagster import AssetExecutionContext, AssetKey, ExecuteInProcessResult, materialize + +from sqlbuild.integrations.dagster import SqlBuildCliResource, SqlBuildProject, sqlbuild_assets +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, + prepare_inline_project, + run_sqb, +) +from tests.e2e.src.sqlbuild.integrations.dagster._test_types import ( + DagsterAuditAttachmentChecksE2ETestCase, +) +from tests.e2e.src.sqlbuild.integrations.dagster.helpers import ( + ATTACHMENT_CHECK_PROJECT_FILES, + AuditCheckOutcome, + audit_check_outcome, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterAuditAttachmentChecksE2ETestCase( + description="build reports every audit attachment kind as checks", + command=("build",), + expected_checked_assets=( + ("main", "products"), + ("raw_orders",), + ("main", "orders"), + ("main", "customers"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_every_audit_attachment_kind_when_building_through_dagster_then_checks_are_reported( + test_case: DagsterAuditAttachmentChecksE2ETestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="attachment_checks", + repo_files=ATTACHMENT_CHECK_PROJECT_FILES, + ) + + sqlbuild_project: SqlBuildProject = SqlBuildProject( + project_dir=project_dir, sqb_command=(str(REPO_ROOT / ".venv" / "bin" / "sqb"),) + ) + sqlbuild_project.prepare() + + @sqlbuild_assets(project=sqlbuild_project, required_resource_keys={"sqb"}) + def sqlbuild_attachment_checks(context: AssetExecutionContext) -> Iterator[object]: + yield from context.resources.sqb.cli(list(test_case.command), context=context).stream() + + result: ExecuteInProcessResult = materialize( + [sqlbuild_attachment_checks], + resources={"sqb": SqlBuildCliResource(project_dir=sqlbuild_project)}, + ) + + checks: AuditCheckOutcome = audit_check_outcome( + result=result, assets=sqlbuild_attachment_checks + ) + + assert checks.success is True + assert { + AssetKey(list(path)) for path in test_case.expected_checked_assets + } <= checks.declared_assets + assert checks.declared <= checks.evaluated + assert checks.all_passed is True + assert "integration output degraded" not in capsys.readouterr().err + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterAuditAttachmentChecksE2ETestCase( + description="audit reports every audit attachment kind as checks", + command=("audit",), + expected_checked_assets=( + ("main", "products"), + ("raw_orders",), + ("main", "orders"), + ("main", "customers"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_every_audit_attachment_kind_when_auditing_through_dagster_then_checks_are_reported( + test_case: DagsterAuditAttachmentChecksE2ETestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="attachment_checks", + repo_files=ATTACHMENT_CHECK_PROJECT_FILES, + ) + prebuild: subprocess.CompletedProcess[str] = run_sqb( + command=("build",), project_dir=project_dir + ) + + sqlbuild_project: SqlBuildProject = SqlBuildProject( + project_dir=project_dir, sqb_command=(str(REPO_ROOT / ".venv" / "bin" / "sqb"),) + ) + sqlbuild_project.prepare() + + @sqlbuild_assets(project=sqlbuild_project, required_resource_keys={"sqb"}) + def sqlbuild_attachment_checks(context: AssetExecutionContext) -> Iterator[object]: + yield from context.resources.sqb.cli(list(test_case.command), context=context).stream() + + result: ExecuteInProcessResult = materialize( + [sqlbuild_attachment_checks], + resources={"sqb": SqlBuildCliResource(project_dir=sqlbuild_project)}, + ) + + checks: AuditCheckOutcome = audit_check_outcome( + result=result, assets=sqlbuild_attachment_checks + ) + + assert prebuild.returncode == 0, prebuild.stdout + prebuild.stderr + assert checks.success is True + assert { + AssetKey(list(path)) for path in test_case.expected_checked_assets + } <= checks.declared_assets + assert checks.declared <= checks.evaluated + assert checks.all_passed is True + assert "integration output degraded" not in capsys.readouterr().err diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 127cffc49b..036d4c22ef 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 6dc1cc9791..2cbd009334 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -983,7 +983,7 @@ class WarmTargetProjectConfigEditTestCase: @dataclass(frozen=True) class RuleExceptionPolicyTestCase: - """One `[rules] allow_exceptions` value with one escape hatch, compiled through the CLI.""" + """One Rule exception mechanism, compiled through the CLI.""" description: str rules_toml: str @@ -994,7 +994,7 @@ class RuleExceptionPolicyTestCase: @dataclass(frozen=True) class FormatExceptionPolicyTestCase: - """One `[rules] allow_exceptions` value read by `sqb format`.""" + """One exception-policy case read by `sqb format`.""" description: str setting: str @@ -1068,6 +1068,7 @@ class StartupImportFootprintTestCase: argv: tuple[str, ...] expected_exit_code: int forbidden_modules: tuple[str, ...] + local_config: str = "" @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py index e0f76b9caf..29dbf2e504 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py @@ -1,4 +1,4 @@ -"""Real-CLI coverage for forbidding Rule exceptions, scoped ignores and inline suppressions.""" +"""Real-CLI coverage for configuration-only Rule exceptions and rejected inline suppressions.""" from __future__ import annotations @@ -22,70 +22,66 @@ 'reason = "Samples intentionally keep one row."\n' ) _DIRECTIVE: str = "-- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row\n" -_SETTING_HELP: str = "[rules]\n allow_exceptions = true" +_PRINTED_ENTRY: str = ( + "[[rules.rule_exceptions]]\n" + ' rule = "SQBRSQL004"\n' + ' path = "models/customers.sql"\n' + ' reason = "the sample intentionally keeps one row"' +) +_REMOVED: str = "sets [rules] allow_exceptions, which SQLBuild no longer supports" @pytest.mark.parametrize( "test_case", [ RuleExceptionPolicyTestCase( - description="exception forbidden", - rules_toml=f"allow_exceptions = false\n\n{_EXCEPTION}", - directive="", + description="inline directive is an error printing its exception", + rules_toml="", + directive=_DIRECTIVE, expected_exit_code=1, expected_fragments=( - "sqlbuild_project.toml has 1 [[rules.rule_exceptions]] entry", - "sets [rules] allow_exceptions = false", - _SETTING_HELP, + "SQBRSQL000", + "Inline Rule suppressions are not supported", + _PRINTED_ENTRY, + "SQBRSQL004", ), ), RuleExceptionPolicyTestCase( - description="scoped ignore forbidden", - rules_toml=f"allow_exceptions = false\n\n{_IGNORE}", + description="allow_exceptions false is a removed setting", + rules_toml=f"allow_exceptions = false\n\n{_EXCEPTION}", directive="", expected_exit_code=1, - expected_fragments=("1 [[rules.rule_ignores]] entry", _SETTING_HELP), + expected_fragments=(_REMOVED, "remove the allow_exceptions line from [rules]"), ), RuleExceptionPolicyTestCase( - description="inline suppression forbidden", - rules_toml="allow_exceptions = false\n", - directive=_DIRECTIVE, + description="allow_exceptions true is a removed setting", + rules_toml="allow_exceptions = true\n", + directive="", expected_exit_code=1, - expected_fragments=( - "SQBRSQL000", - "Inline Rule suppressions are not allowed in this project", - "SQBRSQL004", - "allow_exceptions = true", - ), + expected_fragments=(_REMOVED,), ), RuleExceptionPolicyTestCase( - description="project-wide ignore stays allowed", - rules_toml='allow_exceptions = false\nignore = ["SQBRSQL004"]\n', + description="configured exception", + rules_toml=_EXCEPTION, directive="", expected_exit_code=0, ), RuleExceptionPolicyTestCase( - description="exception allowed explicitly", - rules_toml=f"allow_exceptions = true\n\n{_EXCEPTION}", + description="scoped ignore", + rules_toml=_IGNORE, directive="", expected_exit_code=0, ), RuleExceptionPolicyTestCase( - description="exception allowed by default", - rules_toml=_EXCEPTION, + description="project-wide ignore", + rules_toml='ignore = ["SQBRSQL004"]\n', directive="", expected_exit_code=0, ), - RuleExceptionPolicyTestCase( - description="inline suppression allowed by default", - rules_toml="", - directive=_DIRECTIVE, - expected_exit_code=0, - ), ], ids=lambda case: case.description, ) -def test_given_allow_exceptions_setting_when_compiling_then_escape_hatches_follow_it( +def test_given_rule_exception_mechanism_when_compiling_then_only_configuration_applies( test_case: RuleExceptionPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -116,30 +112,23 @@ def test_given_allow_exceptions_setting_when_compiling_then_escape_hatches_follo "test_case", [ FormatExceptionPolicyTestCase( - description="string false is rejected like compile", - setting='allow_exceptions = "false"', - expected_exit_code=1, - expected_fragment="invalid rules config", - expected_compile_fragment="invalid rules config", - ), - FormatExceptionPolicyTestCase( - description="boolean false rejects the inline suppression", + description="removed setting is rejected like compile", setting="allow_exceptions = false", expected_exit_code=1, - expected_fragment="Inline Rule suppressions are not allowed in this project", - expected_compile_fragment="Inline Rule suppressions are not allowed in this project", + expected_fragment=_REMOVED, + expected_compile_fragment=_REMOVED, ), FormatExceptionPolicyTestCase( - description="boolean true keeps the inline suppression", - setting="allow_exceptions = true", - expected_exit_code=0, - expected_fragment="FAULT=0", - expected_compile_fragment="", + description="inline directive is rejected like compile", + setting="", + expected_exit_code=1, + expected_fragment="Inline Rule suppressions are not supported", + expected_compile_fragment="Inline Rule suppressions are not supported", ), ], ids=lambda case: case.description, ) -def test_given_allow_exceptions_setting_when_formatting_then_it_is_validated_like_compile( +def test_given_exception_policy_when_formatting_then_it_is_validated_like_compile( test_case: FormatExceptionPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index e8b679ba72..e123c6fd33 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -53,8 +53,7 @@ _CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' _CACHE_THRESHOLD_TOML: str = _CACHE_RULES_TOML + "thresholds = { max_literal_length = 7 }\n" -_CACHE_POLICY_TOML: str = _CACHE_THRESHOLD_TOML + "allow_exceptions = false\n" -_CACHE_SELECTION_TOML: str = _CACHE_POLICY_TOML.replace( +_CACHE_SELECTION_TOML: str = _CACHE_THRESHOLD_TOML.replace( '"SQBRSQL004"', '"SQBRSQL004", "SQBRSQL001"' ) _CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n' @@ -2879,10 +2878,10 @@ def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remai @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("unused inline suppression is diagnosed", 1, "SQBRSQL000")], + [RulesIntegrationTestCase("inline directive without a finding fails", 1, "SQBRSQL000")], ids=lambda case: case.description, ) -def test_given_unused_inline_suppression_when_compiling_then_stale_directive_is_reported( +def test_given_inline_directive_without_finding_when_compiling_then_directive_is_reported( test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -3034,7 +3033,6 @@ def test_given_empty_rule_selection_when_compiling_then_only_configuration_is_va (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), (_CACHE_THRESHOLD_TOML, _CACHE_EDITED_SQL), - (_CACHE_POLICY_TOML, _CACHE_EDITED_SQL), (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), ), @@ -3045,7 +3043,6 @@ def test_given_empty_rule_selection_when_compiling_then_only_configuration_is_va (1, 0), (0, 1), (0, 1), - (0, 1), (1, 0), ), ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 85d8a7db1d..aaf1acff51 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -62,6 +62,26 @@ "sqlbuild.adapter.contract.classes.base_adapter", "sqlbuild.compiler.pipeline", ) +INSPECTION_FORBIDDEN: tuple[str, ...] = ( + *WAREHOUSE_AND_INTEGRATION_MODULES, + "filelock", + "polyglot_sql", + "pydantic", + "yaml", + "sqlbuild.lint", + "sqlbuild.rule_engine", + *PRESENTATION_AND_PROVIDER_MODULES, +) +PROJECT_DISCOVERY_MODULES: tuple[str, ...] = ( + "sqlbuild.compiler.discovery._helpers.validation", + "sqlbuild.compiler.macro_bridge", + "sqlbuild.compiler.python_nodes", +) +QUERY_FORBIDDEN: tuple[str, ...] = ( + *(module for module in INSPECTION_FORBIDDEN if module != "duckdb"), + *PROJECT_DISCOVERY_MODULES, +) +MEMORY_CONNECTION: str = '[connection]\ndatabase = ":memory:"\n' REUSED_COMPILE_FORBIDDEN: tuple[str, ...] = ( *WAREHOUSE_AND_INTEGRATION_MODULES, *FULL_COMPILE_MODULES, @@ -118,6 +138,31 @@ expected_exit_code=0, forbidden_modules=COMPILE_FORBIDDEN, ), + StartupImportFootprintTestCase( + description="scope", + argv=("scope", "orders"), + expected_exit_code=0, + forbidden_modules=INSPECTION_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="lineage", + argv=("lineage", "orders", "--json"), + expected_exit_code=0, + forbidden_modules=INSPECTION_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="dag", + argv=("dag", "--json"), + expected_exit_code=0, + forbidden_modules=INSPECTION_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="ad hoc query", + argv=("query", "SELECT 1 AS order_id"), + expected_exit_code=0, + forbidden_modules=QUERY_FORBIDDEN, + local_config=MEMORY_CONNECTION, + ), ], ids=lambda case: case.description, ) @@ -125,6 +170,7 @@ def test_given_fresh_interpreter_when_running_cli_then_heavy_modules_stay_unload test_case: StartupImportFootprintTestCase, tmp_path: Path ) -> None: write_compile_startup_project(tmp_path) + _ = (tmp_path / "sqlbuild_local.toml").write_text(test_case.local_config, encoding="utf-8") argv: list[str] = ["--project-dir", str(tmp_path), *test_case.argv] result: subprocess.CompletedProcess[str] = subprocess.run( diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 0db3751b7d..c479d5d40f 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -282,6 +282,29 @@ def prepare_custom_rules(directory: Path) -> None: (directory / "rules").mkdir() +def prepare_removed_allow_exceptions(directory: Path) -> None: + """Write the project still setting the removed `[rules] allow_exceptions`.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + "\n[rules]\nallow_exceptions = true\n", + encoding="utf-8", + ) + + +def prepare_published_layer_options(directory: Path) -> None: + """Write the project with configured published-layer Rule options.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + + "\n[rules.rule_options.SQBRPROJECT107]\n" + + 'published_layers = ["mart"]\n' + + 'allowed_qualifiers = [{ qualifier = "csv_export", reason = "Export views." }]\n', + encoding="utf-8", + ) + + def prepare_named_pipe_project(directory: Path) -> None: """Make the project file a named pipe that blocks its reader.""" diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py index fe5412e49b..747ab04902 100644 --- a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -22,6 +22,8 @@ prepare_named_pipe_project, prepare_nothing, prepare_project, + prepare_published_layer_options, + prepare_removed_allow_exceptions, prepare_stale_skills, python_starts, reset_directory, @@ -179,6 +181,34 @@ expected_exit_code=0, expected_captures=1, ), + PythonFreeTestCase( + description="rules show with a string-list option default", + args=("rules", "show", "SQBRGRAPH103"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show with a table-list option", + args=("rules", "show", "SQBRPROJECT107"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show cross join", + args=("rules", "show", "SQBRSQL038"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show inline directive rule", + args=("rules", "show", "SQBRSQL000"), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=1, + ), PythonFreeTestCase( description="rules show unknown", args=("rules", "show", "SQBRNOPE101"), @@ -262,6 +292,18 @@ def test_given_python_free_invocation_when_running_sqb_then_python_never_starts_ prepare=prepare_custom_rules, expected_python_starts=1, ), + PythonStartTestCase( + description="removed allow_exceptions setting", + args=("rules", "list"), + prepare=prepare_removed_allow_exceptions, + expected_python_starts=1, + ), + PythonStartTestCase( + description="configured rule options", + args=("rules", "show", "SQBRPROJECT107"), + prepare=prepare_published_layer_options, + expected_python_starts=1, + ), PythonStartTestCase( description="dbt passthrough help", args=("dbt", "plan", "--help"), diff --git a/tests/integration/src/sqlbuild/cli/output/_test_types.py b/tests/integration/src/sqlbuild/cli/output/_test_types.py index 5d3f28690c..806697685e 100644 --- a/tests/integration/src/sqlbuild/cli/output/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/output/_test_types.py @@ -2,7 +2,8 @@ from dataclasses import dataclass -from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditOutcome +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind @dataclass(frozen=True) @@ -11,3 +12,15 @@ class MeasurementEnvelopeTestCase: outcome: AuditOutcome expected_status: str expected_passed: bool + + +@dataclass(frozen=True) +class AttachmentKindEnvelopeTestCase: + description: str + attachment_kind: AuditAttachmentKind + attached_target_kind: AttachedAuditTargetKind | None + attached_target_name: str | None + expected_attachment_kind: str + expected_asset_name: str | None + expected_degraded: bool = False + expected_passed: bool = True diff --git a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py index aeb5db14ee..37a802f2e5 100644 --- a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py +++ b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py @@ -1,4 +1,4 @@ -"""Measurement audit integration-envelope round-trip coverage.""" +"""Audit integration-envelope round-trip coverage.""" from __future__ import annotations @@ -19,10 +19,17 @@ AuditSeverity, ThresholdOperator, ) +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.models import LifecycleEvent -from tests.integration.src.sqlbuild.cli.output._test_types import MeasurementEnvelopeTestCase -from tests.unit.src.sqlbuild.cli.output._helpers.helpers import written_integration_record +from tests.integration.src.sqlbuild.cli.output._test_types import ( + AttachmentKindEnvelopeTestCase, + MeasurementEnvelopeTestCase, +) +from tests.unit.src.sqlbuild.cli.output._helpers.helpers import ( + audit_terminal, + written_integration_record, +) from tests.unit.src.sqlbuild.runtime.observability.helpers import lifecycle_event @@ -86,3 +93,70 @@ def test_given_measurement_execution_result_when_building_envelope_then_decode_r assert check.thresholds == {"warn": {"operator": "below", "limit": 100.0}} assert check.evidence_count == 0 assert check.evidence_truncated is False + + +@pytest.mark.parametrize( + "test_case", + ( + AttachmentKindEnvelopeTestCase( + "model audit", + AuditAttachmentKind.MODEL, + AttachedAuditTargetKind.MODEL, + "orders", + expected_attachment_kind="model", + expected_asset_name="orders", + ), + AttachmentKindEnvelopeTestCase( + "source audit", + AuditAttachmentKind.SOURCE, + AttachedAuditTargetKind.SOURCE, + "raw_orders", + expected_attachment_kind="source", + expected_asset_name="raw_orders", + ), + AttachmentKindEnvelopeTestCase( + "seed audit", + AuditAttachmentKind.SEED, + AttachedAuditTargetKind.SEED, + "products", + expected_attachment_kind="seed", + expected_asset_name="products", + ), + AttachmentKindEnvelopeTestCase( + "end-of-run audit", + AuditAttachmentKind.END, + None, + None, + expected_attachment_kind="end", + expected_asset_name=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_every_attachment_kind_when_building_envelope_then_check_is_not_degraded( + test_case: AttachmentKindEnvelopeTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + result: AuditExecutionResult = AuditExecutionResult( + audit_name="not_null", + audit_definition_name="not_null", + attachment_kind=test_case.attachment_kind, + attached_target_kind=test_case.attached_target_kind, + attached_target_name=test_case.attached_target_name, + severity=AuditSeverity.ERROR, + outcome=AuditOutcome.PASS, + row_count=0, + executed_sql="SELECT 1 WHERE FALSE", + ) + + envelope: IntegrationResultEnvelope = written_integration_record( + result=result, terminal=audit_terminal(result), directory=tmp_path, command="audit" + ) + + check: IntegrationCheckResult = envelope.checks[0] + assert envelope.projection_degraded is test_case.expected_degraded + assert check.attachment_kind == test_case.expected_attachment_kind + assert check.asset_name == test_case.expected_asset_name + assert check.passed is test_case.expected_passed + assert "integration output degraded" not in capsys.readouterr().err diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index 033d1dc48d..ed977f959d 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -64,9 +64,7 @@ def test_given_increasing_findings_when_publishing_cache_then_findings_are_store project=project, model_paths=frozenset(paths), selected=(SimpleNamespace(code="SQBRSQL004", custom=False),), - effective_config=SimpleNamespace( - thresholds={}, allow_exceptions=True, cache=SimpleNamespace(enabled=True) - ), + effective_config=SimpleNamespace(thresholds={}, cache=SimpleNamespace(enabled=True)), ) result: SqlRulesEvaluation = native.run_planned_sql_rules( plan=cast(Any, plan), diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py index 695632573f..afbc4b0c70 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py @@ -68,7 +68,7 @@ def __call__(self, **kwargs: Any) -> RunCostSummary: collection_error=True, close_error=False, expected_status=CostStatus.COLLECTION_FAILED, - expected_message_fragment="collection failed (PermissionError)", + expected_message_fragment="collection failed (PermissionError: query history denied)", ), SnowflakeCostCollectionTestCase( description="close failure downgrades successful collection to partial", @@ -113,6 +113,6 @@ def test_given_telemetry_stage_failure_when_collecting_then_safe_status_is_retur assert summary.status == test_case.expected_status assert connect.config["login_timeout"] == 5 - assert connect.config["network_timeout"] == 5 + assert connect.config["network_timeout"] == 15 assert (test_case.expected_message_fragment or "") in (summary.message or "") assert (test_case.expected_limitation_fragment or "") in " ".join(summary.limitations) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 4f4499d076..5a48424c59 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -1039,3 +1039,12 @@ class OversizedDurationTestCase: description: str section: str expected_error_fragment: str + + +@dataclass(frozen=True) +class TargetConcurrencyConfigTestCase: + description: str + target_lines: str + expected_concurrency: int | None = None + expected_error_fragment: str = "" + expected_error_code: str = "D001" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py new file mode 100644 index 0000000000..4a1e1523ad --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py @@ -0,0 +1,78 @@ +"""Loading and validation of per-target concurrency.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import ProjectConfig, ProjectConfigError +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( + TargetConcurrencyConfigTestCase, +) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import load_project_config + +_PROJECT: str = 'name = "shop"\nadapter = "duckdb"\n\n[targets.prod]\nschema = "main"\n' + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyConfigTestCase( + description="a positive integer is loaded", + target_lines="concurrency = 8\n", + expected_concurrency=8, + ), + TargetConcurrencyConfigTestCase( + description="an absent value leaves the target unset", + target_lines="", + expected_concurrency=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_valid_target_concurrency_when_loading_then_it_is_kept( + test_case: TargetConcurrencyConfigTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + _PROJECT + test_case.target_lines, encoding="utf-8" + ) + + config: ProjectConfig = load_project_config(project_dir=tmp_path) + + assert config.targets["prod"].concurrency == test_case.expected_concurrency + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyConfigTestCase( + description="zero is rejected", + target_lines="concurrency = 0\n", + expected_error_fragment="targets.prod.concurrency must be an integer >= 1", + ), + TargetConcurrencyConfigTestCase( + description="a boolean is rejected", + target_lines="concurrency = true\n", + expected_error_fragment="targets.prod.concurrency must be an integer >= 1", + ), + TargetConcurrencyConfigTestCase( + description="a string is rejected", + target_lines='concurrency = "8"\n', + expected_error_fragment="targets.prod.concurrency must be an integer >= 1", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_target_concurrency_when_loading_then_coded_error_is_raised( + test_case: TargetConcurrencyConfigTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + _PROJECT + test_case.target_lines, encoding="utf-8" + ) + + with pytest.raises(ProjectConfigError) as error: + load_project_config(project_dir=tmp_path) + + assert error.value.code == test_case.expected_error_code + assert test_case.expected_error_fragment in str(error.value) diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index ac1e03b09f..158999812c 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from datetime import timedelta from pathlib import Path from sqlbuild.cost.types import CostStatus @@ -101,3 +102,41 @@ class CostResourceScopeTestCase: expected_resource_name: str expected_phase: str expected_attempt: int + + +@dataclass(frozen=True) +class SnowflakeScaleTestCase: + description: str + statement_count: int + duration_minutes: int + timeout_window: timedelta | None + expected_status: CostStatus + expected_query_count: int + expected_message_fragments: tuple[str, ...] = () + expected_max_sql_length: int = 2_000 + expected_minimum_reads: int = 2 + expected_maximum_reads: int = 1_000 + + +@dataclass(frozen=True) +class SnowflakeLargeRunCostTestCase: + description: str + timeout_window: timedelta | None + expected_status: CostStatus + expected_message_fragment: str + expected_max_sql_length: int = 2_000 + + +@dataclass(frozen=True) +class SnowflakeCostFailureMessageTestCase: + description: str + expected_status: CostStatus + expected_message: str + + +@dataclass(frozen=True) +class SnowflakeCostBudgetTestCase: + description: str + seconds_per_read: float + expected_status: CostStatus + expected_message_fragment: str = "" diff --git a/tests/unit/src/sqlbuild/cost/helpers.py b/tests/unit/src/sqlbuild/cost/helpers.py index 6cb346cdb9..cab18bd7b3 100644 --- a/tests/unit/src/sqlbuild/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cost/helpers.py @@ -1,5 +1,8 @@ +import json from datetime import UTC, datetime, timedelta from decimal import Decimal +from pathlib import Path +from typing import Any from sqlbuild._native import CostRunRecord, QueryCostObservation, ResourceCost from sqlbuild.cost.models import RunCostSummary @@ -67,3 +70,60 @@ def build_cost_run_record(*, run_id: str, completed_at: datetime) -> CostRunReco estimated_usd=resource.estimated_usd, ), ) + + +def history_row(*, query_id: str, ended_at: datetime) -> tuple[Any, ...]: + """One successful one-second warehouse query ending at `ended_at`.""" + + return ( + query_id, + "", + "TRANSFORM_WH", + "X-Small", + "STANDARD", + 1, + ended_at - timedelta(seconds=1), + ended_at, + 1000, + 100, + "SUCCESS", + "INSERT", + None, + ended_at, + ) + + +def run_history_rows( + *, origin: datetime, statement_count: int, duration: timedelta +) -> list[tuple[Any, ...]]: + """History rows of one run whose statements end evenly across `duration`.""" + + step: timedelta = duration / statement_count + return [ + history_row(query_id=f"q-{index:05d}", ended_at=origin + step * (index + 1)) + for index in range(statement_count) + ] + + +def write_ledger(*, path: Path, run_id: str, rows: list[tuple[Any, ...]]) -> None: + """Write one statement-ledger entry for each history row.""" + + with path.open("w", encoding="utf-8") as ledger: + for index, row in enumerate(rows): + ledger.write( + json.dumps( + { + "statement_id": f"statement-{index}", + "run_id": run_id, + "resource_type": "model", + "resource_name": f"orders_{index % 1400}", + "phase": "execute", + "attempt": 1, + "query_id": row[0], + "status": "success", + "started_at": row[6].isoformat(), + "completed_at": row[7].isoformat(), + } + ) + + "\n" + ) diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index 0a558b5c77..fabafcbfca 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -75,8 +75,10 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: "RESULT_LIMIT => 10000", "EXECUTION_STATUS", "QUERY_TYPE", - "CASE WHEN QUERY_ID IN ('q-create') AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE " - "'CREATE_%') THEN QUERY_TEXT END AS QUERY_TEXT", + "CASE WHEN (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') " + 'AND QUERY_TAG LIKE \'%"run_id":"run-1"%\' ' + 'AND QUERY_TAG NOT LIKE \'%"resource_type":"hook"%\' ' + "THEN QUERY_TEXT END AS QUERY_TEXT", "CURRENT_TIMESTAMP()", ), expected_excluded_fragment="WHERE EXECUTION_STATUS = 'SUCCESS'", @@ -91,7 +93,7 @@ def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_t database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), completed_at=datetime(2026, 8, 23, 10, 5, tzinfo=UTC), - classification_query_ids=frozenset({"q-create"}), + run_id="run-1", ) for fragment in test_case.expected_fragments: @@ -110,8 +112,10 @@ def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_t "RESULT_LIMIT => 10000", "EXECUTION_STATUS", "QUERY_TYPE", - "CASE WHEN QUERY_ID IN ('q-create') AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE " - "'CREATE_%') THEN QUERY_TEXT END AS QUERY_TEXT", + "CASE WHEN (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') " + 'AND QUERY_TAG LIKE \'%"run_id":"run-1"%\' ' + 'AND QUERY_TAG NOT LIKE \'%"resource_type":"hook"%\' ' + "THEN QUERY_TEXT END AS QUERY_TEXT", "CURRENT_TIMESTAMP()", ), expected_excluded_fragment="END_TIME_RANGE_END", @@ -125,7 +129,7 @@ def test_given_start_time_when_rendering_open_history_then_running_queries_are_r sql: str = render_snowflake_query_history_sql( database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), - classification_query_ids=frozenset({"q-create"}), + run_id="run-1", ) for fragment in test_case.expected_fragments: @@ -149,7 +153,6 @@ def test_given_no_classifiable_create_ids_when_rendering_history_then_query_text ) -> None: sql: str = render_snowflake_query_history_sql( started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), - classification_query_ids=frozenset(), ) for fragment in test_case.expected_fragments: @@ -161,9 +164,9 @@ def test_given_no_classifiable_create_ids_when_rendering_history_then_query_text "test_case", [ RenderQueryHistorySqlTestCase( - description="exact create hook is omitted from query text allowlist", + description="a run of only hooks requests no query text", expected_fragments=("NULL AS QUERY_TEXT",), - expected_excluded_fragment="q-hook", + expected_excluded_fragment="THEN QUERY_TEXT", ) ], ids=lambda case: case.description, @@ -865,7 +868,7 @@ def test_given_retry_delay_exceeds_remaining_deadline_when_collecting_then_parti None, origin + timedelta(seconds=1), ) - monotonic_values: Iterator[float] = iter((0.0, 0.0, 0.0, 0.0, 9.5)) + monotonic_values: Iterator[float] = iter((0.0, 0.0, 0.0, 0.0, 29.5)) monkeypatch.setattr( "time.monotonic", lambda: next(monotonic_values), diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py new file mode 100644 index 0000000000..7664da0998 --- /dev/null +++ b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py @@ -0,0 +1,439 @@ +"""Snowflake cost collection over runs with tens of thousands of statements.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field, replace +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import ( + CostCommandRequest, + CostRunRecord, + collect_snowflake_cost, + read_cost_run, + refresh_pending_cost_run, + write_cost_run, +) +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.cost.models import RunCostSummary +from sqlbuild.cost.types import CostStatus +from tests.unit.src.sqlbuild.cli.commands._helpers.cost.helpers import write_snowflake_project +from tests.unit.src.sqlbuild.cost._test_types import ( + SnowflakeCostBudgetTestCase, + SnowflakeCostFailureMessageTestCase, + SnowflakeLargeRunCostTestCase, + SnowflakeScaleTestCase, +) +from tests.unit.src.sqlbuild.cost.helpers import ( + build_cost_run_record, + history_row, + run_history_rows, + write_ledger, +) + +_ORIGIN: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) +_RUN_ID: str = "run-nightly" + + +_START_PATTERN: re.Pattern[str] = re.compile( + r"END_TIME_RANGE_START => TO_TIMESTAMP_LTZ\('([^']+)'\)" +) +_END_PATTERN: re.Pattern[str] = re.compile(r"END_TIME_RANGE_END => TO_TIMESTAMP_LTZ\('([^']+)'\)") +_LIMIT_PATTERN: re.Pattern[str] = re.compile(r"RESULT_LIMIT => (\d+)") +STATEMENT_TIMEOUT_ERRNO: int = 630 + + +class StatementTimeoutError(Exception): + """The error a Snowflake read raises when it reaches its statement timeout.""" + + def __init__(self, *, seconds: str) -> None: + super().__init__( + f"000630 (57014): Statement reached its statement or warehouse timeout of " + f"{seconds} second(s) and was canceled." + ) + self.errno: int = STATEMENT_TIMEOUT_ERRNO + + +@dataclass +class HistoryRead: + """One query-history read: its window, SQL length and statement timeout.""" + + start: datetime + end: datetime | None + sql_length: int + timeout_seconds: str + + +class _Cursor: + def __init__(self, rows: list[tuple[Any, ...]]) -> None: + self.rows: list[tuple[Any, ...]] = rows + + def fetchall(self) -> list[tuple[Any, ...]]: + return self.rows + + +@dataclass +class _Clock: + """A monotonic clock that advances only when the fake history answers a read.""" + + seconds: float = 0.0 + + def monotonic(self) -> float: + return self.seconds + + +@dataclass +class _FakeQueryHistory: + """Answers `INFORMATION_SCHEMA.QUERY_HISTORY` reads from fixed rows, like Snowflake does. + + Rows are filtered by end time, the open-ended read also returns running queries, and at most + `RESULT_LIMIT` rows are returned. A read wider than `timeout_window` raises Snowflake's + statement-timeout error, standing in for a read too slow for its statement timeout. + """ + + rows: list[tuple[Any, ...]] + timeout_window: timedelta | None = None + reads: list[HistoryRead] = field(default_factory=list) + clock: _Clock = field(default_factory=_Clock) + seconds_per_read: float = 0.0 + + @property + def longest_sql(self) -> int: + """The length of the longest SQL statement read.""" + + return max(read.sql_length for read in self.reads) + + def execute(self, sql: str, **kwargs: Any) -> _Cursor: + assert "INFORMATION_SCHEMA.QUERY_HISTORY" in sql + start_match: re.Match[str] | None = _START_PATTERN.search(sql) + limit_match: re.Match[str] | None = _LIMIT_PATTERN.search(sql) + assert start_match is not None and limit_match is not None + end_match: re.Match[str] | None = _END_PATTERN.search(sql) + start: datetime = datetime.fromisoformat(start_match.group(1)) + end: datetime | None = ( + datetime.fromisoformat(end_match.group(1)) if end_match is not None else None + ) + timeout_seconds: str = kwargs["statement_params"]["STATEMENT_TIMEOUT_IN_SECONDS"] + self.clock.seconds += self.seconds_per_read + self.reads.append( + HistoryRead(start=start, end=end, sql_length=len(sql), timeout_seconds=timeout_seconds) + ) + latest_end: datetime = end if end is not None else max(row[7] or start for row in self.rows) + if self.timeout_window is not None and latest_end - start > self.timeout_window: + raise StatementTimeoutError(seconds=timeout_seconds) + matched: list[tuple[Any, ...]] = [ + row + for row in self.rows + if (row[7] is None and end is None) + or (row[7] is not None and row[7] >= start and (end is None or row[7] < end)) + ] + return _Cursor(matched[: int(limit_match.group(1))]) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeScaleTestCase( + description="an unsaturated 8-hour run is read with one query", + statement_count=5_000, + duration_minutes=480, + timeout_window=None, + expected_status=CostStatus.COMPLETE, + expected_query_count=5_000, + expected_minimum_reads=1, + expected_maximum_reads=1, + ), + SnowflakeScaleTestCase( + description="a 78-minute run of 20,000 statements splits its saturated window", + statement_count=20_000, + duration_minutes=78, + timeout_window=None, + expected_status=CostStatus.COMPLETE, + expected_query_count=20_000, + ), + SnowflakeScaleTestCase( + description="a window that reaches the statement timeout is split and retried", + statement_count=6_500, + duration_minutes=42, + timeout_window=timedelta(minutes=8), + expected_status=CostStatus.COMPLETE, + expected_query_count=6_500, + ), + SnowflakeScaleTestCase( + description="history that times out at every width is reported pending with its cause", + statement_count=6_500, + duration_minutes=42, + timeout_window=timedelta(0), + expected_status=CostStatus.PENDING, + expected_query_count=0, + expected_message_fragments=( + "10-second statement timeout", + f"Run `sqb cost {_RUN_ID}` to retry.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_large_run_when_collecting_snowflake_cost_then_sql_stays_bounded( + test_case: SnowflakeScaleTestCase, + tmp_path: Path, +) -> None: + duration: timedelta = timedelta(minutes=test_case.duration_minutes) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=test_case.statement_count, duration=duration + ) + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id=_RUN_ID, rows=rows) + history: _FakeQueryHistory = _FakeQueryHistory( + rows=rows, timeout_window=test_case.timeout_window + ) + + summary: RunCostSummary = collect_snowflake_cost( + connection=history, + run_id=_RUN_ID, + started_at=_ORIGIN, + completed_at=_ORIGIN + duration, + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + attempts=1, + ) + + assert summary.status == test_case.expected_status + assert summary.query_count == test_case.expected_query_count + assert summary.expected_statement_count == test_case.statement_count + for fragment in test_case.expected_message_fragments: + assert fragment in (summary.message or "") + assert history.longest_sql < test_case.expected_max_sql_length + assert {read.timeout_seconds for read in history.reads} == {"10"} + assert test_case.expected_minimum_reads <= len(history.reads) + assert len(history.reads) <= test_case.expected_maximum_reads + + +class _ClosingHistory(_FakeQueryHistory): + def close(self) -> None: + return None + + +class ProgrammingError(Exception): + """Stands in for the Snowflake connector's error of the same name.""" + + +class _FailingHistory(_ClosingHistory): + def execute(self, sql: str, **kwargs: Any) -> Any: + del sql, kwargs + raise ProgrammingError( + "002003 (02000): SQL compilation error:\nDatabase 'ANALYTICS' does not exist." + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeLargeRunCostTestCase( + description="a large run's history is collected completely", + timeout_window=None, + expected_status=CostStatus.COMPLETE, + expected_message_fragment="", + ), + SnowflakeLargeRunCostTestCase( + description="history too slow to read leaves the run pending, naming the timeout", + timeout_window=timedelta(0), + expected_status=CostStatus.PENDING, + expected_message_fragment="statement timeout", + ), + ], + ids=lambda case: case.description, +) +def test_given_large_run_when_adapter_collects_cost_then_failures_stay_non_fatal( + test_case: SnowflakeLargeRunCostTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + origin: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) + rows: list[tuple[Any, ...]] = [ + history_row(query_id=f"q-{index:05d}", ended_at=origin + timedelta(seconds=index // 3)) + for index in range(12_000) + ] + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id="run-large", rows=rows) + history: _ClosingHistory = _ClosingHistory(rows=rows, timeout_window=test_case.timeout_window) + monkeypatch.setattr(adapter, "connect", lambda config: history) + + summary: RunCostSummary = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id="run-large", + started_at=origin, + completed_at=origin + timedelta(seconds=4_000), + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert summary.status == test_case.expected_status + assert test_case.expected_message_fragment in (summary.message or "") + assert history.longest_sql < test_case.expected_max_sql_length + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostFailureMessageTestCase( + description="a multi-line Snowflake error is named on one line", + expected_status=CostStatus.COLLECTION_FAILED, + expected_message=( + "Snowflake cost collection failed (ProgrammingError: 002003 (02000): " + "SQL compilation error: Database 'ANALYTICS' does not exist.)." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_query_history_error_when_adapter_collects_cost_then_message_names_cause( + test_case: SnowflakeCostFailureMessageTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + origin: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) + rows: list[tuple[Any, ...]] = [history_row(query_id="q-1", ended_at=origin)] + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id="run-failed", rows=rows) + monkeypatch.setattr(adapter, "connect", lambda config: _FailingHistory(rows=rows)) + + summary: RunCostSummary = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id="run-failed", + started_at=origin, + completed_at=origin + timedelta(seconds=1), + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert summary.status == test_case.expected_status + assert summary.message == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostBudgetTestCase( + description="the at-exit budget leaves a slow eight-hour run pending", + seconds_per_read=1.0, + expected_status=CostStatus.PENDING, + expected_message_fragment="30-second collection deadline exceeded", + ), + ], + ids=lambda case: case.description, +) +def test_given_slow_history_when_collecting_at_exit_then_run_stays_pending( + test_case: SnowflakeCostBudgetTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + duration: timedelta = timedelta(hours=8) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=2_000, duration=duration + ) + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id=_RUN_ID, rows=rows) + history: _ClosingHistory = _ClosingHistory( + rows=rows, + timeout_window=timedelta(minutes=10), + seconds_per_read=test_case.seconds_per_read, + ) + adapter: SnowflakeAdapter = SnowflakeAdapter() + monkeypatch.setattr(adapter, "connect", lambda config: history) + monkeypatch.setattr("time.monotonic", history.clock.monotonic) + + summary: RunCostSummary = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id=_RUN_ID, + started_at=_ORIGIN, + completed_at=_ORIGIN + duration, + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert summary.status == test_case.expected_status + assert test_case.expected_message_fragment in (summary.message or "") + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostBudgetTestCase( + description="sqb cost completes a slow eight-hour run left pending at exit", + seconds_per_read=1.0, + expected_status=CostStatus.COMPLETE, + ), + ], + ids=lambda case: case.description, +) +def test_given_pending_slow_run_when_refreshing_with_sqb_cost_then_larger_budget_completes_it( + test_case: SnowflakeCostBudgetTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + duration: timedelta = timedelta(hours=8) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=2_000, duration=duration + ) + ledger_dir: Path = tmp_path / "target" / "executions" / _RUN_ID + ledger_dir.mkdir(parents=True) + write_ledger(path=ledger_dir / "statements.jsonl", run_id=_RUN_ID, rows=rows) + record: CostRunRecord = build_cost_run_record(run_id=_RUN_ID, completed_at=_ORIGIN + duration) + write_cost_run( + project_dir=tmp_path, + record=replace( + record, + started_at=_ORIGIN, + cost=replace(record.cost, status=CostStatus.PENDING, resources=(), query_count=0), + ), + ) + write_snowflake_project(tmp_path) + history: _ClosingHistory = _ClosingHistory( + rows=rows, + timeout_window=timedelta(minutes=10), + seconds_per_read=test_case.seconds_per_read, + ) + adapter: SnowflakeAdapter = SnowflakeAdapter() + monkeypatch.setattr(adapter, "connect", lambda config: history) + monkeypatch.setattr("time.monotonic", history.clock.monotonic) + monkeypatch.setattr( + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", + lambda **kwargs: adapter, + ) + + refresh_pending_cost_run( + CostCommandRequest( + project_dir=tmp_path, + selector=_RUN_ID, + no_color=True, + limit=None, + no_limit=False, + sort=None, + order=None, + since=None, + until=None, + json_output=False, + json_output_path=None, + ) + ) + + refreshed: CostRunRecord | None = read_cost_run(project_dir=tmp_path, run_id=_RUN_ID) + assert refreshed is not None + assert refreshed.cost.status == test_case.expected_status + assert refreshed.cost.query_count == len(rows) + assert history.clock.seconds > 30 diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 67f78c8c4b..be67c4acfe 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -7,6 +7,15 @@ from sqlbuild._native import SqlLexicalSyntax +@dataclass(frozen=True) +class InlineDirectiveTestCase: + """One model with an inline suppression directive above a query.""" + + description: str + query: str + expected_violations: tuple[tuple[str, int, str], ...] + + @dataclass(frozen=True) class LintBehaviorTestCase: """Identity for one non-tabular lint behavior test.""" diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index a9e9193d7a..e93b623929 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -11,7 +11,7 @@ LintRunResult, LintSeverity, LintViolation, - apply_lint_suppressions, + reject_lint_suppressions, run_lint, ) from tests.unit.src.sqlbuild.lint._test_types import ( @@ -19,6 +19,7 @@ FormatDescriptionResolutionTestCase, FormatNewlineTestCase, FormatProjectTestCase, + InlineDirectiveTestCase, LintBehaviorTestCase, LintProjectTestCase, ) @@ -407,46 +408,24 @@ def test_given_fixture_ceremony_variants_when_formatting_and_linting_then_forms_ assert linted.violations == () -@pytest.mark.parametrize( - "test_case", - [LintBehaviorTestCase(description="reasoned matching local suppression", expected_value=())], - ids=lambda case: case.description, -) -def test_given_reasoned_local_suppression_when_linting_then_matching_warning_is_removed( - test_case: LintBehaviorTestCase, - tmp_path: Path, -) -> None: - _ = test_case - _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - target: Path = tmp_path / "models" / "sample.sql" - target.parent.mkdir() - _ = target.write_text( - 'MODEL (description "ok");\n' - "-- sqb: ignore SQBRSQL004 because this fixture intentionally samples one row\n" - "SELECT value FROM items LIMIT 1\n", - encoding="utf-8", - ) - - result: LintRunResult = run_lint( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) - - assert result.violations == test_case.expected_value - - @pytest.mark.parametrize( "test_case", [ - LintBehaviorTestCase( - description="forbidden suppression keeps the finding and fails", - expected_value=(("SQBRSQL000", 2, "fault"), ("SQBRSQL004", 3, "warning")), - ) + InlineDirectiveTestCase( + description="matching directive keeps the finding and fails", + query="SELECT value FROM items LIMIT 1\n", + expected_violations=(("SQBRSQL000", 2, "fault"), ("SQBRSQL004", 3, "warning")), + ), + InlineDirectiveTestCase( + description="directive without a finding still fails", + query="SELECT value FROM items ORDER BY value LIMIT 1\n", + expected_violations=(("SQBRSQL000", 2, "fault"),), + ), ], ids=lambda case: case.description, ) -def test_given_suppressions_forbidden_when_linting_then_directive_is_a_fault_and_suppresses_nothing( - test_case: LintBehaviorTestCase, +def test_given_inline_directive_when_linting_then_it_is_a_fault_printing_the_exception_entry( + test_case: InlineDirectiveTestCase, tmp_path: Path, ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") @@ -455,73 +434,42 @@ def test_given_suppressions_forbidden_when_linting_then_directive_is_a_fault_and _ = target.write_text( 'MODEL (description "ok");\n' "-- sqb: ignore SQBRSQL004 because this fixture intentionally samples one row\n" - "SELECT value FROM items LIMIT 1\n", + f"{test_case.query}", encoding="utf-8", ) result: LintRunResult = run_lint( project_dir=tmp_path, - config=LintConfig(dialect="duckdb", allow_suppressions=False), + config=LintConfig(dialect="duckdb"), ) assert ( tuple((item.code, item.line, str(item.severity)) for item in result.violations) - == test_case.expected_value + == test_case.expected_violations ) remediation: str = result.violations[0].remediation or "" - assert "sqlbuild_project.toml sets [rules] allow_exceptions = false" in remediation - assert "[rules]\n allow_exceptions = true" in remediation - - -@pytest.mark.parametrize( - "test_case", - [ - LintBehaviorTestCase( - description="unused local suppression", - expected_value="Unused suppression for SQBRSQL004", - ) - ], - ids=lambda case: case.description, -) -def test_given_unused_local_suppression_when_linting_then_reports_suppression_warning( - test_case: LintBehaviorTestCase, - tmp_path: Path, -) -> None: - _ = test_case - _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - target: Path = tmp_path / "models" / "ordered.sql" - target.parent.mkdir() - _ = target.write_text( - 'MODEL (description "ok");\n' - "-- sqb: ignore SQBRSQL004 because this query used to sample one row\n" - "SELECT value FROM items ORDER BY value LIMIT 1\n", - encoding="utf-8", - ) - - result: LintRunResult = run_lint( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) - - assert tuple(item.code for item in result.violations) == ("SQBRSQL000",) - assert result.violations[0].message == test_case.expected_value + assert ( + "[[rules.rule_exceptions]]\n" + ' rule = "SQBRSQL004"\n' + ' path = "models/sample.sql"\n' + ' reason = "this fixture intentionally samples one row"' + ) in remediation @pytest.mark.parametrize( "test_case", [ LintBehaviorTestCase( - description="mandatory fault suppression attempt", + description="mandatory fault directive", expected_value={"SQBRSQL000", "description-present"}, ) ], ids=lambda case: case.description, ) -def test_given_header_fault_suppression_when_linting_then_mandatory_fault_remains( +def test_given_header_fault_directive_when_rejecting_then_mandatory_fault_remains( test_case: LintBehaviorTestCase, tmp_path: Path, ) -> None: - _ = test_case target: Path = tmp_path / "models" / "missing_description.sql" contents: str = ( "-- sqb: ignore description-present because this must not bypass compiler policy\n" @@ -537,6 +485,10 @@ def test_given_header_fault_suppression_when_linting_then_mandatory_fault_remain engine=LINT_ENGINE_SQLBUILD, ) - result: list[LintViolation] = apply_lint_suppressions([mandatory], {target: contents}, True) + result: list[LintViolation] = reject_lint_suppressions( + [mandatory], {target: contents}, tmp_path + ) + + assert {item.code for item in result} == test_case.expected_value assert {item.code for item in result} == test_case.expected_value diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index b125b43287..ef45160c21 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -13,12 +13,13 @@ class RulesConfigErrorTestCase: @dataclass(frozen=True) -class AllowExceptionsConfigTestCase: - """One `[rules] allow_exceptions` configuration that loads.""" +class RelationExceptionConfigTestCase: + """One `[[rules.rule_exceptions]]` configuration that loads, with its relation scope.""" description: str source: str - expected_allow_exceptions: bool + expected_relation: str | None + expected_cte: str | None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 81a9a6ec02..c7558ff763 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -43,12 +43,14 @@ "SQBRDECLARATION306", "SQBRGRAPH101", "SQBRGRAPH102", + "SQBRGRAPH103", "SQBRPROJECT101", "SQBRPROJECT102", "SQBRPROJECT103", "SQBRPROJECT104", "SQBRPROJECT105", "SQBRPROJECT106", + "SQBRPROJECT107", "SQBRPROJECT201", "SQBRPROJECT202", "SQBRPROJECT203", @@ -104,12 +106,14 @@ "SQBRDECLARATION306", "SQBRGRAPH101", "SQBRGRAPH102", + "SQBRGRAPH103", "SQBRPROJECT101", "SQBRPROJECT102", "SQBRPROJECT103", "SQBRPROJECT104", "SQBRPROJECT105", "SQBRPROJECT106", + "SQBRPROJECT107", "SQBRPROJECT201", "SQBRPROJECT202", "SQBRPROJECT203", diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index adc5dd4395..ad73e6c27f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -13,8 +13,8 @@ load_rules_config, ) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( - AllowExceptionsConfigTestCase, PolicyLayoutConfigTestCase, + RelationExceptionConfigTestCase, RuleIgnoreConfigTestCase, RulesConfigErrorTestCase, SqlTestRulesConfigTestCase, @@ -166,9 +166,33 @@ expected_error_pattern="must be a normalized path relative to tests/unit", ), RulesConfigErrorTestCase( - description="non-boolean allow_exceptions", - source='[rules]\nallow_exceptions = "no"\n', - expected_error_pattern="invalid rules config", + description="relation for a Rule that reports no relation", + source=( + '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\npath = "models/orders.sql"\n' + 'relation = "customers"\nreason = "A reviewed sample"\n' + ), + expected_error_pattern="sets relation, but only SQBRSQL038 report a joined relation", + ), + RulesConfigErrorTestCase( + description="cte without relation", + source=( + '[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "models/orders.sql"\n' + 'cte = "checks"\nreason = "Deliberate"\n' + ), + expected_error_pattern="sets cte without relation", + ), + RulesConfigErrorTestCase( + description="unknown published layer", + source='[rules.rule_options.SQBRGRAPH103]\npublished_layers = ["gold"]\n', + expected_error_pattern="published_layers entries must be model name layers", + ), + RulesConfigErrorTestCase( + description="allowed qualifier without a reason", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ qualifier = "csv_export" }]\n' + ), + expected_error_pattern="allowed_qualifiers entries must be", ), RulesConfigErrorTestCase( description="unknown SQL test rules key", @@ -192,31 +216,21 @@ def test_given_invalid_rules_config_when_loading_then_raises_clear_error( "test_case", ( RulesConfigErrorTestCase( - description="exception entry", + description="false", source=f"[rules]\nallow_exceptions = false\n{_EXCEPTION_ENTRY}", expected_error_pattern=( - r"has 1 \[\[rules\.rule_exceptions\]\] entry, which this project does not " - r"allow; sqlbuild_project\.toml sets \[rules\] allow_exceptions = false" + r"sets \[rules\] allow_exceptions, which SQLBuild no longer supports" ), ), RulesConfigErrorTestCase( - description="scoped ignore entry", - source=f"[rules]\nallow_exceptions = false\n{_IGNORE_ENTRY}", - expected_error_pattern=r"has 1 \[\[rules\.rule_ignores\]\] entry,", - ), - RulesConfigErrorTestCase( - description="exceptions and scoped ignores", - source=f"[rules]\nallow_exceptions = false\n{_EXCEPTION_ENTRY}{_EXCEPTION_ENTRY}" - f"{_IGNORE_ENTRY}", - expected_error_pattern=( - r"has 2 \[\[rules\.rule_exceptions\]\] entries and " - r"1 \[\[rules\.rule_ignores\]\] entry," - ), + description="true", + source="[rules]\nallow_exceptions = true\n", + expected_error_pattern=r"inline -- sqb: ignore suppressions were removed", ), ), ids=lambda case: case.description, ) -def test_given_exceptions_forbidden_when_loading_then_raises_error_with_exact_setting_help( +def test_given_removed_allow_exceptions_when_loading_then_raises_error_explaining_the_removal( tmp_path: Path, test_case: RulesConfigErrorTestCase, ) -> None: @@ -225,42 +239,42 @@ def test_given_exceptions_forbidden_when_loading_then_raises_error_with_exact_se with pytest.raises(RulesConfigError, match=test_case.expected_error_pattern) as raised: load_rules_config(project_dir=tmp_path) - assert raised.value.help.endswith( - "set this in sqlbuild_project.toml:\n [rules]\n" - " allow_exceptions = true" - ) + assert "remove the allow_exceptions line from [rules]" in raised.value.help + assert "[[rules.rule_exceptions]]" in raised.value.help @pytest.mark.parametrize( "test_case", ( - AllowExceptionsConfigTestCase( - description="absent setting allows exceptions", + RelationExceptionConfigTestCase( + description="file-wide exception and scoped ignore", source=_EXCEPTION_ENTRY + _IGNORE_ENTRY, - expected_allow_exceptions=True, - ), - AllowExceptionsConfigTestCase( - description="true allows exceptions", - source=f"[rules]\nallow_exceptions = true\n{_EXCEPTION_ENTRY}{_IGNORE_ENTRY}", - expected_allow_exceptions=True, + expected_relation=None, + expected_cte=None, ), - AllowExceptionsConfigTestCase( - description="false with only the project-wide ignore list", - source='[rules]\nallow_exceptions = false\nignore = ["SQBRSQL004"]\n', - expected_allow_exceptions=False, + RelationExceptionConfigTestCase( + description="relation-level exception narrowed to a CTE", + source=( + '[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "models/orders.sql"\n' + 'relation = "order_policy"\ncte = "order_checks"\n' + 'reason = "Every order is checked against every policy"\n' + ), + expected_relation="order_policy", + expected_cte="order_checks", ), ), ids=lambda case: case.description, ) -def test_given_allow_exceptions_setting_when_loading_then_config_records_it( +def test_given_rule_exceptions_when_loading_then_config_records_their_scope( tmp_path: Path, - test_case: AllowExceptionsConfigTestCase, + test_case: RelationExceptionConfigTestCase, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text(test_case.source, encoding="utf-8") config: RulesConfig = load_rules_config(project_dir=tmp_path) - assert config.allow_exceptions is test_case.expected_allow_exceptions + assert config.rule_exceptions[0].relation == test_case.expected_relation + assert config.rule_exceptions[0].cte == test_case.expected_cte @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index 577e9b4db1..5b534e4ed2 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -11,6 +11,9 @@ class EventExporterDispatcherTestCase: class EventExporterTeardownTestCase: description: str expected_teardown_count: int + release_after_seconds: float + min_close_seconds: float + max_close_seconds: float @dataclass(frozen=True) @@ -47,3 +50,14 @@ class InterpreterExitTestCase: description: str script: str expected_returncode: int + expected_stderr: str = "" + + +@dataclass(frozen=True) +class OutcomeDeliveryTestCase: + description: str + queue_capacity: int + backlog: int + sink_seconds: float + shutdown_timeout_seconds: float + expected_outcomes: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py index af644e7136..b62bc6c831 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py @@ -1,6 +1,6 @@ from datetime import UTC, datetime from pathlib import Path -from threading import Event +from threading import Event, current_thread from typing import ClassVar from sqlbuild._native import ( @@ -19,6 +19,7 @@ class BlockingProvider(Provider): release_export: ClassVar[Event] = Event() teardown_finished: ClassVar[Event] = Event() teardown_count: ClassVar[int] = 0 + teardown_threads: ClassVar[list[str]] = [] @classmethod def reset(cls) -> None: @@ -26,6 +27,7 @@ def reset(cls) -> None: cls.release_export.clear() cls.teardown_finished.clear() cls.teardown_count = 0 + cls.teardown_threads = [] def export(self) -> None: self.entered.set() @@ -33,6 +35,7 @@ def export(self) -> None: def teardown(self) -> None: type(self).teardown_count += 1 + type(self).teardown_threads.append(current_thread().name) self.teardown_finished.set() diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py index 4f40adb060..fb66a4c64c 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py @@ -3,7 +3,7 @@ import time from datetime import UTC, datetime from pathlib import Path -from threading import Event +from threading import Event, Timer, current_thread import pytest @@ -31,10 +31,13 @@ @pytest.mark.parametrize( "test_case", - (EventExporterTeardownTestCase("deferred provider teardown", 1),), + ( + EventExporterTeardownTestCase("sink returns during the idle wait", 1, 0.1, 0.08, 0.9), + EventExporterTeardownTestCase("sink still hung after the idle wait", 1, 30.0, 0.9, 3.0), + ), ids=lambda case: case.description, ) -def test_given_live_timed_out_exporter_when_scope_closes_then_provider_waits_for_invocation( +def test_given_live_timed_out_exporter_when_scope_closes_then_provider_tears_down_on_closing_thread( test_case: EventExporterTeardownTestCase, tmp_path: Path, ) -> None: @@ -54,16 +57,19 @@ def test_given_live_timed_out_exporter_when_scope_closes_then_provider_waits_for ) dispatcher.enqueue(lifecycle_event()) assert BlockingProvider.entered.wait(timeout=0.2) + release: Timer = Timer(test_case.release_after_seconds, BlockingProvider.release_export.set) + release.start() started: float = time.monotonic() summary: EventExportSummary = scope.close() - assert time.monotonic() - started < 0.2 - assert summary.failed == 1 - assert BlockingProvider.teardown_count == 0 + elapsed: float = time.monotonic() - started + release.cancel() BlockingProvider.release_export.set() - assert BlockingProvider.teardown_finished.wait(timeout=0.2) + assert test_case.min_close_seconds <= elapsed < test_case.max_close_seconds + assert summary.failed == 1 assert BlockingProvider.teardown_count == test_case.expected_teardown_count + assert BlockingProvider.teardown_threads == [current_thread().name] _ = scope.close() assert BlockingProvider.teardown_count == test_case.expected_teardown_count diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 0135132e6e..0eb28211fb 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -2,8 +2,10 @@ import threading import time +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from queue import SimpleQueue from typing import cast import pytest @@ -18,13 +20,20 @@ ) from sqlbuild.observability import LifecycleEvent from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError +from sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink import ( + invoke_lifecycle_event_sink, +) from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterDispatcherTestCase, HealthIntervalTestCase, + OutcomeDeliveryTestCase, +) +from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import ( + correlated_event, + lifecycle_event, ) -from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import lifecycle_event @pytest.mark.parametrize( @@ -118,7 +127,7 @@ def publish(*, event: LifecycleEvent) -> None: (EventExporterDispatcherTestCase("failure isolation", 1),), ids=lambda case: case.description, ) -def test_given_multiple_exporters_when_one_fails_then_later_exporter_runs_in_order( +def test_given_multiple_exporters_when_one_fails_then_the_other_exporter_still_runs( test_case: EventExporterDispatcherTestCase, ) -> None: calls: list[str] = [] @@ -142,7 +151,7 @@ def succeeding(*, event: LifecycleEvent) -> None: summary: EventExportSummary = dispatcher.shutdown() - assert calls == ["first", "second"] + assert sorted(calls) == ["first", "second"] assert summary.delivered == test_case.expected_delivered assert summary.failed == 1 assert summary.accepted == summary.delivered + summary.failed + summary.dropped @@ -243,7 +252,7 @@ def publish(*, event: LifecycleEvent) -> None: invocation_timeout_seconds=0.2, ) for index in range(100): - dispatcher.enqueue(lifecycle_event(index)) + dispatcher.enqueue(correlated_event(index, "statement_submitted")) time.sleep(0.02) release.set() @@ -259,25 +268,26 @@ def publish(*, event: LifecycleEvent) -> None: (EventExporterDispatcherTestCase("last live invocation finalizes once", 0),), ids=lambda case: case.description, ) -def test_given_multiple_timed_out_invocations_when_registering_finalizer_then_last_return_runs_once( +def test_given_multiple_timed_out_invocations_when_finalizing_then_runs_once_on_caller_after_last( test_case: EventExporterDispatcherTestCase, ) -> None: first_entered: threading.Event = threading.Event() second_entered: threading.Event = threading.Event() first_release: threading.Event = threading.Event() second_release: threading.Event = threading.Event() - finalized: threading.Event = threading.Event() - finalizer_calls: list[str] = [] + calls: list[str] = [] def first(*, event: LifecycleEvent) -> None: del event first_entered.set() first_release.wait() + calls.append("first returned") def second(*, event: LifecycleEvent) -> None: del event second_entered.set() second_release.wait() + calls.append("second returned") dispatcher: EventExporterDispatcher = EventExporterDispatcher( exporters=( @@ -285,24 +295,29 @@ def second(*, event: LifecycleEvent) -> None: BoundEventExporter("second", second, {}), ), shutdown_timeout_seconds=0.1, - invocation_timeout_seconds=0.02, + invocation_timeout_seconds=0.3, ) dispatcher.enqueue(lifecycle_event()) - assert first_entered.wait(timeout=0.2) - assert second_entered.wait(timeout=0.2) + assert first_entered.wait(timeout=1.0) + assert second_entered.wait(timeout=1.0) summary: EventExportSummary = dispatcher.shutdown() def finalize() -> None: - finalizer_calls.append("finalized") - finalized.set() + calls.append(f"finalized on {threading.current_thread().name}") + releases: tuple[threading.Timer, ...] = ( + threading.Timer(0.05, first_release.set), + threading.Timer(0.1, second_release.set), + ) + for release in releases: + release.start() dispatcher.finalize_when_idle(finalize) - first_release.set() - assert not finalized.wait(timeout=0.02) - second_release.set() - assert finalized.wait(timeout=0.2) - assert finalizer_calls == ["finalized"] + assert calls == [ + "first returned", + "second returned", + f"finalized on {threading.current_thread().name}", + ] assert summary.delivered == test_case.expected_delivered @@ -500,19 +515,19 @@ def second(*, event: LifecycleEvent) -> None: queue_capacity=1, invocation_timeout_seconds=1, ) - dispatcher.enqueue(lifecycle_event(1)) + dispatcher.enqueue(correlated_event(1, "statement_submitted")) assert entered.wait(timeout=0.2) - dispatcher.enqueue(lifecycle_event(2)) - dispatcher.enqueue(lifecycle_event(3, "invocation_failed")) + dispatcher.enqueue(correlated_event(2, "statement_submitted")) + dispatcher.enqueue(correlated_event(3, "operation_failed")) release.set() summary: EventExportSummary = dispatcher.shutdown() assert len(calls) == test_case.expected_delivered - assert calls == [ - ("first", "invocation_started"), - ("first", "invocation_failed"), - ("second", "invocation_failed"), + assert sorted(calls) == [ + ("first", "operation_failed"), + ("first", "statement_submitted"), + ("second", "operation_failed"), ] first_counts: EventExporterCounts = summary.per_exporter[0].counts second_counts: EventExporterCounts = summary.per_exporter[1].counts @@ -562,18 +577,24 @@ def test_given_sensitive_exporter_exception_when_reporting_then_only_safe_dimens test_case: EventExporterDispatcherTestCase, ) -> None: failures: list[object] = [] + reported: threading.Event = threading.Event() def failing(*, event: LifecycleEvent) -> None: del event raise RuntimeError("password=secret sql=select-sensitive destination=broker") + def report(failure: object) -> None: + failures.append(failure) + reported.set() + dispatcher: EventExporterDispatcher = EventExporterDispatcher( exporters=(BoundEventExporter("safe_name", failing, {}),), - failure_callback=failures.append, + failure_callback=report, ) dispatcher.enqueue(lifecycle_event()) dispatcher.shutdown() + assert reported.wait(timeout=1.0) assert len(failures) == test_case.expected_delivered rendered: str = repr(failures[0]) assert "RuntimeError" in rendered @@ -735,3 +756,137 @@ def summary_callback(summary: EventExportSummary) -> None: assert len(exported) == test_case.expected_delivered assert summaries[0].accepted == 1 dispatcher.shutdown() + + +@pytest.mark.parametrize( + "test_case", + ( + OutcomeDeliveryTestCase( + "saturated queue still admits outcomes", + queue_capacity=4, + backlog=100, + sink_seconds=0.0, + shutdown_timeout_seconds=5.0, + expected_outcomes=("run_completed", "invocation_completed"), + ), + OutcomeDeliveryTestCase( + "tight budget drops backlog for outcomes", + queue_capacity=1024, + backlog=300, + sink_seconds=0.02, + shutdown_timeout_seconds=0.4, + expected_outcomes=("run_completed", "invocation_completed"), + ), + ), + ids=lambda case: case.description, +) +def test_given_backlog_when_shutting_down_then_outcome_events_arrive_and_backlog_is_dropped( + test_case: OutcomeDeliveryTestCase, +) -> None: + held: threading.Event = threading.Event() + release: threading.Event = threading.Event() + delivered: list[str] = [] + + def record(*, event: LifecycleEvent) -> None: + delivered.append(event.event_type) + held.set() + _ = release.wait() + time.sleep(test_case.sink_seconds) + + dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("recorder", record, {}),), + queue_capacity=test_case.queue_capacity, + shutdown_timeout_seconds=test_case.shutdown_timeout_seconds, + invocation_timeout_seconds=5.0, + ) + dispatcher.enqueue(correlated_event(0, "run_started")) + assert held.wait(timeout=1.0) + for index in range(test_case.backlog): + dispatcher.enqueue(correlated_event(index + 1, "statement_completed")) + for offset, event_type in enumerate(test_case.expected_outcomes): + dispatcher.enqueue(correlated_event(test_case.backlog + 1 + offset, event_type)) + release.set() + + summary: EventExportSummary = dispatcher.shutdown() + + assert ( + tuple(filter(test_case.expected_outcomes.__contains__, delivered)) + == test_case.expected_outcomes + ) + assert delivered[-1] == "invocation_completed" + assert summary.dropped > 0 + assert summary.delivered + summary.dropped + summary.failed == summary.accepted + + +@pytest.mark.parametrize( + "test_case", + (EventExporterDispatcherTestCase("second call started before the reserve", 4),), + ids=lambda case: case.description, +) +def test_given_lane_running_its_second_call_when_reserve_begins_then_it_counts_delivered( + test_case: EventExporterDispatcherTestCase, +) -> None: + event_ids: tuple[str, ...] = ("event-1", "event-2", "event-3", "event-4") + started: dict[str, threading.Event] = {event_id: threading.Event() for event_id in event_ids} + gates: dict[str, threading.Event] = {event_id: threading.Event() for event_id in event_ids} + for event_id in ("event-1", "event-3", "event-4"): + gates[event_id].set() + release_second: threading.Event = gates["event-2"] + second_started: threading.Event = started["event-2"] + delivered: list[str] = [] + + def record(*, event: LifecycleEvent) -> None: + delivered.append(event.event_id) + started[event.event_id].set() + _ = gates[event.event_id].wait() + + dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("recorder", record, {}),), + shutdown_timeout_seconds=2.0, + invocation_timeout_seconds=5.0, + ) + dispatcher.enqueue(correlated_event(1, "statement_completed")) + dispatcher.enqueue(correlated_event(2, "statement_completed")) + dispatcher.enqueue(correlated_event(3, "run_completed")) + dispatcher.enqueue(lifecycle_event(4, "invocation_completed")) + assert second_started.wait(timeout=1.0) + release: threading.Timer = threading.Timer(1.7, release_second.set) + release.start() + + summary: EventExportSummary = dispatcher.shutdown() + + release.cancel() + assert tuple(delivered) == event_ids + assert summary.delivered == test_case.expected_delivered + assert (summary.dropped, summary.failed) == (0, 0) + + +@pytest.mark.parametrize( + "test_case", + (EventExporterDispatcherTestCase("claimed call runs, cancelled call is skipped", 1),), + ids=lambda case: case.description, +) +def test_given_claims_when_delivery_thread_runs_calls_then_only_unclaimed_cancellations_skip( + test_case: EventExporterDispatcherTestCase, +) -> None: + calls: SimpleQueue[ + tuple[Callable[..., object], Mapping[str, object], LifecycleEvent, int] | None + ] = SimpleQueue() + results: SimpleQueue[tuple[int, BaseException | None]] = SimpleQueue() + claims: dict[int, bool] = {2: False} + seen: list[str] = [] + + def record(*, event: LifecycleEvent) -> None: + seen.append(event.event_id) + assert claims.setdefault(1, False) + + calls.put((record, {}, correlated_event(1, "statement_completed"), 1)) + calls.put((record, {}, correlated_event(2, "statement_completed"), 2)) + calls.put(None) + + invoke_lifecycle_event_sink(calls=calls, results=results, claims=claims) + + assert seen == ["event-1"] + assert results.qsize() == test_case.expected_delivered + assert results.get_nowait() == (1, None) + assert claims == {1: True} diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py index f73489d8f4..3ccb374c2c 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py @@ -69,6 +69,35 @@ def slow(*, event): ), 0, ), + InterpreterExitTestCase( + "provider teardown waiting for a timed-out sink", + dedent( + """ + from types import SimpleNamespace + + from sqlbuild._native import BoundEventExporter, EventExporterDispatcher + + def slow(*, event): + time.sleep(0.3) + + def teardown(): + time.sleep(0.5) + print("teardown finished", file=sys.stderr, flush=True) + + dispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("slow", slow, {}),), + shutdown_timeout_seconds=0.0, + ) + dispatcher.enqueue(SimpleNamespace(event_type="statement_submitted")) + time.sleep(0.05) + dispatcher.shutdown() + dispatcher.finalize_when_idle(teardown) + time.sleep(0.35) + """ + ), + 0, + "teardown finished", + ), InterpreterExitTestCase( "dispatcher never shut down", dedent( @@ -102,3 +131,4 @@ def test_given_live_exporter_threads_when_interpreter_finalizes_then_process_exi assert result.returncode == test_case.expected_returncode, result.stderr assert "finalizing True" in result.stderr + assert test_case.expected_stderr in result.stderr diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py index 10cb6afcb4..dae71eb80e 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py @@ -25,7 +25,11 @@ PriorityPairTestCase( "3 then 2", "operation_failed", "statement_completed", ("event-1",), 1 ), - PriorityPairTestCase("3 then 3", "operation_failed", "run_failed", ("event-1",), 1), + PriorityPairTestCase("3 then 3", "operation_failed", "statement_failed", ("event-1",), 1), + PriorityPairTestCase("3 then outcome 3", "operation_failed", "run_failed", ("event-2",), 1), + PriorityPairTestCase( + "outcome 2 then 3", "run_completed", "operation_failed", ("event-1",), 1 + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/cli/query.mdx b/website/src/content/docs/docs/cli/query.mdx index 388a675e77..94128a39b7 100644 --- a/website/src/content/docs/docs/cli/query.mdx +++ b/website/src/content/docs/docs/cli/query.mdx @@ -21,6 +21,8 @@ sqb --project-dir query --file my_query.sql File paths are resolved from the current working directory. +`sqb query` reads only `sqlbuild_project.toml` and `sqlbuild_local.toml` to connect. It does not parse models or import project Python, so it starts quickly in large projects and still runs when a model or Python file is broken. The SQL runs as written: SQLBuild does not resolve model names in it. + ## Flags | Flag | Description | diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 76d70017ad..829697736e 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -118,10 +118,20 @@ contract rate. Use `sqb cost`, `sqb cost latest`, `sqb cost `, or provide a versioned, decimal-safe output contract. Pending detail records are refreshed from Snowflake when inspected again. +SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` for the whole run in one query when the run +finishes. Each read has a 10-second statement timeout; a window that reaches it, or that returns +more rows than one read allows, is split in half and read again. Collection at the end of a command +has a 30-second budget so the command exits promptly. If history cannot be read within it, the +build is unaffected and its cost is recorded as `pending`, with a message naming the timeout. +`sqb cost ` collects it later with a 5-minute budget, which completes long runs. Any other +failure is recorded as `collection_failed` with the Snowflake error that caused it. + These values are attributed compute credits and estimated cost, not Snowflake-billed credits or invoice reconciliation. The estimate uses only query history visible to the executing role and does not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60-second minimum, -cloud-services credits, contract adjustments, or multi-cluster billing. Run metadata and query IDs +cloud-services credits, contract adjustments, or multi-cluster billing. A statement run with a `QUERY_TAG` +of your own is not recognised as a metadata-only `CREATE`, so it may report missing warehouse +metadata. Run metadata and query IDs are stored under `target/executions//`; that statement ledger stores only an SQL digest, not SQL text. Executed SQL artifacts are stored separately under the sensitive `target/run/` tree. diff --git a/website/src/content/docs/docs/concepts/observability/sinks.mdx b/website/src/content/docs/docs/concepts/observability/sinks.mdx index c5c19e20a8..f58c00658b 100644 --- a/website/src/content/docs/docs/concepts/observability/sinks.mdx +++ b/website/src/content/docs/docs/concepts/observability/sinks.mdx @@ -79,7 +79,10 @@ min_severity = "warning" `shutdown_timeout` sets how long a command waits at exit for lifecycle sinks to drain queued events. It is a fixed duration from `"0s"` to `"10m"`, defaults to `"2s"`, and is accepted only under `[sinks.lifecycle]`, not per named sink. `"0s"` drops anything still queued at exit. The -setting does not change the one-second timeout for each sink call. +final quarter of the budget is reserved for run and invocation outcomes: when it begins, other +queued events are dropped and counted so the outcomes still arrive. The setting does not change the one-second timeout for each sink call. A sink call still running when +the budget ends gets up to that one second to return before providers are torn down; teardown +always runs on the command's own thread. ```toml [sinks.lifecycle] @@ -92,15 +95,21 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. ## Delivery behavior -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority. +- Each sink receives events in order on its own long-lived delivery thread. +- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; + `invocation_completed` and `invocation_failed` come after every other queued event. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. + Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, + `invocation_completed`, `invocation_failed`) are always admitted and never displaced. - A failing or timed-out sink is isolated from command correctness. - `audit_completed` facts are published as each audit finishes (model audits in `sqb build` when their model finishes), not in one burst at the end of the run. - When lifecycle events are dropped or fail, the command prints one warning to stderr with the counts and affected sinks, for example `Warning: command completed successfully, but lifecycle event export was incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). Increase sinks.lifecycle.shutdown_timeout or check sink health.` - The exit code and stdout, including `--json` output, are unchanged. + A successful command then ends with + `sqb audit completed successfully; only its lifecycle event export is affected.` The exit code + and stdout, including `--json` output, are unchanged. - Provider setup and declaration errors fail before execution because the project configuration is invalid. - Providers shared by multiple sinks are set up and torn down once per command. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index d2badeca2e..22e328e7ab 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -163,6 +163,7 @@ A target is a named build context - the database and schema you build into, plus | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | | `warehouses` | Default Snowflake warehouse per command group, `build` and `query` (see below) | +| `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | ```toml [connections.warehouse] @@ -230,6 +231,33 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Target concurrency + +Targets often run on very different warehouses. A target can set its own worker limit, so local +and CI runs use many workers while production stays within what its warehouse runs without +queuing: + +```toml +[settings] +concurrency = 16 + +[targets.prod] +concurrency = 8 +``` + +`sqb build`, `sqb test`, `sqb audit`, `sqb seed`, `sqb load`, and `sqb scenario test` use the first +value set in this order: + +1. `--concurrency` on the command line, for that invocation only +2. The selected target's `concurrency`, from `[targets.]` in either file +3. `[settings] concurrency` in `sqlbuild_local.toml` +4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) + +The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build +--verbose` shows the effective limit and the setting it came from, for example +`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and +uses the active target's value. + ### Command-group warehouses Builds and ad-hoc inspection usually need different compute. A Snowflake target can choose a @@ -638,7 +666,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](/docs/concepts/models/configuration#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; `--concurrency` overrides it | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](/docs/concepts/python-nodes/loaders). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | @@ -690,11 +718,16 @@ select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See -[Compiler-integrated Rules](/docs/concepts/rules) for configuration, authoring, and suppressions. +[Compiler-integrated Rules](/docs/concepts/rules) for configuration, authoring, and exceptions. + +| Key | Description | +| --- | --- | +| `ignore` | Rule codes or families turned off for the whole project. | +| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQBRSQL038`. Stale entries fail compilation. See [Findings and exceptions](/docs/concepts/rules/findings-and-exceptions). | +| `[[rules.rule_ignores]]` | Rules turned off for `paths` or `selectors`, with a `reason`. | -| Key | Default | Description | -| --- | --- | --- | -| `allow_exceptions` | `true` | When `false`, `[[rules.rule_exceptions]]` and `[[rules.rule_ignores]]` entries are configuration errors and inline `-- sqb: ignore` directives are `SQBRSQL000` errors. The project-wide `ignore` list stays allowed. See [Forbidding exceptions](/docs/concepts/rules/findings-and-exceptions#forbidding-exceptions). | +Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSQL000` errors +that print the entry to add, and the former `allow_exceptions` setting is a configuration error. ## Scopes diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index ddaf3b5a6d..2b66832607 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -199,10 +199,10 @@ need to execute SQL against controlled data. Use an audit when the answer depend A finding contains a stable code, project-relative path, line, column, explanation, and remediation. This makes the same requirement usable in a terminal, JSON output, CI annotation, or agent workflow. -Intentional departures remain explicit. Exact exceptions and path- or resource-scoped ignores -require a reason; exact exceptions are stale-checked so obsolete suppressions do not silently -accumulate. Mandatory compiler correctness cannot be suppressed. A project can forbid exceptions, -scoped ignores and inline suppressions entirely with `[rules] allow_exceptions = false`. +Intentional departures remain explicit and live only in `sqlbuild_project.toml`. Exact and +relation-level exceptions and path- or resource-scoped ignores require a reason; exceptions are +stale-checked so obsolete entries do not silently accumulate. There are no inline suppression +comments, and mandatory compiler correctness cannot be excepted at all. See [Findings and exceptions](/docs/concepts/rules/findings-and-exceptions) for configuration examples. @@ -229,6 +229,9 @@ and [Execution and caching](/docs/concepts/rules/execution-and-caching) for the Understand diagnostics, exact exceptions, path ignores, and stale checks. + + Keep each domain's published layer free of consumer-specific variants. + Learn evaluation order, focused execution, and dependency-aware reuse. diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 3ef7a3e73e..070b4e38f4 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -61,6 +61,10 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE `min_tests_per_model`. Entries that name no existing test are reported as stale. See [Empty-input tests](/docs/concepts/testing#empty-input-tests). +`SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts +`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +[Published-layer Rules](/docs/concepts/rules/published-layer-rules). + Unknown Rule codes, option names, and invalid option values fail configuration. ## Focused selection diff --git a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx index 5928733c12..5ce22f0539 100644 --- a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx +++ b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx @@ -6,6 +6,10 @@ description: "Understand Rule diagnostics and record intentional exceptions safe A Rule finding identifies an exact code, project-relative path, line, column, message, and remediation. Error findings make compilation fail and prevent artifact completion. +Every deliberate deviation from a selected Rule lives in `sqlbuild_project.toml`, never in SQL +comments. A change to the project file is something a reviewer stops to look at; a comment in a +large SQL diff is not. + ## Exact exceptions Use an exact exception for one known finding. Exceptions require a reason and are stale-checked: @@ -20,6 +24,38 @@ reason = "This example intentionally demonstrates one sampled row." An exception refers to one exact Rule code and path. If the finding disappears, SQLBuild reports the stale exception so obsolete configuration does not accumulate silently. +## Relation-level exceptions + +A file-wide exception for a deliberate cartesian product would also hide any accidental cross join +added to the same file later. `SQBRSQL038` findings name the joined relation and, inside a CTE, +that CTE, so an exception can cover exactly one join: + +```toml +[[rules.rule_exceptions]] +rule = "SQBRSQL038" +path = "models/orders/order_checks.sql" +relation = "order_policy" +cte = "policy_checks" # optional: only the join inside this top-level CTE +reason = "Every order is evaluated against every policy check." +``` + +`relation` is the joined relation as written, with `__ref`, `__seed` and `__source` calls resolved +to the referenced name. Without `cte`, the exception covers that relation anywhere in the file. +Relation-level exceptions are stale-checked like exact exceptions: an entry that matches no finding +fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be +combined with relation-level exceptions for the same code and path. + +A join that reads no named relation, such as `CROSS JOIN UNNEST(...)` or `CROSS JOIN LATERAL`, has +no relation to name. Its finding says so and prints the file-wide exception (rule and path) that is +the only way to cover it. + +`SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most +intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a +declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an +aggregate without `GROUP BY`, a one-row `VALUES`, `LIMIT 1`, or `QUALIFY ROW_NUMBER() OVER (...) = 1` +without a partition, carried through CTEs and subqueries that only filter or project. A join, +`UNNEST`/`FLATTEN`, set operation or `GROUP BY` in between stops the proof, and the join is reported. + ## Path- and resource-scoped ignores Use a path ignore when a documented project area intentionally follows a different convention: @@ -56,36 +92,35 @@ reason = "These intermediate SQL files intentionally preserve upstream columns." `paths` and `selectors` may be combined in one scoped ignore. Both forms require a reason. -## Inline suppressions +To turn a Rule off for the whole project, add its code to `[rules] ignore`. -Suppress one SQL Rule finding on the next SQL line with a reasoned comment: - -```sql --- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row -SELECT customer_id FROM customers LIMIT 1 -``` +## No inline suppressions -A directive without a reason, or one that no longer matches a finding, is reported as -`SQBRSQL000`. +SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` or `/* sqb: ignore ... */` +comment is a `SQBRSQL000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +exact entry to add instead: -## Forbidding exceptions +```text +error[SQBRSQL000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml + --> models/orders/order_checks.sql:4:1 + = help: Remove this comment, then fix the SQBRSQL038 finding, or record it as a reviewed exception in sqlbuild_project.toml: + [[rules.rule_exceptions]] + rule = "SQBRSQL038" + path = "models/orders/order_checks.sql" + relation = "order_policy" + reason = "every order meets every policy" +``` -Set `allow_exceptions = false` to require that every selected Rule passes everywhere: +Copy the entry into `sqlbuild_project.toml`, adjust the reason for a reviewer, and delete the +comment. -```toml -[rules] -select = ["SQBRSQL", "SQBRMODEL"] -ignore = ["SQBRSQL004"] -allow_exceptions = false # default true -``` +## Removed settings -With exceptions forbidden, any `[[rules.rule_exceptions]]` or `[[rules.rule_ignores]]` entry fails -configuration loading with an error that counts the entries and shows the setting to change, so -`sqb compile` and `sqb rules run` stop before evaluating Rules. Each inline `-- sqb: ignore` -directive becomes a `SQBRSQL000` error at the directive and suppresses nothing, so the finding it -was hiding is reported too. The project-wide `ignore` list stays allowed: it is the adoption switch -for a Rule across the whole project, not a scoped escape hatch. `[[rules.graph_edge_exceptions]]`, -`[[rules.select_star_allow]]` and `[[rules.threshold_overrides]]` are not affected. +`[rules] allow_exceptions` was removed. Configured exceptions are always allowed because the project +file is the review gate, and inline suppressions no longer exist. A project that still sets +`allow_exceptions` fails configuration loading with an error that explains the removal: delete the +line, keep reviewed deviations as `[[rules.rule_exceptions]]` entries, and use `[rules] ignore` to +turn a Rule off for the whole project. ## Mandatory correctness diff --git a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx new file mode 100644 index 0000000000..a7fcb6472b --- /dev/null +++ b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx @@ -0,0 +1,78 @@ +--- +title: "Published-layer Rules" +description: "Keep each domain's published layer free of consumer-specific variants." +--- + +A domain's published layer is its interface to the rest of the project. Under pressure to make +something work, it is tempting to add a consumer-specific variant next to an existing published +model: + +```text +sales__int_clean__orders -> sales__mart__orders -> sales__mart__orders__finance -> finance__stg_v__orders +``` + +`sales__mart__orders__finance` reshapes a published model for one consumer in another domain. That +work belongs in the consumer's domain, or in the producing domain's intermediate layer. Two +built-in Rules catch it: `SQBRPROJECT107` by its name as soon as it is created, and `SQBRGRAPH103` +by its dependency even when the name looks uniform. Together they leave one fix: logic shared within +a domain belongs in its intermediate layer. + +Both Rules read model names of the form `____[__]`. Models whose +names do not follow that grammar are not checked. Both are errors and are selected with their +families (`SQBRGRAPH`, `SQBRPROJECT`) or `SQBR`. + +## SQBRGRAPH103: no published reads within a domain + +A model in a published layer must not read another published model of its own domain. The one +built-in allowance is a view of its own table: `__mart_v__[__]` may read +`__mart__`, or `__mart____` with the same qualifier. Reading another domain's published models is the point of publishing +them and is allowed. + +```text +error[SQBRGRAPH103]: sales__mart__orders__finance reads sales__mart__orders; published models must not read published models of their own domain + --> models/sales/mart/sales__mart__orders__finance.sql:1:1 + = help: Move the shared logic into a sales__int_* model that both published models read, or move this model into the domain that consumes it. To move it into intermediate: sqb mv sales__mart__orders__finance models/sales/intermediate/enriched/sales__int_enriched__orders__finance.sql +``` + +## SQBRPROJECT107: one name shape per domain + +Within one domain's published layer, every model name has the same number of segments: either all +`____` or all `______`. The models that do +not fit the domain's majority shape are reported, with the remediation: + +1. If a model with a qualifier adapts a model without one, move that logic into intermediate or into + the consumer's domain. +2. If the domain really is split by qualifier, rename the models without one into the family, for + example `sqb rename sales__mart__customers sales__mart__customers__core`. +3. If the qualifier is intentional and structural, allowlist it with a reason. + +Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left +out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain +only: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { qualifier = "csv_export", reason = "File export views of a published entity." }, + { domain = "sales", qualifier = "fra", reason = "Published once per country." }, +] +``` + +## Published layers + +Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule +to change that; each Rule reads its own option: + +```toml +[rules.rule_options.SQBRGRAPH103] +published_layers = ["mart", "mart_v"] + +[rules.rule_options.SQBRPROJECT107] +published_layers = ["mart", "mart_v"] +``` + +## Deviating + +There are no inline suppressions. Turn a Rule off for the whole project with `[rules] ignore`, or +record one reviewed model as a `[[rules.rule_exceptions]]` entry with its path and a reason. See +[Findings and exceptions](/docs/concepts/rules/findings-and-exceptions). diff --git a/website/src/content/docs/docs/integrations/dagster.mdx b/website/src/content/docs/docs/integrations/dagster.mdx index c8b0939558..34d319eced 100644 --- a/website/src/content/docs/docs/integrations/dagster.mdx +++ b/website/src/content/docs/docs/integrations/dagster.mdx @@ -86,7 +86,7 @@ precedence; SQLBuild does not silently combine it with Dagster's implicit asset SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster asset checks: - **Unit tests** become checks attached to the models they test -- **Audits** become checks attached to the model or source they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN` +- **Audits** become checks attached to the model, source, or seed they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN`. A singular audit that runs at the end of the build becomes a check on each model it reads - **Scenarios** become checks attached to the models they exercise - **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 770ed3304d..443cd3b656 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -118,6 +118,7 @@ "docs/concepts/rules", "docs/concepts/rules/configuration-and-selection", "docs/concepts/rules/findings-and-exceptions", + "docs/concepts/rules/published-layer-rules", "docs/concepts/rules/execution-and-caching" ] },