diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f51ee10759..c75806fac3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,13 @@ Do not compare development-profile timings with release benchmarks. Dependency o changes still require `uv sync --all-extras`; `--skip-install` does not refresh installed metadata, dependencies or console scripts. +Always pass `--skip-install` to `maturin develop`, or use `make dev-native` or `make fast-native`. +Without it, maturin installs the `sqlbuild-cli` version that `pyproject.toml` pins from the package +index, not this checkout's `crates/sqlbuild-cli`: the `sqb` executable is then a released build, +and between a release commit and its publication the install fails because that version does not +exist yet. `uv sync` and `uv run` install the workspace `sqlbuild-cli` and rebuild the extension +when Rust sources change. + Before opening a pull request, run: ```bash diff --git a/Cargo.lock b/Cargo.lock index 8f8ef6301a..b7c080f05a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" +source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" +source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" +source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index a251e9b503..3747015868 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/Makefile b/Makefile index b9bf0cd60c..e1ffa034ee 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,8 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_load_sources.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_source_end_audits.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py @@ -370,9 +372,13 @@ planner-differential: skills: uv run python -m scripts.generate_docs_skill - uv run sqb skills --global --target opencode uv run fensu skills +# Installs this checkout's development skill into your user-level OpenCode skills; never run by CI. +.PHONY: install-global-skill +install-global-skill: + uv run sqb skills --global --target opencode + DBT_EXECUTABLE ?= dbt export DBT_EXECUTABLE diff --git a/crates/sqlbuild-analysis/src/binding_locations/constants.rs b/crates/sqlbuild-analysis/src/binding_locations/constants.rs index 9bc5d6c902..93fce95212 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/constants.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/constants.rs @@ -4,3 +4,5 @@ pub(crate) const UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; /// The severity whose repeats at a known span compile drops. pub(crate) const ERROR_SEVERITY: &str = "error"; +/// SQLite temporal-cast warnings, which every binding pass reports, so they are kept once. +pub(crate) const SQLITE_TEMPORAL_CAST_CODE: &str = "B104"; diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs index 7c717f791d..c305fbf6c4 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -2,7 +2,9 @@ use std::collections::HashSet; -use crate::binding_locations::constants::{ERROR_SEVERITY, UNKNOWN_TABLE_INTERNAL_CODE}; +use crate::binding_locations::constants::{ + ERROR_SEVERITY, SQLITE_TEMPORAL_CAST_CODE, UNKNOWN_TABLE_INTERNAL_CODE, +}; use crate::binding_locations::models::{ AuthoredSpan, BindingSource, LocatedDiagnostic, SeenDiagnostic, }; @@ -56,7 +58,10 @@ pub fn locate_binding_diagnostics( line, column, }; - if seen.contains(&key) && start.is_some() && severity == ERROR_SEVERITY { + if seen.contains(&key) + && start.is_some() + && (severity == ERROR_SEVERITY || code == SQLITE_TEMPORAL_CAST_CODE) + { continue; } seen.insert(key); diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs index 3f6e1724f9..da86454d6c 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs @@ -40,5 +40,6 @@ pub(crate) fn plain_model_facts(model: PlainModel) -> CompiledModelFacts { constant_declarations: model.constant_declarations, analysis: None, typed_calls: model.typed_calls, + expansion_passes: model.expansion_passes, } } diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index 309221dca8..455cceb3f5 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use serde_json::Value; +use sqlbuild_sqltext::compiler::models::CharSpan; /// One compile's project facts; models keep the order the compile-input stage produced them. #[derive(Clone, Debug, Default, PartialEq)] @@ -34,6 +35,8 @@ pub struct CompiledModelFacts { pub analysis: Option, /// Each expression SQL macro call in `query_sql`, by byte range. pub typed_calls: Vec, + /// The expansion passes from `authored_query_sql` to `query_sql`; empty when unknown. + pub expansion_passes: Vec>, } /// A model's effective config values with sorted keys, as the rules request has always read them. @@ -217,6 +220,8 @@ pub struct PlainModel { pub constant_declarations: Vec, /// Each expression SQL macro call in `query_sql`, by byte range. pub typed_calls: Vec, + /// The expansion passes from `authored_query_sql` to `query_sql`; empty when unknown. + pub expansion_passes: Vec>, } /// A one-based line and column, counted in characters. diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs index 959d41a9fc..8f8e991ae1 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs @@ -24,7 +24,20 @@ fn node_marker_calls(mut calls: Vec, node: &Expression) -> Vec calls.extend(from_markers(from)), Expression::Delete(delete) => calls.extend(join_markers(&delete.joins)), - Expression::JoinedTable(table) => calls.extend(join_markers(&table.joins)), + Expression::JoinedTable(table) => { + calls.extend( + marker(&table.left) + .into_iter() + .chain(join_markers(&table.joins)), + ); + } + Expression::Update(update) => calls.extend( + update + .from_clause + .iter() + .flat_map(from_markers) + .chain(join_markers(&update.from_joins)), + ), Expression::DataDeletionProperty(property) => { return property.on.dfs().fold(calls, node_marker_calls); } diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index ef30aaadb8..dea0e94b09 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -1574,17 +1574,18 @@ type MarkerCalls = Option>; /// The original JSON-tree walk, kept as the oracle the direct relation walk must match. fn json_relation_marker_calls(value: &Value) -> Vec<(String, String)> { - let from_markers = value - .get("from") - .and_then(|from| from.get("expressions")) - .and_then(Value::as_array) + let from_markers = ["from", "from_clause"] .into_iter() + .filter_map(|key| value.get(key)) + .filter_map(|from| from.get("expressions")) + .filter_map(Value::as_array) .flatten() + .chain(value.get("left").filter(|_| value.get("joins").is_some())) .filter_map(json_marker); - let join_markers = value - .get("joins") - .and_then(Value::as_array) + let join_markers = ["joins", "from_joins"] .into_iter() + .filter_map(|key| value.get(key)) + .filter_map(Value::as_array) .flatten() .filter_map(|join| join.get("this")) .filter_map(json_marker); diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs index f054efe809..f22adc6a39 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs @@ -63,6 +63,18 @@ fn given_parsed_statements_when_collecting_relation_markers_then_json_walk_order sql: "SELECT __ref(\"orders\") AS value FROM __table_fn(\"orders\", 2) WHERE id IN (SELECT id FROM t)", expected_calls: &[], }, + RelationMarkersTestCase { + description: "a parenthesised join keeps its left side", + dialect: "duckdb", + sql: "SELECT * FROM (__ref(\"orders\") JOIN __ref(\"customers\") ON TRUE)", + expected_calls: &[("__ref", "orders"), ("__ref", "customers")], + }, + RelationMarkersTestCase { + description: "update from relations are collected", + dialect: "postgres", + sql: "UPDATE t SET x = 1 FROM __ref(\"orders\") JOIN __ref(\"customers\") ON TRUE", + expected_calls: &[("__ref", "orders"), ("__ref", "customers")], + }, RelationMarkersTestCase { description: "set operations walk each branch", dialect: "postgres", diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index 8603bf2d2d..64a04fda35 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -72,7 +72,12 @@ pub(crate) fn model_contract_diagnostics( .map(|column| (column.name.as_str(), column)) .collect(); if contract_enforced { - diagnostics.extend(extra_columns(model, schema, inferred_columns)); + diagnostics.extend(extra_columns( + model, + schema, + inferred_columns, + types.folds_upper(), + )); } let unchecked: HashSet<&str> = model .unchecked_output_columns @@ -80,7 +85,12 @@ pub(crate) fn model_contract_diagnostics( .map(String::as_str) .collect(); for (index, declared) in schema.columns.iter().enumerate() { - let Some(inferred) = inferred_by_name.get(declared.name.as_str()) else { + let Some(inferred) = inferred_by_name.get(declared.name.as_str()).or_else(|| { + types + .folds_upper() + .then(|| inferred_by_name.get(declared.name.to_uppercase().as_str())) + .flatten() + }) else { if validate_declared_shape && !model.fast_lineage_has_star { diagnostics.push(missing_column(model, index, declared, contract_enforced)); } @@ -147,15 +157,20 @@ fn missing_declarations(model: &ContractModel) -> ContractDiagnostic { } } +/// Inferred columns no declared column names; a folded `order_id` also names a star's `ORDER_ID`. fn extra_columns( model: &ContractModel, schema: &ContractSchema, inferred_columns: &[InferredOutputColumn], + folds_upper: bool, ) -> Vec { - let declared_names: HashSet<&str> = schema + let declared_names: HashSet = schema .columns .iter() - .map(|column| column.name.as_str()) + .flat_map(|column| { + let folded: Option = folds_upper.then(|| column.name.to_uppercase()); + std::iter::once(column.name.clone()).chain(folded) + }) .collect(); let help: &str = if schema.named_schema { EXTRA_COLUMN_NAMED_SCHEMA_HELP diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs index a922a1dc2a..b72f041818 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs @@ -3,6 +3,7 @@ use std::cell::RefCell; use std::collections::HashMap; +use crate::contracts::constants::UPPER_FOLDING_DIALECT; use crate::type_system::errors::TypeNormalizationError; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::{NormalizedType, TypeNormalization}; @@ -21,6 +22,11 @@ pub(crate) struct TypeComparer<'a> { } impl<'a> TypeComparer<'a> { + /// Whether the dialect folds unquoted identifiers to upper case, as star expansion names them. + pub(crate) fn folds_upper(&self) -> bool { + self.dialect == UPPER_FOLDING_DIALECT + } + pub(crate) fn new(dialect: &'a str) -> Self { Self { dialect, diff --git a/crates/sqlbuild-analysis/src/contracts/constants.rs b/crates/sqlbuild-analysis/src/contracts/constants.rs index 26e6e274b7..35d840309d 100644 --- a/crates/sqlbuild-analysis/src/contracts/constants.rs +++ b/crates/sqlbuild-analysis/src/contracts/constants.rs @@ -39,3 +39,5 @@ pub(crate) const PROMOTION_REMOVE_CONTRACT_HELP: &str = "or remove the model's e pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; +/// The dialect that folds unquoted identifiers to upper case. +pub(crate) const UPPER_FOLDING_DIALECT: &str = "snowflake"; diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs index a384f09acf..9998ac560d 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs @@ -196,16 +196,15 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( expected_detail: Some("Expected 1 statement, found 2"), }, ParsedLineageTestCase { - description: "a parser panic on truncated T-SQL fails as an internal error", + description: "truncated T-SQL reports its parse error", sql: "SELECT IF(region > 1, re", dialect: Some("tsql"), inferred_columns: &[], - expected_status: "error", + expected_status: "unparsed", expected_lines: &[], expected_has_star: false, expected_detail: Some( - "NativeCompilerError: native SQL compilation panicked \ - (native fast lineage of request model 0)", + "Parse error at line 1, column 25: Expected ')' to close the IF condition", ), }, ParsedLineageTestCase { diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs index b86da84c9f..949b41cb4e 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs @@ -2,19 +2,19 @@ use crate::planner_warehouse::models::PhysicalCursorQuery; -/// Select the requested bounds of the cursor column as text. -pub fn cursor_query_sql(query: &PhysicalCursorQuery) -> String { +/// Select the requested bounds of the cursor column as text, each through `as_text`. +pub fn cursor_query_sql(query: &PhysicalCursorQuery, as_text: &dyn Fn(&str) -> String) -> String { let mut parts: Vec = Vec::new(); if !query.min_tags.is_empty() { parts.push(format!( - "CAST(MIN({}) AS VARCHAR) AS _min", - query.cursor_column + "{} AS _min", + as_text(&format!("MIN({})", query.cursor_column)) )); } if !query.max_tags.is_empty() { parts.push(format!( - "CAST(MAX({}) AS VARCHAR) AS _max", - query.cursor_column + "{} AS _max", + as_text(&format!("MAX({})", query.cursor_column)) )); } format!("SELECT {} FROM {}", parts.join(", "), query.relation) diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs index 10b026f12e..df78d914d7 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs @@ -47,7 +47,10 @@ fn given_logical_bounds_when_grouping_then_reads_each_relation_once_in_first_ord ]; for test_case in test_cases { let groups = cursor_query_groups(&test_case.requests); - let sql: Vec = groups.iter().map(cursor_query_sql).collect(); + let sql: Vec = groups + .iter() + .map(|group| cursor_query_sql(group, &|bound| format!("CAST({bound} AS VARCHAR)"))) + .collect(); let identities: Vec = groups .iter() .enumerate() diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs index 5e96bc534d..5a272e337d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs @@ -30,6 +30,8 @@ static SQL_TEST_CTE: LazyLock> = LazyLock::new(|| compiled(SQL_TEST_CTE_PATTERN)); const INTEGER_CURSOR_FAMILIES: &[TypeFamily] = &[TypeFamily::Integer, TypeFamily::Decimal]; +/// The dialect that stores timestamps as ISO-8601 text, so a text column is a timestamp cursor. +const TEXT_TIMESTAMP_DIALECT: &str = "sqlite"; const TEMPORAL_CURSOR_FAMILIES: &[TypeFamily] = &[ TypeFamily::Timestamp, TypeFamily::Datetime, @@ -170,7 +172,7 @@ fn reference_errors( )?); } } - if let Some(error) = cursor_error(model, shape, context.families)? { + if let Some(error) = cursor_error(model, shape, (context.families, context.dialect))? { errors.push(error); } Ok(errors) @@ -180,7 +182,7 @@ fn reference_errors( fn cursor_error( model: &MetadataModel, shape: &[(String, String)], - families: &Families, + (families, dialect): (&Families, Option<&str>), ) -> Result, SemanticFailure> { let (Some(cursor), Some(cursor_type)) = (model.cursor.as_deref(), model.cursor_type.as_deref()) else { @@ -203,6 +205,12 @@ fn cursor_error( if family == TypeFamily::Other || allowed.contains(&family) { return Ok(None); } + if family == TypeFamily::String + && allowed == TEMPORAL_CURSOR_FAMILIES + && dialect == Some(TEXT_TIMESTAMP_DIALECT) + { + return Ok(None); + } let message = format!("cursor_type {cursor_type} does not match column '{cursor}' type {actual}"); Ok(Some(model_error( diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index 0f5f5702c9..98800e3e4d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -132,6 +132,11 @@ pub(crate) const SEMANTIC_HELP: &[(&str, &str)] = &[ "rewrite the truth-value test null-safely, for example COALESCE(x, FALSE) for x IS TRUE, \ and NOT COALESCE(x, FALSE) or x IS DISTINCT FROM TRUE for x IS NOT TRUE", ), + ( + "B104", + "keep the ISO-8601 text without a cast, or convert it with datetime(), date() or \ + strftime()", + ), ( "B210", "use an expression whose inferred type satisfies this SQL construct", diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index 17175e9868..b198ae8a54 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -2,6 +2,7 @@ use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; +use crate::semantic_validation::_helpers::sqlite_temporal_casts; use crate::semantic_validation::_helpers::truth_tests; use crate::semantic_validation::_helpers::type_locations::locate_type_findings; use crate::semantic_validation::models::DiagnosticRow; @@ -167,6 +168,9 @@ pub(crate) fn map_diagnostics( sql, dialect, probes, )); errors.extend(truth_tests::unsupported_truth_test_errors(sql, dialect)); + errors.extend(sqlite_temporal_casts::sqlite_temporal_cast_warnings( + sql, dialect, + )); let valid = !errors .iter() .any(|error| error.severity == ValidationSeverity::Error); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs index b75be9835b..664231fa4b 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs @@ -4,5 +4,6 @@ pub mod function_names; pub mod identifiers; pub mod normalization; pub mod positions; +pub mod sqlite_temporal_casts; pub mod truth_tests; pub mod type_locations; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs new file mode 100644 index 0000000000..c3a37312e8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs @@ -0,0 +1,120 @@ +//! `CAST(... AS TIMESTAMP)` and its date and time siblings on SQLite, which keep only a number. + +use crate::semantic_validation::_helpers::function_names::{Cursor, is_word_byte}; +use polyglot_sql::{DialectType, ValidationError}; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::models::{QuotePolicy, Unclosed}; + +pub(crate) const SQLITE_TEMPORAL_CAST_CODE: &str = "B104"; +const CAST_KEYWORD: &str = "CAST"; +const AS_KEYWORD: &str = "AS"; +const TEMPORAL_TYPES: [&str; 5] = ["TIMESTAMP", "TIMESTAMPTZ", "DATETIME", "DATE", "TIME"]; + +/// One authored `CAST(... AS )`, spanning `CAST` through the type name. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct TemporalCast { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) type_name: String, +} + +/// Located B104 warnings for temporal casts, which SQLite applies with NUMERIC affinity. +pub(crate) fn sqlite_temporal_cast_warnings( + sql: &str, + dialect: DialectType, +) -> Vec { + if dialect != DialectType::SQLite { + return Vec::new(); + } + let mut cursor: Cursor = Cursor::new(); + temporal_casts(sql) + .into_iter() + .map(|cast| { + let message: String = format!( + "SQLite casts to {} with NUMERIC affinity and keeps only the leading number, so \ + '2026-01-01 10:00:00' becomes 2026", + cast.type_name + ); + cursor.advance(sql, cast.start); + let start: usize = cursor.characters; + let end: usize = start + sql[cast.start..cast.end].chars().count(); + ValidationError::warning(message, SQLITE_TEMPORAL_CAST_CODE) + .with_location(cursor.line, cursor.column) + .with_span(Some(start), Some(end)) + }) + .collect() +} + +/// Every top-level `AS ` inside a `CAST(...)`, outside comments and quoted text. +pub(crate) fn temporal_casts(sql: &str) -> Vec { + let bytes: &[u8] = sql.as_bytes(); + let mut casts: Vec = Vec::new(); + let mut open_casts: Vec<(usize, usize)> = Vec::new(); + let mut depth: usize = 0; + let mut pending_cast: Option = None; + let mut after_as: Option = None; + let mut index: usize = 0; + while index < bytes.len() { + let byte: u8 = bytes[index]; + if byte.is_ascii_whitespace() { + index += 1; + continue; + } + match non_code_end(bytes, index, QuotePolicy::COMPILER) { + Ok(Some(end)) => { + index = end; + continue; + } + Ok(None) => {} + Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => break, + } + if byte == b'(' { + depth += 1; + if let Some(start) = pending_cast.take() { + open_casts.push((start, depth)); + } + after_as = None; + index += 1; + continue; + } + pending_cast = None; + if byte == b')' { + if open_casts.last().is_some_and(|(_, level)| *level == depth) { + open_casts.pop(); + } + depth = depth.saturating_sub(1); + after_as = None; + index += 1; + continue; + } + if !is_word_byte(byte) { + after_as = None; + index += 1; + continue; + } + let word_start: usize = index; + while index < bytes.len() && is_word_byte(bytes[index]) { + index += 1; + } + let word: &str = &sql[word_start..index]; + if let Some(cast_start) = after_as.take() { + let upper: String = word.to_ascii_uppercase(); + if TEMPORAL_TYPES.contains(&upper.as_str()) { + casts.push(TemporalCast { + start: cast_start, + end: index, + type_name: upper, + }); + } + continue; + } + if word.eq_ignore_ascii_case(CAST_KEYWORD) { + pending_cast = Some(word_start); + } else if word.eq_ignore_ascii_case(AS_KEYWORD) + && let Some((cast_start, _)) = open_casts.last().filter(|(_, level)| *level == depth) + { + after_as = Some(*cast_start); + } + } + casts +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests.rs index 21e41d1335..e5b49dd034 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests.rs @@ -6,6 +6,8 @@ mod normalize_batch; mod positions; #[path = "tests/test_semantic_validation.rs"] mod semantic_validation; +#[path = "tests/test_sqlite_temporal_casts.rs"] +mod sqlite_temporal_casts; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_truth_tests.rs"] diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs new file mode 100644 index 0000000000..b0d4f5d2e5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs @@ -0,0 +1,61 @@ +use crate::semantic_validation::_helpers::sqlite_temporal_casts::{ + TemporalCast, sqlite_temporal_cast_warnings, temporal_casts, +}; +use crate::semantic_validation::tests::test_types::{ + TemporalCastDialectTestCase, TemporalCastScanTestCase, +}; +use polyglot_sql::DialectType; + +#[test] +fn given_authored_sql_when_scanning_temporal_casts_then_finds_only_cast_types() { + let test_cases = [ + TemporalCastScanTestCase { + description: "timestamp and date casts are found in any case", + sql: "SELECT CAST(a AS TIMESTAMP), cast(b as date), CAST(c AS TEXT) FROM t", + expected_casts: &[(7, 26, "TIMESTAMP"), (29, 43, "DATE")], + }, + TemporalCastScanTestCase { + description: "a nested cast's type is not the outer cast's", + sql: "SELECT CAST(COALESCE(a, CAST(1 AS TEXT)) AS DATETIME) FROM t", + expected_casts: &[(7, 52, "DATETIME")], + }, + TemporalCastScanTestCase { + description: "aliases, quoted text and comments are ignored", + sql: "SELECT (SELECT 1 AS date) AS d, 'CAST(x AS DATE)' -- CAST(y AS DATE)\nFROM t", + expected_casts: &[], + }, + ]; + for test_case in test_cases { + let found: Vec = temporal_casts(test_case.sql); + let spans: Vec<(usize, usize, &str)> = found + .iter() + .map(|cast| (cast.start, cast.end, cast.type_name.as_str())) + .collect(); + assert_eq!(spans, test_case.expected_casts, "{}", test_case.description); + } +} + +#[test] +fn given_dialect_when_checking_temporal_casts_then_only_sqlite_warns() { + let test_cases = [ + TemporalCastDialectTestCase { + description: "SQLite warns about a timestamp cast", + dialect: DialectType::SQLite, + expected_warnings: 1, + }, + TemporalCastDialectTestCase { + description: "DuckDB casts timestamps natively", + dialect: DialectType::DuckDB, + expected_warnings: 0, + }, + ]; + for test_case in test_cases { + assert_eq!( + sqlite_temporal_cast_warnings("SELECT CAST(a AS TIMESTAMP) FROM t", test_case.dialect) + .len(), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs index 9f58f8b22e..201a5dc6fc 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs @@ -73,3 +73,15 @@ pub(crate) struct LocatedTruthTestDiagnosticTestCase { pub(crate) expected_location: (Option, Option), pub(crate) expected_span: (Option, Option), } + +pub(crate) struct TemporalCastScanTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_casts: &'static [(usize, usize, &'static str)], +} + +pub(crate) struct TemporalCastDialectTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: polyglot_sql::DialectType, + pub(crate) expected_warnings: usize, +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs b/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs index 73d58f8154..87bb3c3c47 100644 --- a/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs +++ b/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs @@ -6,31 +6,35 @@ use std::path::{Path, PathBuf}; use crate::artifacts::_helpers::sql_test_records::{ file_sha256, is_safe_relative_sql_path, metadata, mtime_ns, }; -use crate::artifacts::models::SqlTestArtifactRecord; +use crate::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss}; -/// The record with its artifact's current mtime, when the artifact is unchanged. -#[must_use] +/// The record with its artifact's current mtime when the artifact is unchanged, else why not. pub fn reusable_sql_test_record( tests_root: &Path, record: &SqlTestArtifactRecord, identity: &str, -) -> Option { - if record.identity != identity || !is_safe_relative_sql_path(&record.relative_path) { - return None; +) -> Result { + if record.identity != identity { + return Err(SqlTestRecordMiss::IdentityChanged); + } + if !is_safe_relative_sql_path(&record.relative_path) { + return Err(SqlTestRecordMiss::UnsafePath); } let artifact_path: PathBuf = tests_root.join(&record.relative_path); - let metadata: Metadata = metadata(&artifact_path)?; - if !metadata.is_file() || metadata.len() != record.size { - return None; + let metadata: Metadata = metadata(&artifact_path) + .filter(Metadata::is_file) + .ok_or(SqlTestRecordMiss::ArtifactMissing)?; + if metadata.len() != record.size { + return Err(SqlTestRecordMiss::SizeChanged); } - let current_mtime_ns: i64 = mtime_ns(&metadata)?; + let current_mtime_ns: i64 = mtime_ns(&metadata).ok_or(SqlTestRecordMiss::UnreadableMtime)?; if current_mtime_ns == record.mtime_ns { - return Some(record.clone()); + return Ok(record.clone()); } - if file_sha256(&artifact_path)? != record.content_sha256 { - return None; + if file_sha256(&artifact_path).as_deref() != Some(record.content_sha256.as_str()) { + return Err(SqlTestRecordMiss::ContentChanged); } - Some(SqlTestArtifactRecord { + Ok(SqlTestArtifactRecord { mtime_ns: current_mtime_ns, ..record.clone() }) diff --git a/crates/sqlbuild-cache/src/artifacts/models.rs b/crates/sqlbuild-cache/src/artifacts/models.rs index d3eb9963e5..fb8d1c2a05 100644 --- a/crates/sqlbuild-cache/src/artifacts/models.rs +++ b/crates/sqlbuild-cache/src/artifacts/models.rs @@ -33,6 +33,32 @@ pub struct SqlTestArtifactRecord { pub content_sha256: String, } +/// Why a cached SQL test artifact could not be kept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SqlTestRecordMiss { + IdentityChanged, + UnsafePath, + ArtifactMissing, + SizeChanged, + UnreadableMtime, + ContentChanged, +} + +impl SqlTestRecordMiss { + /// The reason as a short snake-case label for debug logs. + #[must_use] + pub const fn label(self) -> &'static str { + match self { + Self::IdentityChanged => "identity_changed", + Self::UnsafePath => "unsafe_path", + Self::ArtifactMissing => "artifact_missing", + Self::SizeChanged => "size_changed", + Self::UnreadableMtime => "unreadable_mtime", + Self::ContentChanged => "content_changed", + } + } +} + /// The fields of one cache file record, already checked to be strings and non-negative integers. #[derive(Clone, Debug, PartialEq, Eq)] pub struct StoredSqlTestRecord { diff --git a/crates/sqlbuild-cache/src/artifacts/tests.rs b/crates/sqlbuild-cache/src/artifacts/tests.rs index 6d280821bd..7a54e11b92 100644 --- a/crates/sqlbuild-cache/src/artifacts/tests.rs +++ b/crates/sqlbuild-cache/src/artifacts/tests.rs @@ -3,6 +3,8 @@ mod helpers; #[cfg(unix)] #[path = "tests/test_remove_stale.rs"] mod remove_stale; +#[path = "tests/test_reusable_record.rs"] +mod reusable_record; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_write_artifacts.rs"] diff --git a/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs index 114464f151..9262e9b2e9 100644 --- a/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs @@ -47,3 +47,13 @@ pub(crate) fn linked_compiled_root(target: &Path) { std::fs::create_dir_all(target.join("elsewhere")).expect("linked folder"); std::os::unix::fs::symlink(target.join("elsewhere"), target.join("compiled")).expect("link"); } + +/// Move `root/t.sql`'s mtime five seconds ahead without changing its bytes. +pub(crate) fn bump_mtime(root: &Path) { + let file = std::fs::File::options() + .write(true) + .open(root.join("t.sql")) + .expect("artifact"); + let later = std::time::SystemTime::now() + std::time::Duration::from_secs(5); + file.set_modified(later).expect("mtime"); +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs new file mode 100644 index 0000000000..e7037a63f9 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs @@ -0,0 +1,68 @@ +use crate::artifacts::main::new_sql_test_record::new_sql_test_record; +use crate::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; +use crate::artifacts::models::SqlTestRecordMiss; +use crate::artifacts::tests::helpers::{bump_mtime, place}; +use crate::artifacts::tests::test_types::ReusableRecordTestCase; + +const IDENTITY: &str = "identity-1"; + +#[test] +fn given_recorded_artifact_when_checking_reuse_then_reports_why_it_missed() { + let test_cases = [ + ReusableRecordTestCase { + description: "an unchanged artifact is kept", + change: |_| {}, + identity: IDENTITY, + expected_outcome: Ok(()), + }, + ReusableRecordTestCase { + description: "a touched artifact with the same bytes is kept", + change: bump_mtime, + identity: IDENTITY, + expected_outcome: Ok(()), + }, + ReusableRecordTestCase { + description: "a new identity misses", + change: |_| {}, + identity: "identity-2", + expected_outcome: Err(SqlTestRecordMiss::IdentityChanged), + }, + ReusableRecordTestCase { + description: "a deleted artifact misses", + change: |root| std::fs::remove_file(root.join("t.sql")).expect("removed"), + identity: IDENTITY, + expected_outcome: Err(SqlTestRecordMiss::ArtifactMissing), + }, + ReusableRecordTestCase { + description: "a resized artifact misses", + change: |root| place(&root.join("t.sql"), b"select 10\n"), + identity: IDENTITY, + expected_outcome: Err(SqlTestRecordMiss::SizeChanged), + }, + ReusableRecordTestCase { + description: "same-size new bytes miss", + change: |root| { + place(&root.join("t.sql"), b"select 2\n"); + bump_mtime(root); + }, + identity: IDENTITY, + expected_outcome: Err(SqlTestRecordMiss::ContentChanged), + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("tests folder"); + place(&root.path().join("t.sql"), b"select 1\n"); + let record = new_sql_test_record(root.path(), &root.path().join("t.sql"), IDENTITY) + .expect("recorded artifact"); + (test_case.change)(root.path()); + + let outcome = reusable_sql_test_record(root.path(), &record, test_case.identity); + + assert_eq!( + outcome.map(|_| ()), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs index a475d71bf5..12583ab0a3 100644 --- a/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs @@ -48,3 +48,12 @@ pub(crate) struct WalkRaceTestCase { pub(crate) change_during_walk: fn(&std::path::Path), pub(crate) expected_stamped: bool, } + +/// A change to a recorded SQL test artifact and whether, or why not, the record is kept. +pub(crate) struct ReusableRecordTestCase { + pub(crate) description: &'static str, + /// Changes the tests folder after the artifact `t.sql` was recorded. + pub(crate) change: fn(&std::path::Path), + pub(crate) identity: &'static str, + pub(crate) expected_outcome: Result<(), crate::artifacts::models::SqlTestRecordMiss>, +} diff --git a/crates/sqlbuild-core/src/dlt_destination/constants.rs b/crates/sqlbuild-core/src/dlt_destination/constants.rs index 920e01ac55..a924286ff8 100644 --- a/crates/sqlbuild-core/src/dlt_destination/constants.rs +++ b/crates/sqlbuild-core/src/dlt_destination/constants.rs @@ -13,6 +13,8 @@ pub(crate) const SQLBUILD_OWNED_DESTINATION_KEYS: [&str; 4] = [ ]; /// The dataset DuckDB destinations land in without an explicit one. pub(crate) const DUCKDB_DEFAULT_DATASET: &str = "main"; +/// The dataset of SQLite's own database file; dlt attaches a file per other dataset. +pub(crate) const SQLITE_MAIN_DATASET: &str = "main"; /// The suffix Snowflake account and host settings drop. pub(crate) const SNOWFLAKE_ACCOUNT_SUFFIX: &str = ".snowflakecomputing.com"; /// SQL Server's default port number. @@ -24,6 +26,9 @@ pub(crate) const SNOWFLAKE_OPTIONAL_SETTINGS: [&str; 4] = pub(crate) const BIGQUERY_ARGUMENTS: [(&str, &str); 2] = [("project_id", "project"), ("location", "location")]; pub(crate) const DUCKDB_ADAPTER: &str = "duckdb"; +pub(crate) const SQLITE_ADAPTER: &str = "sqlite"; +/// The dlt destination factory that writes SQLite through SQLAlchemy. +pub(crate) const SQLALCHEMY_FACTORY: &str = "sqlalchemy"; pub(crate) const MOTHERDUCK_ADAPTER: &str = "motherduck"; pub(crate) const POSTGRES_ADAPTER: &str = "postgres"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; diff --git a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs index 3e06044eb3..07f5ea30cf 100644 --- a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs +++ b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs @@ -2,7 +2,9 @@ use crate::dlt_destination::_helpers::credentials::{adapter_call, credentials, required}; use crate::dlt_destination::_helpers::request::{check_dataset_name, check_destination_keys}; -use crate::dlt_destination::constants::{DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET}; +use crate::dlt_destination::constants::{ + DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET, SQLALCHEMY_FACTORY, SQLITE_ADAPTER, SQLITE_MAIN_DATASET, +}; use crate::dlt_destination::errors::DltDestinationError; use crate::dlt_destination::models::{ DltDataset, DltDestinationPlan, DltDestinationRequest, DltValue, @@ -26,6 +28,18 @@ pub fn dlt_destination_plan( }, }); } + if adapter == SQLITE_ADAPTER { + let path: &str = required(request.connection, "path", SQLITE_ADAPTER)?; + return Ok(DltDestinationPlan { + factory: SQLALCHEMY_FACTORY, + arguments: credentials(DltValue::Text(format!("sqlite:///{path}"))), + dataset: if request.dataset_name.truthy { + DltDataset::Given + } else { + DltDataset::Default(SQLITE_MAIN_DATASET) + }, + }); + } check_dataset_name(adapter, request.dataset_name)?; let Some((factory, arguments)) = adapter_call(adapter, request.connection)? else { return Err(DltDestinationError { diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs index 418fd98879..5174d1a6d8 100644 --- a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs +++ b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs @@ -23,6 +23,22 @@ fn given_adapter_connection_when_planning_dlt_destination_then_matches_python_ma dataset: DltDataset::Default("main"), }), }, + DestinationPlanTestCase { + description: "sqlite writes its database file through sqlalchemy into main", + adapter: "sqlite", + strings: &[("path", "/data/warehouse.sqlite")], + port: None, + destination_keys: &[], + dataset_name: None, + expected_plan: Ok(DltDestinationPlan { + factory: "sqlalchemy", + arguments: vec![( + "credentials", + DltValue::Text("sqlite:////data/warehouse.sqlite".to_owned()), + )], + dataset: DltDataset::Default("main"), + }), + }, DestinationPlanTestCase { description: "postgres connection string uses host default and port", adapter: "postgres", diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs index 86b6532aa8..f485354ef2 100644 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs @@ -19,6 +19,14 @@ pub(crate) fn duckdb(connection: &LoaderConnection) -> Result Result { + Ok(format!( + "sqlite:///{}", + required(connection, "path", "sqlite")? + )) +} + /// MotherDuck's database URI with its optional token. pub(crate) fn motherduck(connection: &LoaderConnection) -> String { let database: &str = optional(connection, "database").unwrap_or_default(); diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs index 198bf35e36..3a472007b0 100644 --- a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs @@ -1,7 +1,7 @@ //! The ingestr destination URI for a SQLBuild warehouse connection. use crate::ingestr_command::_helpers::destinations::{ - bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlserver, + bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlite, sqlserver, }; use crate::ingestr_command::errors::IngestrCommandError; use crate::loader_connection::models::LoaderConnection; @@ -13,6 +13,7 @@ pub fn ingestr_destination_uri( ) -> Result { match adapter_name { "duckdb" => duckdb(connection), + "sqlite" => sqlite(connection), "motherduck" => Ok(motherduck(connection)), "postgres" => postgres(connection), "bigquery" => bigquery(connection), diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs index e35dd77651..16bf812ffe 100644 --- a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs @@ -18,6 +18,13 @@ fn given_connection_when_building_destination_uri_then_matches_python_quoting() port: None, expected_uri: Ok("duckdb:///warehouse.duckdb"), }, + DestinationUriTestCase { + description: "sqlite path", + adapter_name: "sqlite", + strings: &[("path", "/data/warehouse.sqlite")], + port: None, + expected_uri: Ok("sqlite:////data/warehouse.sqlite"), + }, DestinationUriTestCase { description: "motherduck prefix and token", adapter_name: "motherduck", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 4e931cd9b7..dc10a996b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -19,7 +19,8 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, none, text, utc_now, }; -use crate::bindings::_helpers::build_progress::footer::format_build_footer; +use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; +use crate::bindings::_helpers::check::check_results::check_result_text; use crate::bindings::_helpers::compile_driver::pipeline_entries::plan_has_executable_work; use crate::bindings::_helpers::compile_outputs::runtime_target::{ write_python_check_runtime_target, write_runtime_target, @@ -35,8 +36,12 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::result_output::documents::format_build_execution_json; use crate::bindings::_helpers::result_output::documents::format_degradable_build_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; +use sqlbuild_core::build_progress::models::PythonNodeText; +use sqlbuild_core::check_command::models::{CheckResultText, CheckStatus}; const SUCCESS_STATUS: &str = "success"; +const CHECK_KIND: &str = "check"; +const CHECK_WARNED: &str = "check warned"; const PENDING_MESSAGE: &str = "Build cost collection has not completed."; const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; @@ -313,6 +318,30 @@ fn any_failed( Ok(false) } +/// A Python check result as the footer counts it: a failure fails the build, a warning warns. +fn check_node_text(check: &CheckResultText) -> PythonNodeText { + let message: Option = check + .error_message + .clone() + .or_else(|| check.message.clone()); + PythonNodeText { + name: check.name.clone(), + kind: CHECK_KIND.to_owned(), + status: if check.status == CheckStatus::Fail { + "failed" + } else { + SUCCESS_STATUS + } + .to_owned(), + warnings: if check.status == CheckStatus::Warn { + vec![message.clone().unwrap_or_else(|| CHECK_WARNED.to_owned())] + } else { + Vec::new() + }, + error_message: message, + } +} + /// Persist and render a completed direct build and return its exit code. pub(crate) fn complete_direct_build<'py>( context: &BuildContext<'py>, @@ -356,14 +385,19 @@ pub(crate) fn complete_direct_build<'py>( ); let cost_seconds: f64 = finish_cost_timing(context, cost_started); let record: Bound<'py, PyAny> = record?; - let footer: String = format_build_footer( + let checks: Vec = check_results + .try_iter()? + .map(|check| Ok(check_node_text(&check_result_text(&check?)?))) + .collect::>()?; + let footer: String = format_build_footer_with( &result.clone(), - native_field(&preparation.callbacks, "elapsed")?.extract::()?, - boolean(py, context.use_color).extract::()?, + ( + native_field(&preparation.callbacks, "elapsed")?.extract::()?, + context.use_color, + ), Some(&outcome.python_results.clone()).filter(|value| !value.is_none()), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .extract()?; + checks, + )?; emit(&context.stream, &format!("\n{footer}\n"))?; let payload: Bound<'py, PyAny> = format_degradable_build_execution_json( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs index 72ba57bd60..894c07caaf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs @@ -16,6 +16,21 @@ pub(crate) fn format_build_footer( elapsed: f64, use_color: bool, python_node_results: Option<&Bound<'_, PyAny>>, +) -> PyResult { + format_build_footer_with( + result, + (elapsed, use_color), + python_node_results, + Vec::new(), + ) +} + +/// The build footer, counting `checks` (Python check results) with the Python node results. +pub(crate) fn format_build_footer_with( + result: &Bound<'_, PyAny>, + (elapsed, use_color): (f64, bool), + python_node_results: Option<&Bound<'_, PyAny>>, + checks: Vec, ) -> PyResult { let mut python: Vec = Vec::new(); if let Some(results) = python_node_results { @@ -23,6 +38,7 @@ pub(crate) fn format_build_footer( python.push(python_node_text(&node?)?); } } + python.extend(checks); Ok(footer( &build_text(result)?, &python, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 1e502bab25..8a824ae6e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -1,12 +1,14 @@ //! Connection config resolution for CLI commands. +use std::path::PathBuf; + use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ - CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, DBT_PROFILE_ROUTING_KEYS, + CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error; @@ -25,6 +27,18 @@ use crate::bindings::_helpers::dbt_interop::profile_connection::resolve_raw_dbt_ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +const ATTACH_KEY: &str = "attach"; +const ATTACH_PATH_KEY: &str = "path"; +const ATTACH_TYPE_KEY: &str = "type"; +/// `ATTACH ... (TYPE ...)` values whose path is a local database file. +const LOCAL_ATTACH_TYPES: [&str; 2] = ["duckdb", "sqlite"]; +const URL_SCHEME_SEPARATOR: &str = "://"; +const MOTHERDUCK_PATH_PREFIX: &str = "md:"; +/// A path naming the home directory, which DuckDB expands from `HOME`. +const HOME_PREFIX: &str = "~"; +const HOME_DIRECTORY_PREFIX: &str = "~/"; +const HOME_VARIABLE: &str = "HOME"; + const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["raw_config", "project_dir", "adapter_name"], &["discovered_inputs", "cli_vars", "selected_target"], @@ -108,28 +122,92 @@ pub(crate) fn anchor_local_database_path( config: &Bound<'_, PyDict>, (project_dir, adapter_name): (&Bound<'_, PyAny>, &str), ) -> PyResult<()> { - let py: Python<'_> = config.py(); let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS .iter() .find(|(adapter, _)| *adapter == adapter_name) else { return Ok(()); }; + if adapter_name == DUCKDB_ADAPTER { + anchor_attach_paths(config, project_dir)?; + } let Some(database) = config.get_item(*path_key)? else { return Ok(()); }; - if !database.is_instance_of::() { - return Ok(()); + if let Some(anchored) = anchored_file_path(&database, project_dir)? { + config.set_item(*path_key, anchored)?; } - if path(py, &database)? - .call_method0("is_absolute")? - .is_truthy()? - || database.extract::()? == DUCKDB_MEMORY_DATABASE - { + Ok(()) +} + +/// The file a local path names: `~` expanded, then `project_dir / path` when still relative. +fn anchored_file_path<'py>( + value: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = value.py(); + if !value.is_instance_of::() { + return Ok(None); + } + let text: String = value.extract()?; + if text == DUCKDB_MEMORY_DATABASE { + return Ok(None); + } + if let Some(rest) = home_relative_rest(&text) { + return Ok(std::env::var_os(HOME_VARIABLE).map(|home| { + let file: PathBuf = PathBuf::from(home).join(rest); + PyString::new(py, &file.to_string_lossy()).into_any() + })); + } + if path(py, value)?.call_method0("is_absolute")?.is_truthy()? { + return Ok(None); + } + Ok(Some(path(py, project_dir)?.div(value)?.str()?.into_any())) +} + +/// What follows `~` or `~/`, the home directory DuckDB expands a path from. +fn home_relative_rest(text: &str) -> Option<&str> { + if text == HOME_PREFIX { + return Some(""); + } + text.strip_prefix(HOME_DIRECTORY_PREFIX) +} + +/// Anchor each relative DuckDB or SQLite file an `attach` entry names; other paths are DSNs or URLs. +fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = config.py(); + let Some(entries) = config.get_item(ATTACH_KEY)? else { + return Ok(()); + }; + let Ok(entries) = entries.try_iter() else { return Ok(()); + }; + let mut anchored: Vec> = Vec::new(); + for entry in entries { + let entry: Bound<'_, PyAny> = entry?; + let Ok(mapping) = entry.cast::() else { + anchored.push(entry); + continue; + }; + let local_file: bool = match mapping.get_item(ATTACH_TYPE_KEY)? { + None => true, + Some(kind) => kind + .extract::() + .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), + }; + let copy: Bound<'_, PyDict> = mapping.copy()?; + if let Some(path_value) = mapping.get_item(ATTACH_PATH_KEY)? + && local_file + && !path_value.extract::().is_ok_and(|text| { + text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) + }) + && let Some(file) = anchored_file_path(&path_value, project_dir)? + { + copy.set_item(ATTACH_PATH_KEY, file)?; + } + anchored.push(copy.into_any()); } - let anchored: Bound<'_, PyAny> = path(py, project_dir)?.div(&database)?.str()?.into_any(); - config.set_item(*path_key, anchored) + config.set_item(ATTACH_KEY, pyo3::types::PyList::new(py, anchored)?) } /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index ffeb324443..1a10104441 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,6 +1,6 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; @@ -11,7 +11,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, wrap_pyfunction}; use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; -use sqlbuild_cache::artifacts::models::SqlTestArtifactRecord; +use sqlbuild_cache::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::ExecutorHost; @@ -58,6 +58,9 @@ const MANIFEST_FILE: &str = "manifest.json"; /// The compile cache file stamping the compiled tree the last stale-file walk left. const COMPILED_TREE_STAMP_FILE: &str = "compiled-tree.bin"; const SQL_FILE_SUFFIX: &str = ".sql"; +const SQL_TEST_CACHE_LOGGER: &str = "sqlbuild.compile.reuse"; +/// The miss reason of a SQL test the cache file holds no record for. +const NOT_RECORDED: &str = "not_recorded"; /// `functions/` path components that already lead a function's path. const LANGUAGE_DIR_DEPTH: usize = 2; const PUBLISHED_STAGING_CHANGED: &str = "changed"; @@ -448,6 +451,20 @@ fn test_identities<'py>( Ok(identities) } +/// Log why a warm SQL test artifact cache could not keep some artifacts, by reason. +fn log_cache_misses(py: Python<'_>, misses: &BTreeMap<&'static str, usize>) -> PyResult<()> { + let reasons: String = misses + .iter() + .map(|(reason, count)| format!("{reason}={count}")) + .collect::>() + .join(","); + host::log_debug_message( + py, + SQL_TEST_CACHE_LOGGER, + &format!("SQL test artifact cache missed: {reasons}"), + ) +} + /// Plan every SQL test whose cached artifact under `target_dir` cannot be reused. fn plan_tests<'py>( target_dir: &Bound<'py, PyAny>, @@ -483,15 +500,23 @@ fn plan_tests<'py>( let mut pending: Vec = Vec::new(); let mut cached_paths: Vec> = Vec::new(); let mut cached_records: Vec<(String, SqlTestArtifactRecord)> = Vec::new(); + let mut misses: BTreeMap<&'static str, usize> = BTreeMap::new(); for (index, (test, identity)) in tests.into_iter().zip(identities).enumerate() { let Some((record_key, identity)) = identity else { pending.push((test.unbind(), None, None)); continue; }; - let reusable: Option = cached + let reusable: Result = cached .get(&record_key) - .and_then(|record| reusable_sql_test_record(&tests_root_path, record, &identity)); - if let Some(record) = reusable { + .ok_or(NOT_RECORDED) + .and_then(|record| { + reusable_sql_test_record(&tests_root_path, record, &identity) + .map_err(SqlTestRecordMiss::label) + }); + if let Err(reason) = reusable { + *misses.entry(reason).or_default() += 1; + } + if let Ok(record) = reusable { let path: Bound<'py, PyAny> = tests_root.div(&record.relative_path)?; artifact_kept(recorder, &path, (record.size, record.mtime_ns))?; cached_paths.push(path.unbind()); @@ -501,6 +526,9 @@ fn plan_tests<'py>( } yield_interpreter(py, index); } + if !cached.is_empty() && !misses.is_empty() { + log_cache_misses(py, &misses)?; + } let pending_tests: Vec> = pending .iter() .map(|(test, _, _)| test.bind(py).clone()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index 2a7f6664d5..8f3f6575fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -587,6 +587,7 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult = texts(args)? + let given: Vec = texts(args)?; + if name != PLAN_COMMAND && given.iter().any(|arg| arg == JSON_FLAG) { + return Err(argument_error( + command.py(), + format!("sqb dbt {name} does not support --json"), + JSON_HELP, + )); + } + let display_free: Vec = given .into_iter() .filter(|arg| !DISPLAY_FLAGS.contains(&arg.as_str())) .collect(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index deaf891fc1..f57e1b8a63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -266,6 +266,16 @@ pub(crate) fn finished_record<'py>( }, }; let (values, encoding_error) = config_values(&stripped); + let expansion_passes: Vec> = if *record.query_sql == *record.expansion.expanded_sql + { + vec![ + record.expansion.interpolation_spans.clone(), + record.expansion.declaration_spans.clone(), + record.expansion.macro_spans.clone(), + ] + } else { + Vec::new() + }; let facts: CompiledModelFacts = plain_model_facts(PlainModel { name, relative_path, @@ -291,6 +301,7 @@ pub(crate) fn finished_record<'py>( enum_declarations: declarations(enums)?, constant_declarations: declarations(constants)?, typed_calls: finished.typed_calls, + expansion_passes, }); Ok((record, facts)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index 5ed20ed5e2..c2904c48dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -64,3 +64,21 @@ impl<'py> HostEntries for PythonHost<'py> { pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult> { PythonHost(py).entry(id.into()) } + +/// `log_debug_event(logger=logging.getLogger(logger), message=message)`. +pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { + let kwargs: Bound<'_, PyDict> = PyDict::new(py); + kwargs.set_item( + "logger", + py.import("logging")? + .getattr("getLogger")? + .call1((logger,))?, + )?; + kwargs.set_item("message", message)?; + entry( + py, + sqlbuild_host::entries::models::ExecutorHost::LogDebugEvent, + )? + .call((), Some(&kwargs)) + .map(|_| ()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 0a82820a15..b2c7779c8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -45,11 +45,15 @@ use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_quer use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; use sqlbuild_analysis::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; +use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; +/// The adapter method that renders one cursor bound expression as text. +const CURSOR_TEXT_METHOD: &str = "render_cursor_text"; + /// Selection and policy inputs for cursor warehouse inspection. pub(crate) struct CursorInputs<'py> { pub(crate) selected_keys: Bound<'py, PyAny>, @@ -692,6 +696,34 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul Ok(values) } +/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. +fn adapter_cursor_sql( + request: &SnapshotRequest<'_>, + query: &PhysicalCursorQuery, +) -> PyResult { + let py = request.adapter.py(); + let render = native_field(&request.adapter, CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let sql: String = cursor_query_sql(query, &|expression: &str| { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }); + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(sql), + } +} + fn execute_queries<'py>( request: &SnapshotRequest<'py>, queries: &[PhysicalCursorQuery], @@ -708,7 +740,7 @@ fn execute_queries<'py>( if !on_progress.is_none() { on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; } - let outcome = read_cursor_query(py, &cursor_query_sql(query), handles)?; + let outcome = read_cursor_query(py, &adapter_cursor_sql(request, query)?, handles)?; report_cursor_query(py, query, (&outcome, &identity), on_progress)?; outcomes.push(outcome); } @@ -722,7 +754,11 @@ fn execute_queries<'py>( } let mut tasks: Vec<_> = Vec::new(); for query in queries { - tasks.push(bound_task(py, cursor_query_sql(query), handles)?); + tasks.push(bound_task( + py, + adapter_cursor_sql(request, query)?, + handles, + )?); } call_with( &host::entry(py, RUN_BOUNDED)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index e976064a6f..4dea3dd407 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_reports::{scope_move_preview, scope_rebound_usages}; use crate::bindings::_helpers::scopes::scope_validation::relocated_declarations; +const SQL_SUFFIX: &str = ".sql"; const GENERIC_DIALECT: &str = "generic"; const MATERIALIZED_KEY: &str = "materialized"; const MIGRATE_FROM_KEY: &str = "migrate_from"; @@ -305,6 +306,18 @@ fn python_locations( Ok(locations) } +/// Whether `request` moves a model without renaming it: a folder, or a file of the same name. +fn move_keeps_name(request: &RefactorRequest) -> bool { + if request.operation != RefactorOperation::MoveModel { + return false; + } + let destination: &str = request.destination.as_deref().unwrap_or_default(); + match destination.strip_suffix(SQL_SUFFIX) { + Some(path) => path.rsplit('/').next() == Some(request.model_name.as_str()), + None => true, + } +} + /// The facts one refactoring plan of `requests` reads. pub(crate) fn refactor_facts( project: &Bound<'_, PyAny>, @@ -312,7 +325,9 @@ pub(crate) fn refactor_facts( ) -> PyResult { let mut locations: Vec = Vec::new(); for request in requests { - locations.extend(python_locations(project, request)?); + if !move_keeps_name(request) { + locations.extend(python_locations(project, request)?); + } } let compiled: Bound<'_, PyAny> = native_field(&native_field(project, "graph")?, "project")?; let discovered: Bound<'_, PyAny> = native_field(project, "discovered")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs index b4da44c688..cdae298365 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs @@ -26,6 +26,7 @@ const SURROGATE_HELP: &str = "Return valid Unicode text from the macro; lone sur come from bytes decoded with errors='surrogateescape'"; const UNARY_HELP: &str = "Return a number from the nested macro, or apply the sign inside the macro"; +const NESTED_KEY_HELP: &str = "Return a string, number, True, False, None, or a tuple of them from a macro used as a dict key"; /// Text addressed by code point offsets, as the native scan reports them. pub(crate) struct CallText<'a> { @@ -344,8 +345,11 @@ impl<'py> ArgumentSource<'_, 'py> { ArgumentValue::Dict(pairs) => { let mapping: Bound<'py, PyDict> = PyDict::new(py); for (key, item) in pairs { - let key: Bound<'py, PyAny> = self.built(key)?; - mapping.set_item(key, self.built(item)?)?; + let built_key: Bound<'py, PyAny> = self.built(key)?; + if let Err(error) = built_key.hash() { + return Err(self.unhashable_key(key, error)); + } + mapping.set_item(built_key, self.built(item)?)?; } mapping.into_any() } @@ -384,6 +388,23 @@ impl<'py> ArgumentSource<'_, 'py> { Err(self.argument_error(self.nested[call].0, &detail, UNARY_HELP)) } + /// The error for a dict key a nested macro call made unhashable, at that call. + fn unhashable_key(&self, key: &ArgumentValue, error: PyErr) -> PyErr { + let Some(call) = nested_calls(key) + .into_iter() + .find(|call| self.values[*call].hash().is_err()) + else { + return error; + }; + let type_name: String = match self.values[call].get_type().name() { + Ok(name) => name.to_string(), + Err(failure) => return failure, + }; + let detail: String = + format!("use the value of a nested macro call, which is a {type_name}, as a dict key"); + self.argument_error(self.nested[call].0, &detail, NESTED_KEY_HELP) + } + /// The `MacroArgumentError` for a native parse error at its line and column. fn parse_error(&self, error: &ArgumentError) -> PyErr { let mut offset: usize = 0; @@ -410,6 +431,22 @@ impl<'py> ArgumentSource<'_, 'py> { } } +/// The nested macro calls inside `value`, in authored order. +fn nested_calls(value: &ArgumentValue) -> Vec { + match value { + ArgumentValue::NestedCall(index) => vec![*index], + ArgumentValue::List(items) | ArgumentValue::Tuple(items) => { + items.iter().flat_map(nested_calls).collect() + } + ArgumentValue::Dict(pairs) => pairs + .iter() + .flat_map(|(key, item)| nested_calls(key).into_iter().chain(nested_calls(item))) + .collect(), + ArgumentValue::Negative(inner) | ArgumentValue::Positive(inner) => nested_calls(inner), + _ => Vec::new(), + } +} + /// `MacroArgumentError` of the call `name` at code point `offset` of `text`. pub(crate) fn macro_argument_error( (expansion, classes): (&NativeMacroExpansion, &ExpansionClasses<'_>), diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 2370e9df16..0c6f68f4a4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -3,10 +3,12 @@ use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, DeclarationFact, ModelSchemaFact, }; +use sqlbuild_sqltext::compiler::models::CharSpan; +use crate::lint_text::models::ExpansionSpan; use crate::models::{ - Declaration, DeclarationMember, DynamicFamilyRow, DynamicProofRow, ModelRow, Reference, - SchemaColumnRow, SchemaRow, + AuthoredQueryMap, Declaration, DeclarationMember, DynamicFamilyRow, DynamicProofRow, ModelRow, + Reference, SchemaColumnRow, SchemaRow, }; /// The row the rules builder reads for one model; the authored query wins over the whole file. @@ -29,6 +31,12 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result &facts.authored_query_sql, &facts.query_sql, ), + query_column_offset: query_column_offset( + &facts.authored_sql, + &facts.authored_query_sql, + &facts.query_sql, + ), + authored_query: authored_query_map(facts), config: facts.config.values.clone(), authored_config_keys: facts.config.header_keys.clone(), logical_schema: facts.config.layer_schema.clone(), @@ -107,6 +115,60 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { } } +/// The expansion passes back to the authored query and where that query starts in the file. +fn authored_query_map(facts: &CompiledModelFacts) -> Option { + let span = |span: &CharSpan| ExpansionSpan { + source_start: character_offset(span.source_start), + source_end: character_offset(span.source_end), + output_start: character_offset(span.output_start), + output_end: character_offset(span.output_end), + }; + if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) { + return None; + } + let start: usize = facts.authored_sql.rfind(&*facts.authored_query_sql)?; + let above: &str = &facts.authored_sql[..start]; + Some(AuthoredQueryMap { + passes: facts + .expansion_passes + .iter() + .map(|pass| pass.iter().map(span).collect()) + .collect(), + line_offset: above.matches('\n').count() as u64, + column_offset: above + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() as u64, + }) +} + +fn character_offset(offset: usize) -> i64 { + i64::try_from(offset).unwrap_or(i64::MAX) +} + +/// The file-line characters before the query's first content that the query itself lacks. +fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { + let line_prefix = |text: &str, at: usize| { + text[..at] + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() + }; + let content_start = |text: &str| text.len() - text.trim_start().len(); + if authored_query.is_empty() { + return 0; + } + file.rfind(authored_query).map_or(0, |start| { + let in_file: usize = line_prefix(file, start + content_start(authored_query)); + let in_query: usize = line_prefix(query, content_start(query)); + in_file.saturating_sub(in_query) as u64 + }) +} + /// The lines of `file` above `query`, the rendered form of its `authored_query`; 0 if unknown. fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 { let leading_lines = |text: &str| { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index bb56b7b76c..006c1474f4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -133,6 +133,8 @@ impl RowWriter { query_sql: _, authored_sql: _, query_line_offset: _, + query_column_offset: _, + authored_query: _, typed_calls: _, config, authored_config_keys, @@ -351,6 +353,8 @@ impl RowReader<'_> { query_sql: String::new(), authored_sql: String::new(), query_line_offset: 0, + query_column_offset: 0, + authored_query: None, typed_calls: Vec::new(), config: { let count: usize = self.count()?; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index 8cf1544beb..3e47c5c188 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -84,6 +84,8 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Model { constant_declarations: vec![declaration], typed_calls: Vec::new(), query_line_offset: 0, + query_column_offset: 0, + authored_query: None, declared_audit_count: 3, targeting_test_count: 4, empty_input_only_test_count: 0, diff --git a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs index cebbf4fb43..819e600020 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs @@ -1227,7 +1227,7 @@ fn given_model_sql_when_stubbing_shape_then_offsets_map_back_to_source_positions .sql .find(test_case.expected_fragment) .ok_or("fragment missing from the model SQL")?; - let source = ModelSource::new(test_case.sql, &shape.offsets, 0); + let source = ModelSource::new(test_case.sql, &shape.offsets, (0, 0)); let end = normalized_start + test_case.expected_fragment.chars().count(); assert_eq!( source.text(normalized_start, end), diff --git a/crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs b/crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs new file mode 100644 index 0000000000..be4a4d6ba2 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs @@ -0,0 +1,10 @@ +//! Map an offset of expanded SQL back to its authored text. + +use crate::lint_text::_helpers::sql_diagnostics::map_offset; +use crate::lint_text::models::ExpansionSpan; + +/// The authored offset of expanded character `offset`, and whether expansion generated it. +#[must_use] +pub fn map_expanded_offset(offset: i64, passes: &[Vec]) -> (i64, bool) { + map_offset(offset, passes) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 61981d45cf..73885850a0 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -4,6 +4,7 @@ pub mod format_fixture_blocks; pub mod format_sql_bodies; pub mod frame_sql_macro_expression; pub mod lint_sql_bodies; +pub mod map_expanded_offset; pub mod named_declaration_role_kind; pub mod neutralize_audit_parameters; pub mod neutralize_context_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index 6d4ddb5c81..3634cbf666 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -95,7 +95,7 @@ pub(crate) struct PreparedBody { } /// One substitution of an expansion pass: authored source range and generated output range. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] pub struct ExpansionSpan { pub source_start: i64, pub source_end: i64, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 8f3a910df5..c78de2bcf8 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -562,6 +562,12 @@ pub struct Model { /// The lines of the model file above `query_sql`, which locations inside it skip. #[serde(default)] pub query_line_offset: u64, + /// The characters before `query_sql`'s first line on its file line, when it shares one. + #[serde(default)] + pub query_column_offset: u64, + /// How `query_sql` maps back to the authored query, when its expansion is known. + #[serde(default)] + pub authored_query: Option, pub config: BTreeMap, pub authored_config_keys: Vec, pub logical_schema: Option, @@ -866,6 +872,17 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } +/// The expansion passes from a model's authored query to `query_sql`, and the query's file place. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct AuthoredQueryMap { + /// Each pass's substitutions in characters, the first pass applied to the authored query. + pub passes: Vec>, + /// The lines of the model file above the authored query. + pub line_offset: u64, + /// The characters before the authored query on its first file line. + pub column_offset: u64, +} + /// One compiled model's facts as the rules request builder receives them. #[derive(Clone, Debug, Default)] pub struct ModelRow { @@ -875,6 +892,10 @@ pub struct ModelRow { pub authored_sql: String, /// The lines of the model file above `query_sql`. pub query_line_offset: u64, + /// The characters before `query_sql`'s first line on its file line. + pub query_column_offset: u64, + /// How `query_sql` maps back to the authored query, when its expansion is known. + pub authored_query: Option, pub config: BTreeMap, pub authored_config_keys: Vec, pub logical_schema: Option, diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs index d000202686..0f2b3ea81d 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs @@ -96,7 +96,7 @@ fn lexical_syntax(dialect: &str) -> LexicalSyntax { }, "clickhouse" => LexicalSyntax { backslash_escape_quotes: strings(&["'"]), - line_comment_prefixes: lines(&["--"]), + line_comment_prefixes: lines(&["--", "#", "//"]), ..LexicalSyntax::default() }, "databricks" | "duckdb" => LexicalSyntax { diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index ace5ffb435..35dd961830 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -1,6 +1,9 @@ use polyglot_sql::Expression; use std::collections::BTreeSet; +use crate::lint_text::main::map_expanded_offset::map_expanded_offset; +use crate::models::AuthoredQueryMap; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum CallKind { Ref, @@ -227,10 +230,27 @@ pub(crate) struct ModelSource<'a> { offsets: &'a [usize], /// The lines of the model file above this SQL. line_offset: u64, + /// The characters before this SQL's first content line on its file line. + column_offset: u64, + /// The 1-based line of this SQL's first content. + first_content_line: usize, + authored: Option>, +} + +/// The authored query that expansion turned into a model's SQL, as positions are read from it. +#[derive(Debug)] +struct AuthoredText<'a> { + characters: Vec, + lines: Vec, + map: &'a AuthoredQueryMap, } impl<'a> ModelSource<'a> { - pub(crate) fn new(sql: &'a str, offsets: &'a [usize], line_offset: u64) -> Self { + pub(crate) fn new( + sql: &'a str, + offsets: &'a [usize], + (line_offset, column_offset): (u64, u64), + ) -> Self { let mut bytes: Vec = Vec::with_capacity(sql.len() + 1); let mut lines: Vec = vec![0]; for (character, (byte, value)) in sql.char_indices().enumerate() { @@ -246,7 +266,65 @@ impl<'a> ModelSource<'a> { lines, offsets, line_offset, + column_offset, + first_content_line: sql[..sql.len() - sql.trim_start().len()] + .matches('\n') + .count() + + 1, + authored: None, + } + } + + /// Report positions in `authored_sql`, the query `map`'s passes expanded into this SQL. + pub(crate) fn with_authored( + mut self, + authored_sql: &str, + map: Option<&'a AuthoredQueryMap>, + ) -> Self { + self.authored = map.map(|map| { + let characters: Vec = authored_sql.chars().collect(); + let mut lines: Vec = vec![0]; + for (index, character) in characters.iter().enumerate() { + if *character == '\n' { + lines.push(index + 1); + } + } + AuthoredText { + characters, + lines, + map, + } + }); + self + } + + /// The file line and column of SQL character `character` in the authored query, if it maps. + fn authored_line_column(&self, character: usize) -> Option<(u64, u64)> { + let authored: &AuthoredText<'_> = self.authored.as_ref()?; + let offset: i64 = i64::try_from(character).unwrap_or(i64::MAX); + let (mapped, generated) = map_expanded_offset(offset, &authored.map.passes); + let mapped: usize = usize::try_from(mapped).unwrap_or(usize::MAX); + let byte: usize = self.bytes.get(character).copied().unwrap_or(self.sql.len()); + let expected: Option = self.sql.get(byte..).and_then(|rest| rest.chars().next()); + let fits: bool = if generated { + mapped < authored.characters.len() + } else { + authored.characters.get(mapped).copied() == expected + }; + if !fits { + return None; } + let line: usize = authored.lines.partition_point(|start| *start <= mapped); + let column: usize = mapped - authored.lines[line - 1] + 1; + let shifted: u64 = if line == 1 { + authored.map.column_offset + } else { + 0 + }; + Some(( + line as u64 + authored.map.line_offset, + column as u64 + shifted, + )) } fn original(&self, offset: usize) -> usize { @@ -267,9 +345,17 @@ impl<'a> ModelSource<'a> { /// The 1-based line and column of normalized character `offset`. pub(crate) fn line_column(&self, offset: usize) -> (u64, u64) { let character = self.original(offset); + if let Some(position) = self.authored_line_column(character) { + return position; + } let line = self.lines.partition_point(|start| *start <= character); let column = character - self.lines[line - 1] + 1; - (line as u64 + self.line_offset, column as u64) + let shifted: u64 = if line == self.first_content_line { + self.column_offset + } else { + 0 + }; + (line as u64 + self.line_offset, column as u64 + shifted) } /// The model's own text between normalized characters `start` and `end`. diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 373f72499a..733cabe15d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -144,7 +144,12 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result tuple[FailureCase, ...]: ), }, ), + failure_case( + name="analysis-sqlite-temporal-cast", + expected_code="B002", + expected_warning_code="B104", + expected_codes=("B002", "B104"), + files={ + FAILURE_CONFIG_PATH: _SQLITE_CONFIG, + **_mart( + "SELECT customer_id, CAST(amount AS TIMESTAMP) AS total_amount, missing\n" + f"FROM {_ORDERS}\n" + ), + }, + ), ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md index fa539c7253..0dfdb25eaa 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md @@ -34,6 +34,9 @@ Build dbt models first (including dbt tests), then SQLBuild models with audits. sqb dbt build [--select ...] [--exclude ...] ``` +`sqb dbt run` and `sqb dbt build` have no JSON output: they reject `--json` with `C238` before +dbt starts. Use `sqb dbt plan --json` for a machine-readable preview of the combined work. + ### Plan safety policies After dbt finishes, `sqb dbt run` and `sqb dbt build` apply the same plan safety policies as diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index 5680d39da6..882cf0a160 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -167,7 +167,7 @@ The command refuses, and writes nothing, when: - a reference is produced by a macro, or a column is read inside SQL that a macro generates; - a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; - a downstream model's `SELECT *` would silently rename its output, without `--cascade`; -- Python code or Python node SQL strings name the model or column; +- Python code or Python node SQL strings name a model being renamed, or a column being renamed. A move that keeps the model's name leaves Python references to it as they are; - a Python import of a moved Python file cannot be rewritten safely. A rename whose new name belongs to another layer's folder, under the layer-folder rules `SQBRPROJECT102` and `SQBRPROJECT105`, fails verification because `sqb rename` keeps the file where it is. When those rules are the only failures, the output says where the model belongs and prints the one-step [`sqb mv`](#rename-or-move) command, with that layer's folder and the current sub-folders filled in: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index 35506f687c..dd8164d1d1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -52,6 +52,13 @@ alias = "external" read_only = true ``` +A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`, +and a path starting with `~/` is relative to your home directory. This also applies to `attach` +entries that come from a dbt profile: they resolve from the SQLBuild project directory, not from +the dbt project or the working directory. +Every connection of a command attaches it once: connections to the same project database share the +attachment. + ## Table promotion mode DuckDB defaults to `staged` promotion: tables are materialized into a staging table, audited, then swapped into the target. This is configurable in `settings`: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md index 97337b9aa1..3517b77ccf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md @@ -40,6 +40,7 @@ schema = "analytics_dev" | `user` | Database user | | `password` | Database password | | `dbname` | Database name | +| `temporary_test_functions` | `false` stops `sqb test` from creating session-temporary (`pg_temp`) copies of project SQL functions, so tests call the built functions. Use it behind a transaction-mode pooler such as PgBouncer, where later statements can run in a session without the copies (default: `true`) | Connection fields are passed to `psycopg.connect()`. See the [psycopg documentation](https://www.psycopg.org/psycopg3/docs/api/connections.html) for all available options. The target remains authoritative for database and schema qualification. @@ -70,4 +71,5 @@ schema = "dev" `sqb test` creates each SQL function the selected tests call in the session's `pg_temp` schema and calls it as `pg_temp.`, so tests use the current project definition without a build. Temporary functions belong to one server session: behind a transaction-mode pooler, run `sqb test` through a -direct connection or a session-mode pool. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). +direct connection or a session-mode pool, or set `temporary_test_functions = false` so tests call the +built functions. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index 1d0dcc04fa..4a2a535e3a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -76,7 +76,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly its declared type; a computed column gets the type SQLBuild infers at compile time. - Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not produce timestamps in SQLite (they keep only the leading number), so write timestamps as text - literals or use SQLite date functions. SQLBuild never applies such a cast itself: source, + literals or use SQLite date functions. Compile warns with `B104` at each such cast in model SQL. SQLBuild never applies such a cast itself: source, contract and type-change casts to a date or time type keep the stored text, and source loaders write Python `datetime` values as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`. - Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the @@ -84,6 +84,9 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against `__cursor_start()` and `__cursor_end()` compare the stored text as is. +- A `cursor_type timestamp` column may be text, such as + `strftime('%Y-%m-%d 00:00:00', ordered_at)`, because SQLite stores timestamps as text. Other + adapters reject a text timestamp cursor with `B301`. - Booleans are stored as `0` and `1`. - JSON, arrays and objects are stored as JSON text. - `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)` @@ -107,6 +110,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly | `sqb diff`, `sqb clone` | Supported; clones are physical copies | | Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | | Statement time limits | Connection interrupt | +| `sqb load`: Python, dlt and ingestr loaders | Supported; dlt writes through its `sqlalchemy` destination | ## Scenarios diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index eca32a9470..4dc3e69973 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -34,8 +34,8 @@ def check_orders_exported(ctx: CheckContext): """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: - return ctx.fail("no orders exported") - return ctx.pass_("orders exported") + return ctx.fail(message="no orders exported") + return ctx.pass_(message="orders exported") ``` The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`. @@ -51,7 +51,7 @@ def orders_present(ctx): """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] - return ctx.pass_() if count else ctx.fail("no orders") + return ctx.pass_() if count else ctx.fail(message="no orders") ``` Do not hard-code relation names in check SQL; see [Hard-coded relation names](sql-references.md#hard-coded-relation-names). @@ -86,7 +86,7 @@ Returning `None` is not allowed - checks must be explicit. def orders_freshness(ctx): """Orders freshness.""" if stale(): - return ctx.fail("export is stale") # recorded as a warning, does not fail the build + return ctx.fail(message="export is stale") # recorded as a warning, does not fail the build return ctx.pass_() ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md index e0cdfed182..afec5249a9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -69,7 +69,7 @@ def orders_pipeline(): @check(depends_on=export) def orders_export_check(ctx): """Check that the orders export is ready.""" - return ctx.pass_("export ready") + return ctx.pass_(message="export ready") return [load, export, orders_export_check] ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md index 6896a8c95e..091f62fa3c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md @@ -393,5 +393,9 @@ See the [CLI reference](../cli/scenario.md) for full command documentation. ## Limitations - Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error. +- Local replay compares VARIANT and JSON output columns by their JSON value, not their type: key + order, whitespace and number spelling do not matter, so a Snowflake capture's pretty-printed + JSON equals an expected `PARSE_JSON(...)`. Values of non-JSON types with the same JSON text also + compare equal, for example `DATE '2024-01-01'::VARIANT` and `'2024-01-01'::VARIANT`. - Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason. - Outside local replay, scenario fixture, expected and assertion SQL runs as authored: SQLBuild replaces `__ref`, `__source`, `__seed` and `__dbt_ref` calls with relation names and may wrap a partial fixture to add its missing columns, but never regenerates the query. Calls inside comments and quoted text are left unchanged. Quoted text and comments follow the project adapter's rules, including with `--local`: backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, and `//` line comments on Snowflake. A call naming a relation the scenario does not provide, or SQL with an unclosed string or comment, fails planning with `S511` naming the scenario file. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md index 9da9c2427e..6dc5b1d9ee 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md @@ -207,6 +207,7 @@ the notice never reduces them to a count alone. | B101 | E202 or SQLBuild | Unknown function, or a built-in spelling the dialect does not accept | | B102 | E203 or project declaration | Invalid function arity | | B103 | SQLBuild | Truth-value test the dialect does not accept | +| B104 | SQLBuild | Warning: a SQLite `CAST` to a date or time type, which keeps only the leading number | | B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | | B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | | B216 | E216 | Set-operation, subquery, or row arity | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 851ae4f9cc..180d5af6a7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -115,7 +115,7 @@ Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_i ### Destination -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): +By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. On SQLite, dlt writes through its `sqlalchemy` destination into the project's database file (dataset `main`); the `sqlbuild[dlt]` extra installs the SQLAlchemy it needs. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): ```yaml dlt_sources: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md index 58cd89e2b4..60544e7351 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -138,6 +138,7 @@ SQLBuild automatically builds the ingestr destination URI from your adapter conn | Adapter | Destination URI format | |---------|----------------------| | DuckDB | `duckdb:///path/to/db.duckdb` | +| SQLite | `sqlite:///path/to/db.sqlite` | | MotherDuck | `motherduck://database` | | PostgreSQL | `postgresql://user:pass@host:port/db` | | Snowflake | `snowflake://user:pass@account/db/schema` | diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 32829b302b..b4f0982e88 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1190,7 +1190,7 @@ def duckdb_build_attach_sql(self, attach_entry: dict[str, object]) -> str: """Build a DuckDB ATTACH statement from one attach config entry.""" path: str = str(attach_entry["path"]) - sql: str = f"ATTACH '{path}'" + sql: str = f"ATTACH IF NOT EXISTS '{path}'" alias: object | None = attach_entry.get("alias") if alias is not None: sql += f" AS {alias}" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index a9a37916aa..ae44b4191c 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -140,6 +140,12 @@ def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> del cursor_type return expression + def render_cursor_text(self, *, expression: str, cursor_type: str) -> str: + """Optional: a cursor bound expression as the text the planner parses as a cursor value.""" + + del cursor_type + return f"CAST({expression} AS VARCHAR)" + def render_create_temporary_function( self, *, name: str, definition: FunctionDefinition ) -> TemporaryFunction | None: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 745f207132..a92876ca3d 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -160,7 +160,7 @@ class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): sql_analysis_dialect_name: ClassVar[str | None] = "clickhouse" sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax( backslash_escape_quotes=frozenset({"'", '"', "`"}), - line_comment_prefixes=frozenset({"--"}), + line_comment_prefixes=frozenset({"--", "#", "//"}), ) max_identifier_length: ClassVar[int] = MAX_IDENTIFIER_LENGTH relation_grants_supported: ClassVar[bool] = False diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 2165dabf5d..edf2f88a2f 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -121,6 +121,7 @@ from sqlbuild.adapters.databricks._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.databricks.classes.databricks_connection import _DatabricksConnection from sqlbuild.adapters.databricks.constants import ( + CURRENT_CATALOG_INFORMATION_SCHEMA, DELTA_COLUMN_MAPPING_NAME_MODE, DELTA_COLUMN_MAPPING_PROPERTY, DELTA_DEFAULT_DELETED_FILE_RETENTION_DAYS, @@ -3126,11 +3127,13 @@ def schema_exists( database: str | None, schema: str, ) -> bool: - if database is None: - return False + schemata: str = ( + CURRENT_CATALOG_INFORMATION_SCHEMA + if database is None + else self._information_schema(database) + ) query: str = ( - f"SELECT 1 FROM {self._information_schema(database)}.schemata " - f"WHERE schema_name = {self._string_literal(schema)}" + f"SELECT 1 FROM {schemata}.schemata WHERE schema_name = {self._string_literal(schema)}" ) cursor: Any = connection.cursor() try: diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index 1557862edf..bc09f33568 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -34,3 +34,4 @@ ) USAGE_UNAVAILABLE_SQLSTATES: frozenset[str] = frozenset({"42501", "42P01", "42704"}) LINEAGE_VIEW_SOURCE_TYPES: tuple[str, ...] = ("VIEW", "MATERIALIZED_VIEW", "METRIC_VIEW") +CURRENT_CATALOG_INFORMATION_SCHEMA: str = "information_schema" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index a0b5c6ac3e..5dd568d615 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -111,6 +111,7 @@ from sqlbuild.adapters.postgres.constants import ( ROW_COUNT_RELATION_KINDS, TABLE_FUNCTION_RETURN_TYPE, + TEMPORARY_TEST_FUNCTIONS_KEY, ) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.diagnostics.main.log_sql import log_sql @@ -140,6 +141,7 @@ def get_columns_for_relations( escape_string_prefix=True, nested_block_comments=True ) max_identifier_length: ClassVar[int] = 63 + temporary_test_functions: bool = True _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, @@ -564,7 +566,7 @@ def render_create_temporary_function( ) -> TemporaryFunction | None: """Copy a SQL function into the session's `pg_temp` schema, called schema-qualified.""" - if definition.language != FunctionLanguage.SQL: + if definition.language != FunctionLanguage.SQL or not self.temporary_test_functions: return None target: str = f"pg_temp.{self.render_identifier(name)}" return TemporaryFunction( @@ -2088,7 +2090,11 @@ def connect(self, config: dict[str, Any]) -> _PostgresConnection: code="A401", ) from error - raw_connection: Any = psycopg.connect(**config, autocommit=True) + connect_config: dict[str, Any] = dict(config) + self.temporary_test_functions = bool( + connect_config.pop(TEMPORARY_TEST_FUNCTIONS_KEY, self.temporary_test_functions) + ) + raw_connection: Any = psycopg.connect(**connect_config, autocommit=True) return _PostgresConnection(raw_connection) def _execute(self, *, connection: _PostgresConnection, sql: str) -> Any: diff --git a/src/sqlbuild/adapters/postgres/constants.py b/src/sqlbuild/adapters/postgres/constants.py index 8910dbb44a..ef548a4e26 100644 --- a/src/sqlbuild/adapters/postgres/constants.py +++ b/src/sqlbuild/adapters/postgres/constants.py @@ -3,3 +3,4 @@ TABLE_FUNCTION_RETURN_TYPE: str = "TABLE" ROW_COUNT_RELATION_KINDS: frozenset[str] = frozenset({"r", "m"}) TRANSACTION_OPEN_STATUS: str = "INTRANS" +TEMPORARY_TEST_FUNCTIONS_KEY: str = "temporary_test_functions" diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index afab51a217..2f200b531d 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -114,6 +114,7 @@ DIFF_UNSUPPORTED_COMPARISON_TYPES, INFORMATION_SCHEMA_NULLABLE_VALUE, INTEGER_TYPE_TOKEN, + ISO8601_CONVERT_STYLE, USER_TABLE_OBJECT_TYPE, ) from sqlbuild.compiler.compile.types import FunctionLanguage @@ -990,6 +991,13 @@ def render_loader_value_literal( def _quote_sql_string(self, value: str) -> str: return "'" + value.replace("'", "''") + "'" + def render_cursor_text(self, *, expression: str, cursor_type: str) -> str: + """Render a timestamp bound in ISO 8601 (style 126), not CAST's 'Apr 4 2026 2:30PM'.""" + + if cursor_type == CursorKind.TIMESTAMP: + return f"CONVERT(VARCHAR(33), {expression}, {ISO8601_CONVERT_STYLE})" + return super().render_cursor_text(expression=expression, cursor_type=cursor_type) + def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: if cursor_type == CursorKind.INTEGER: return value diff --git a/src/sqlbuild/adapters/sqlserver/constants.py b/src/sqlbuild/adapters/sqlserver/constants.py index 8c8b78377b..611c496afc 100644 --- a/src/sqlbuild/adapters/sqlserver/constants.py +++ b/src/sqlbuild/adapters/sqlserver/constants.py @@ -12,3 +12,4 @@ INFORMATION_SCHEMA_NULLABLE_VALUE: str = "YES" INTEGER_TYPE_TOKEN: str = "INT" USER_TABLE_OBJECT_TYPE: str = "U" +ISO8601_CONVERT_STYLE: int = 126 diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 7ff175af98..267c97b175 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -8,7 +8,7 @@ from collections.abc import Callable, Iterable from concurrent.futures import Future, ThreadPoolExecutor from contextvars import copy_context -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, cast from uuid import uuid4 @@ -36,6 +36,7 @@ install_temporary_test_functions, rewrite_test_entry, ) +from sqlbuild.executor.pipeline.exceptions import TemporaryTestFunctionError from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( raise_warehouse_connection_failure, ) @@ -132,11 +133,18 @@ def run_test_pipeline( connection=connections[0], ) for ordinal, connection in enumerate(connections[1:], start=2): + label: str = f"test connection {ordinal} of {len(connections)}" install_temporary_test_functions( adapter=adapter, connection=connection, functions=temporary_functions, - connection_label=f"test connection {ordinal} of {len(connections)}", + connection_label=label, + ) + _install_python_test_functions( + adapter=adapter, + connection=connection, + functions=temporary_functions, + connection_label=label, ) installed_functions.append(temporary_functions) entries = tuple( @@ -330,6 +338,12 @@ def _connect_test_connection( functions=functions, connection_label="a replacement test connection", ) + _install_python_test_functions( + adapter=adapter, + connection=connection, + functions=functions, + connection_label="a replacement test connection", + ) except BaseException: adapter.close(connection) raise @@ -407,12 +421,13 @@ def _prepare_test_functions( name=key.name, reason=temporary_functions.failures.get(key) ) - _register_connection_scoped_python_functions( + registered: tuple[FunctionPlanEntry, ...] = _register_connection_scoped_python_functions( missing_by_key=missing_by_key, function_entries=function_entries, adapter=adapter, connection=connection, ) + temporary_functions = replace(temporary_functions, python_functions=registered) missing_by_test: dict[CompiledObjectKey, tuple[str, ...]] = {} for test_entry in plan.test_entries: @@ -445,24 +460,76 @@ def _register_connection_scoped_python_functions( function_entries: dict[CompiledObjectKey, FunctionPlanEntry], adapter: BaseAdapter, connection: Any, -) -> None: +) -> tuple[FunctionPlanEntry, ...]: if adapter.persists_python_functions(): - return + return () + registered: list[FunctionPlanEntry] = [] key: CompiledObjectKey for key in tuple(missing_by_key): function_entry: FunctionPlanEntry | None = function_entries.get(key) if function_entry is None or function_entry.language != FunctionLanguage.PYTHON: continue - result: FunctionExecutionResult = execute_function( - function_entry=function_entry, - adapter=adapter, - connection=connection, - statement_recorder=StatementRecorder(), - run_id="test", - query_change_tracking=False, + result: FunctionExecutionResult = _register_python_function( + adapter=adapter, connection=connection, function_entry=function_entry ) if result.status == ExecutionStatus.SUCCESS: del missing_by_key[key] + registered.append(function_entry) + return tuple(registered) + + +def _register_python_function( + *, adapter: BaseAdapter, connection: Any, function_entry: FunctionPlanEntry +) -> FunctionExecutionResult: + return execute_function( + function_entry=function_entry, + adapter=adapter, + connection=connection, + statement_recorder=StatementRecorder(), + run_id="test", + query_change_tracking=False, + ) + + +def _install_python_test_functions( + *, + adapter: BaseAdapter, + connection: Any, + functions: TemporaryTestFunctions, + connection_label: str, +) -> None: + """Register the first connection's Python functions on another, or fail with T006.""" + + function_entry: FunctionPlanEntry + for function_entry in functions.python_functions: + result: FunctionExecutionResult = _register_python_function( + adapter=adapter, connection=connection, function_entry=function_entry + ) + if result.status == ExecutionStatus.SUCCESS or _function_exists( + adapter=adapter, connection=connection, function_entry=function_entry + ): + continue + raise TemporaryTestFunctionError( + function_name=function_entry.name, + connection=connection_label, + reason=result.error_message or result.status.value, + ) + + +def _function_exists( + *, adapter: BaseAdapter, connection: Any, function_entry: FunctionPlanEntry +) -> bool: + """Whether the connection already sees the function, as connections to one database file do.""" + + destination: CompiledRelationLocation = function_entry.destination + return bool( + adapter.list_functions( + connection=connection, + database=destination.database, + schemas=(destination.schema,) if destination.schema is not None else None, + names=(destination.name,), + ) + ) def _group_function_names( diff --git a/src/sqlbuild/executor/pipeline/models.py b/src/sqlbuild/executor/pipeline/models.py index c821813fe3..f8b73cae46 100644 --- a/src/sqlbuild/executor/pipeline/models.py +++ b/src/sqlbuild/executor/pipeline/models.py @@ -6,7 +6,12 @@ from dataclasses import dataclass, field from typing import Any -from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, SqlTestPlanEntry +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + FunctionPlanEntry, + SqlTestPlanEntry, +) from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildCallbacks, @@ -52,6 +57,8 @@ class TemporaryTestFunctions: created_keys: frozenset[CompiledObjectKey] = frozenset() replacements: dict[str, str] = field(default_factory=dict) failures: dict[CompiledObjectKey, str] = field(default_factory=dict) + python_functions: tuple[FunctionPlanEntry, ...] = () + """Connection-scoped Python functions the first connection registered, in order.""" @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py new file mode 100644 index 0000000000..8ae5db92c7 --- /dev/null +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py @@ -0,0 +1,34 @@ +"""One canonical JSON text per JSON value, so equal values compare equal as text.""" + +from __future__ import annotations + +import json +from decimal import Decimal + + +def canonical_json(text: str | None) -> str | None: + """`text` as JSON with sorted keys, no whitespace and numbers in their shortest exact form.""" + + if text is None: + return None + return _canonical(json.loads(text, parse_float=Decimal, parse_int=Decimal)) + + +def _canonical(value: object) -> str: + if isinstance(value, dict): + members: list[str] = [ + f"{json.dumps(key, ensure_ascii=False)}:{_canonical(value[key])}" + for key in sorted(value) + ] + return "{" + ",".join(members) + "}" + if isinstance(value, list): + return "[" + ",".join(_canonical(item) for item in value) + "]" + if isinstance(value, Decimal): + return _canonical_number(value) + return json.dumps(value, ensure_ascii=False) + + +def _canonical_number(number: Decimal) -> str: + if number.is_finite() and number == number.to_integral_value(): + return str(int(number)) + return format(number.normalize(), "f") diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py new file mode 100644 index 0000000000..4a5172ba28 --- /dev/null +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py @@ -0,0 +1,69 @@ +"""Compare DuckDB VARIANT and JSON outputs by their JSON value, so equal JSON text matches.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.relations.types import RelationLocation +from sqlbuild.executor.scenario._helpers.lifecycle.canonical_json import canonical_json +from sqlbuild.executor.scenario.constants import ( + CANONICAL_JSON_FUNCTION, + IDENTIFIER_QUOTE, + SEMI_STRUCTURED_COMPARISON_TEMPLATE, + SEMI_STRUCTURED_LOCAL_TYPES, +) + + +def variant_compared_columns( + *, + adapter: BaseAdapter, + connection: Any, + location: RelationLocation, + compared_columns: tuple[str, ...] | None, +) -> tuple[str, ...] | None: + """The compared columns, with VARIANT and JSON ones compared by JSON value, not by type.""" + + if compared_columns is None or not isinstance(adapter, DuckDbBackedAdapter): + return compared_columns + actual: tuple[ColumnInfo, ...] = adapter.get_columns( + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + semi_structured: frozenset[str] = frozenset( + column.name.lower() + for column in actual + if column.type.strip().upper() in SEMI_STRUCTURED_LOCAL_TYPES + ) + canonicalised: tuple[str, ...] = tuple( + SEMI_STRUCTURED_COMPARISON_TEMPLATE.format(column=column) + if _identifier_name(column).lower() in semi_structured + else column + for column in compared_columns + ) + if canonicalised != compared_columns: + _register_canonical_json(adapter=adapter, connection=connection) + return canonicalised + + +def _identifier_name(column: str) -> str: + inner: str = column.removeprefix(IDENTIFIER_QUOTE).removesuffix(IDENTIFIER_QUOTE) + if len(inner) + len(IDENTIFIER_QUOTE) * 2 != len(column): + return column + return inner.replace(IDENTIFIER_QUOTE * 2, IDENTIFIER_QUOTE) + + +def _register_canonical_json(*, adapter: DuckDbBackedAdapter, connection: Any) -> None: + registered: object = adapter.execute( + connection=connection, + sql=( + "SELECT COUNT(*) FROM duckdb_functions() " + f"WHERE function_name = '{CANONICAL_JSON_FUNCTION}'" + ), + ).fetchone() + if registered is not None and registered[0] == 0: + connection.create_function(CANONICAL_JSON_FUNCTION, canonical_json, ["VARCHAR"], "VARCHAR") diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index 313c83e736..ff719a1ee7 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any @@ -15,12 +16,15 @@ scenario_snapshot_root, ) from sqlbuild.executor.scenario.constants import ( + LOCAL_CAPTURE_ADAPTER, + LOCAL_TYPE_VARIANT, SCENARIO_LOCAL_JSONL_INVALID, SCENARIO_LOCAL_LOAD_FAILED, SCENARIO_LOCAL_MANIFEST_INVALID, SCENARIO_LOCAL_SNAPSHOT_MISSING, SCENARIO_LOCAL_SNAPSHOT_STALE, SCENARIO_LOCAL_TYPE_INVALID, + VARIANT_FROM_JSON_PLACEHOLDER, ) from sqlbuild.executor.scenario.models import ( ScenarioLocalSnapshotLoadedRelation, @@ -111,6 +115,7 @@ def load_scenario_snapshot_into_duckdb( snapshot_root=snapshot_root, manifest_path=manifest_path, relation=relation, + capture_adapter=manifest.capture_adapter, ) ) @@ -214,6 +219,7 @@ def _load_snapshot_relation( snapshot_root: Path, manifest_path: Path, relation: ScenarioSnapshotRelation, + capture_adapter: str, ) -> ScenarioLocalSnapshotLoadedRelation: del manifest_path file_path: Path = snapshot_root / relation.file_path @@ -242,6 +248,7 @@ def _load_snapshot_relation( relation=relation, table_name=table_name, rows=rows, + native_values=capture_adapter == LOCAL_CAPTURE_ADAPTER, ) return ScenarioLocalSnapshotLoadedRelation( kind=relation.kind, @@ -276,16 +283,24 @@ def _insert_rows( relation: ScenarioSnapshotRelation, table_name: str, rows: tuple[dict[str, object], ...], + native_values: bool, ) -> None: if not rows: return column_names: tuple[str, ...] = tuple(column.name for column in relation.columns) - insert_sql: str = _build_insert_sql(table_name=table_name, column_names=column_names) + variant_columns: tuple[bool, ...] = tuple( + not native_values and column.local_type.strip().upper() == LOCAL_TYPE_VARIANT + for column in relation.columns + ) + insert_sql: str = _build_insert_sql( + table_name=table_name, column_names=column_names, variant_columns=variant_columns + ) row_values: list[tuple[object, ...]] = [] for row in rows: values: list[object] = [] - for column_name in column_names: - values.append(_row_value(row=row, column_name=column_name)) + for column_name, variant in zip(column_names, variant_columns, strict=True): + value: object = _row_value(row=row, column_name=column_name) + values.append(_variant_json_text(value=value) if variant else value) row_values.append(tuple(values)) try: connection.executemany(insert_sql, row_values) @@ -340,9 +355,27 @@ def _build_create_table_sql(*, table_name: str, columns: tuple[ScenarioSnapshotC return f"CREATE TABLE {_quote_identifier(table_name)} ({column_defs})" -def _build_insert_sql(*, table_name: str, column_names: tuple[str, ...]) -> str: +def _variant_json_text(*, value: object) -> str | None: + """Another adapter's VARIANT value as JSON text; a string that parses is already JSON text.""" + + if value is None: + return None + if isinstance(value, str): + try: + json.loads(value) + except ValueError: + return json.dumps(value) + return value + return json.dumps(value) + + +def _build_insert_sql( + *, table_name: str, column_names: tuple[str, ...], variant_columns: tuple[bool, ...] +) -> str: columns: str = ", ".join(_quote_identifier(column_name) for column_name in column_names) - placeholders: str = ", ".join("?" for _ in column_names) + placeholders: str = ", ".join( + VARIANT_FROM_JSON_PLACEHOLDER if variant else "?" for variant in variant_columns + ) return f"INSERT INTO {_quote_identifier(table_name)} ({columns}) VALUES ({placeholders})" diff --git a/src/sqlbuild/executor/scenario/constants.py b/src/sqlbuild/executor/scenario/constants.py index 44ca0876c3..1f87454379 100644 --- a/src/sqlbuild/executor/scenario/constants.py +++ b/src/sqlbuild/executor/scenario/constants.py @@ -38,6 +38,7 @@ LOCAL_TYPE_TIME: str = "TIME" LOCAL_TYPE_TIMESTAMP: str = "TIMESTAMP" LOCAL_TYPE_TIMESTAMPTZ: str = "TIMESTAMPTZ" +LOCAL_TYPE_VARIANT: str = "VARIANT" LOCAL_TYPE_UINT128: str = "UINT128" LOCAL_TYPE_VARCHAR: str = "VARCHAR" @@ -162,3 +163,12 @@ UNTYPED_ARRAY_TYPE: str = "[]" JSONL_BLANK_LINE: str = "" LOCAL_SCENARIO_DIALECT: str = "duckdb" + +LOCAL_CAPTURE_ADAPTER: str = "duckdb" +VARIANT_FROM_JSON_PLACEHOLDER: str = "CAST(CAST(? AS JSON) AS VARIANT)" +CANONICAL_JSON_FUNCTION: str = "sqb_canonical_json" +SEMI_STRUCTURED_COMPARISON_TEMPLATE: str = ( + CANONICAL_JSON_FUNCTION + "(CAST({column} AS JSON)) AS {column}" +) +SEMI_STRUCTURED_LOCAL_TYPES: frozenset[str] = frozenset({"VARIANT", "JSON"}) +IDENTIFIER_QUOTE: str = '"' diff --git a/src/sqlbuild/executor/scenario/main/_expected_expectations.py b/src/sqlbuild/executor/scenario/main/_expected_expectations.py index 873065cd4c..9206a76ad0 100644 --- a/src/sqlbuild/executor/scenario/main/_expected_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_expected_expectations.py @@ -9,6 +9,9 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) +from sqlbuild.executor.scenario._helpers.lifecycle.variant_comparison import ( + variant_compared_columns, +) from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_EXPECTED_ERRORED, SCENARIO_EXEC_EXPECTED_FAILED, @@ -30,13 +33,18 @@ def execute_scenario_expected_expectation( actual_relation: str = resolve_relation_location_qualified_name( adapter=adapter, location=expectation.actual_destination ) - comparison_sql: str = build_scenario_expected_comparison_sql( - actual_sql=f"SELECT * FROM {actual_relation}", - expected_sql=expectation.expected_sql, - set_difference_operator=adapter.render_set_difference_operator(), - compared_columns=expectation.compared_columns, - ) try: + comparison_sql: str = build_scenario_expected_comparison_sql( + actual_sql=f"SELECT * FROM {actual_relation}", + expected_sql=expectation.expected_sql, + set_difference_operator=adapter.render_set_difference_operator(), + compared_columns=variant_compared_columns( + adapter=adapter, + connection=connection, + location=expectation.actual_destination, + compared_columns=expectation.compared_columns, + ), + ) cursor: Any = adapter.execute(connection=connection, sql=comparison_sql) row: Any | None = cursor.fetchone() except Exception as exc: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index d065846d30..75cf231937 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1461,3 +1461,25 @@ class SourceEndAuditBuildE2ETestCase: build_arguments: tuple[str, ...] expected_exit_code: int expected_audit_reported: bool + + +@dataclass(frozen=True) +class PythonCheckBuildE2ETestCase: + """A build whose Python check returns `check_body`, and the footer and exit it must end with.""" + + description: str + check_decorator: str + check_body: str + expected_exit_code: int + expected_footer_fragment: str + + +@dataclass(frozen=True) +class DuckDbAttachE2ETestCase: + """A DuckDB project attaching a second database file, built with some connections.""" + + description: str + concurrency: int + expected_archived_rows: list[tuple[object, ...]] + attach_path: str = "archive.duckdb" + expected_archive_root: str = "project" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py new file mode 100644 index 0000000000..0798368e7e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py @@ -0,0 +1,87 @@ +"""E2E coverage of DuckDB `attach` entries across connections, run from the repository root.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import DuckDbAttachE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + '[[connection.attach]]\npath = "{attach_path}"\nalias = "archive"\n\n' + "[scopes]\nenforce_placement = false\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + DuckDbAttachE2ETestCase( + description="one connection attaches the file from the project directory", + concurrency=1, + expected_archived_rows=[(1,)], + ), + DuckDbAttachE2ETestCase( + description="several connections attach the same file once", + concurrency=2, + expected_archived_rows=[(1,)], + ), + DuckDbAttachE2ETestCase( + description="a home-relative path attaches the file in the home directory", + concurrency=1, + expected_archived_rows=[(1,)], + attach_path="~/archive.duckdb", + expected_archive_root="home", + ), + ], + ids=lambda case: case.description, +) +def test_given_attached_duckdb_file_when_building_then_models_land_in_it( + test_case: DuckDbAttachE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML.format(attach_path=test_case.attach_path), + "models/archived_orders.sql": ( + "MODEL (description 'Test model archived_orders.', materialized table, " + "database archive, schema main);\nSELECT 1 AS order_id\n" + ), + "models/customers.sql": ( + "MODEL (description 'Test model customers.', materialized table);\n" + "SELECT 1 AS customer_id\n" + ), + "models/order_ids.sql": ( + "MODEL (description 'Test model order_ids.', materialized table);\n" + 'SELECT o.order_id FROM __ref("archived_orders") AS o\n' + ), + }, + ) + home: Path = tmp_path / "home" + home.mkdir() + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--concurrency", str(test_case.concurrency)), + project_dir=project_dir, + env={"HOME": str(home)}, + ) + archive_root: Path = {"project": project_dir, "home": home}[test_case.expected_archive_root] + + assert built.returncode == 0, built.stdout + built.stderr + assert ( + query_duckdb( + db_path=archive_root / "archive.duckdb", sql="SELECT order_id FROM archived_orders" + ) + == test_case.expected_archived_rows + ), test_case.description + assert not (REPO_ROOT / "archive.duckdb").exists() + assert not (project_dir / "~").exists() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py index 757f96b935..46aa758a33 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py @@ -114,17 +114,25 @@ def test_given_completion_write_failure_during_rebuild_when_retried_then_live_ta "AND partition_end = '2026-01-01T02:00:00'" ), )[0][0] + state_after_failure: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT record_type, partition_start, partition_end FROM main._sqlbuild_microbatches " + "ORDER BY partition_start, record_type" + ), + ) recovered: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build"), project_dir=project_dir, working_dir=project_dir, ) + failure_evidence: str = f"state={state_after_failure}\n{failed.stdout}{failed.stderr}" assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert failed.returncode != test_case.expected_exit_code + assert failed.returncode != test_case.expected_exit_code, failure_evidence assert "simulated completion write failure" in failed.stdout + failed.stderr - assert target_after_failed_state_write == [(1, "a")] - assert completion_count_after_failure == 0 + assert target_after_failed_state_write == [(1, "a")], failure_evidence + assert completion_count_after_failure == 0, failure_evidence assert recovered.returncode == test_case.expected_exit_code, recovered.stdout + recovered.stderr assert ( query_duckdb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py new file mode 100644 index 0000000000..ebe2b25d60 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py @@ -0,0 +1,79 @@ +"""E2E coverage that a build's completion line and exit code account for its Python checks.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + PythonCheckBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckBuildE2ETestCase( + description="a failing check fails the build", + check_decorator='@check(depends_on=model("fact_orders"))', + check_body='return ctx.fail(message="orders present")', + expected_exit_code=1, + expected_footer_fragment="✗ Completed with errors PASS=1 WARN=0 FAIL=1", + ), + PythonCheckBuildE2ETestCase( + description="a check that raises fails the build", + check_decorator='@check(depends_on=model("fact_orders"))', + check_body='raise RuntimeError("boom")', + expected_exit_code=1, + expected_footer_fragment="✗ Completed with errors PASS=1 WARN=0 FAIL=1", + ), + PythonCheckBuildE2ETestCase( + description="a failing warn-severity check warns", + check_decorator='@check(depends_on=model("fact_orders"), severity="warn")', + check_body='return ctx.fail(message="orders present")', + expected_exit_code=0, + expected_footer_fragment="Completed with warnings PASS=1 WARN=1 FAIL=0", + ), + PythonCheckBuildE2ETestCase( + description="a passing check passes", + check_decorator='@check(depends_on=model("fact_orders"))', + check_body='return ctx.pass_(message="orders checked")', + expected_exit_code=0, + expected_footer_fragment="✓ Completed successfully PASS=2 WARN=0 FAIL=0", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_check_when_building_then_completion_line_reports_its_outcome( + test_case: PythonCheckBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_check_outcomes", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "python_check_outcomes"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "warehouse.duckdb"\n\n[defaults]\nmaterialized = "table"\n' + ), + "models/fact_orders.sql": ( + "MODEL (description 'Test model fact_orders.');\nSELECT 1 AS order_id\n" + ), + "python/checks/orders.py": ( + "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n\n\n" + f"{test_case.check_decorator}\n" + "def orders_checked(ctx):\n" + " '''Test check orders_checked.'''\n" + f" {test_case.check_body}\n" + ), + }, + ) + + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert built.returncode == test_case.expected_exit_code, built.stdout + built.stderr + assert test_case.expected_footer_fragment in built.stdout, built.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f9f228811d..b668045bc4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1312,3 +1312,44 @@ class TypedCallContractTestCase: projection: str expected_exit_code: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class MacroDictKeyCompileE2ETestCase: + """A macro call whose dict argument has a key, and the compile outcome it must give.""" + + description: str + argument: str + expected_exit_code: int + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeStarContractE2ETestCase: + """A Snowflake model reading a star under a lowercase contract, and the K005 it reports.""" + + description: str + upstream_columns: str + expected_exit_code: int + expected_k005_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class DialectSyntaxCompileE2ETestCase: + """A project using dialect syntax the SQL parser must analyse, and the compile outcome.""" + + description: str + adapter: str + models: dict[str, str] + expected_exit_code: int + expected_codes: tuple[str, ...] + expected_column_counts: dict[str, int] + + +@dataclass(frozen=True) +class SqlTestCacheMissE2ETestCase: + """A change to compiled SQL test artifacts and the miss reasons a warm compile logs.""" + + description: str + change: Callable[[Path], None] + expected_debug_line: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 291e7ac39e..a3838018fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5338,3 +5338,23 @@ def run_benchmark_generator( check=False, timeout=300, ) + + +def dialect_syntax_model(description: str, sql: str, config: str = "") -> str: + """Build a model file with a description, optional extra header config and its query.""" + + return f"MODEL (description '{description}'{config});\n{sql}\n" + + +def append_to_compiled_sql_tests(project_dir: Path) -> None: + """Append a comment to every compiled SQL test artifact, changing its size.""" + + for artifact in sorted((project_dir / "target/compiled/tests").rglob("*.sql")): + _ = artifact.write_text(artifact.read_text() + "-- changed\n") + + +def flip_compiled_sql_test_bytes(project_dir: Path) -> None: + """Swap one character of every compiled SQL test artifact, keeping its size.""" + + for artifact in sorted((project_dir / "target/compiled/tests").rglob("*.sql")): + _ = artifact.write_text(artifact.read_text().replace("SELECT", "select", 1)) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index 32bd9cb50d..52b91805db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -480,9 +480,6 @@ def test_given_a_typed_call_case_when_compiling_then_it_reports_as_expected( assert fragment not in printed, printed -@pytest.mark.xfail( - reason="a * REPLACE item takes its source column's type; a known polyglot gap", strict=True -) @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py new file mode 100644 index 0000000000..7c55560bcd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py @@ -0,0 +1,207 @@ +"""E2E coverage of dialect syntax that compile analysis parses, types and expands.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DialectSyntaxCompileE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import dialect_syntax_model +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_CONNECTIONS: dict[str, str] = { + "duckdb": '[connection]\ndatabase = "shop.duckdb"\n', + "snowflake": ( + '[connection]\naccount = "example"\nuser = "builder"\n\n' + '[defaults]\ndatabase = "ANALYTICS"\nschema = "MARTS"\n' + ), + "clickhouse": ( + '[connection]\nhost = "localhost"\ndatabase = "default"\n\n[defaults]\nschema = "default"\n' + ), + "bigquery": ( + '[connection]\nproject = "example"\ndataset = "marts"\n\n' + '[defaults]\ndatabase = "example"\nschema = "marts"\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + DialectSyntaxCompileE2ETestCase( + description="stars over derived tables expand to their columns", + adapter="duckdb", + models={ + "unaliased": dialect_syntax_model( + "Unaliased.", "SELECT * FROM (SELECT 1 AS order_id, 'a' AS status)" + ), + "set_operation": dialect_syntax_model( + "Set operation.", + "SELECT * FROM (SELECT 1 AS order_id UNION ALL SELECT 2) AS s", + ), + "renamed": dialect_syntax_model( + "Renamed.", "SELECT * FROM (SELECT 1 AS order_id, 'a' AS status) AS s(a, b)" + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"unaliased": 2, "set_operation": 1, "renamed": 2}, + ), + DialectSyntaxCompileE2ETestCase( + description="star EXCLUDE drops a column before set operation arms are compared", + adapter="duckdb", + models={ + "orders": dialect_syntax_model( + "Orders.", + "SELECT * EXCLUDE (status) FROM (SELECT 1 AS order_id, 'a' AS status) AS t " + "UNION ALL SELECT 2 AS order_id", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"orders": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="column alias lists name table function and table columns", + adapter="duckdb", + models={ + "series": dialect_syntax_model( + "Series.", "SELECT g.n FROM generate_series(1, 3) AS g(n)" + ), + "orders": dialect_syntax_model("Orders.", "SELECT 1 AS id, 'a' AS items"), + "renamed_orders": dialect_syntax_model( + "Renamed orders.", 'SELECT o.a, o.b FROM __ref("orders") AS o(a, b)' + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"series": 1, "renamed_orders": 2}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake typed lambdas and subscripts before a colon path parse", + adapter="snowflake", + models={ + "totals": dialect_syntax_model( + "Totals.", + "SELECT REDUCE(ARRAY_CONSTRUCT(1, 2), 0::DOUBLE, " + "(total DOUBLE, item VARIANT) -> total + item::DOUBLE) AS total", + ), + "names": dialect_syntax_model( + "Names.", + "SELECT o.f[0]:name::VARCHAR AS name " + "FROM (SELECT PARSE_JSON('[{\"name\": 1}]') AS f) AS o", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"totals": 1, "names": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake rejects Oracle's CONNECT_BY_ISLEAF pseudo-column", + adapter="snowflake", + models={ + "tree": dialect_syntax_model( + "Tree.", + "SELECT e.id, CONNECT_BY_ISLEAF AS leaf, LEVEL AS depth " + "FROM (SELECT 1 AS id, NULL AS parent_id) AS e " + "START WITH e.parent_id IS NULL CONNECT BY PRIOR e.id = e.parent_id", + ), + }, + expected_exit_code=1, + expected_codes=("B002",), + expected_column_counts={}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake rejects a CTE whose body is a bare VALUES list", + adapter="snowflake", + models={ + "codes": dialect_syntax_model( + "Codes.", "WITH v AS (VALUES (1), (2)) SELECT v.column1 AS code FROM v" + ), + }, + expected_exit_code=1, + expected_codes=("P001",), + expected_column_counts={}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake accepts VALUES in FROM inside a CTE", + adapter="snowflake", + models={ + "codes": dialect_syntax_model( + "Codes.", + "WITH v AS (SELECT t.code FROM (VALUES (1), (2)) AS t(code)) " + "SELECT v.code FROM v", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"codes": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="a ClickHouse DATE_TRUNC timestamp cursor keeps its timestamp type", + adapter="clickhouse", + models={ + "events": dialect_syntax_model("Events.", "SELECT CAST(now() AS DateTime) AS ts"), + "hourly_events": dialect_syntax_model( + "Hourly events.", + "SELECT DATE_TRUNC('hour', e.ts) AS hour_ts FROM __ref(\"events\") AS e", + ", materialized incremental, incremental_strategy append, cursor hour_ts, " + "cursor_type timestamp, cursor_grain hour", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"hourly_events": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="BigQuery RANGE types parse in casts", + adapter="bigquery", + models={ + "periods": dialect_syntax_model( + "Periods.", + "SELECT CAST(NULL AS RANGE) AS period, " + "CAST(NULL AS RANGE) AS window", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"periods": 2}, + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_syntax_when_compiling_then_analysis_parses_and_types_it( + test_case: DialectSyntaxCompileE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + f'name = "shop"\nadapter = "{test_case.adapter}"\n\n' + f"{_CONNECTIONS[test_case.adapter]}\n[scopes]\nenforce_placement = false\n" + ), + **{f"models/{name}.sql": sql for name, sql in test_case.models.items()}, + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + report: dict[str, Any] = json.loads(compiled.stdout) + column_counts: dict[str, int] = { + model["name"]: model["column_count"] + for model in report.get("resources", {}).get("models", []) + } + assert compiled.returncode == test_case.expected_exit_code, compiled.stdout + compiled.stderr + assert tuple(item["code"] for item in report["diagnostics"]) == test_case.expected_codes + assert { + name: column_counts[name] for name in test_case.expected_column_counts + } == test_case.expected_column_counts, test_case.description diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py new file mode 100644 index 0000000000..9a8b3717d0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py @@ -0,0 +1,87 @@ +"""E2E coverage of macro argument dict keys that Python cannot hash.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + MacroDictKeyCompileE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_MACROS: str = ( + "def show(value):\n" + ' """Show a value."""\n' + " return repr(str(value))\n\n\n" + "def items():\n" + ' """A list of items."""\n' + " return [1]\n\n\n" + "def label():\n" + ' """A label."""\n' + ' return "orders"\n' +) +_HELP: str = ( + "help: Return a string, number, True, False, None, or a tuple of them from a macro used as a " + "dict key" +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroDictKeyCompileE2ETestCase( + description="a nested call returning a list as a dict key is a located P001", + argument="{@py.items(): 2}", + expected_exit_code=1, + expected_fragments=( + "error[P001]: Macro arguments of '@py.show'", + "use the value of a nested macro call, which is a list, as a dict key at line 2, " + "column 18", + _HELP, + ), + ), + MacroDictKeyCompileE2ETestCase( + description="the unhashable call inside a tuple key is the one reported", + argument="{(@py.label(), @py.items()): 2}", + expected_exit_code=1, + expected_fragments=("which is a list, as a dict key at line 2, column 32", _HELP), + ), + MacroDictKeyCompileE2ETestCase( + description="a nested call returning a string is a valid dict key", + argument="{@py.label(): 2}", + expected_exit_code=0, + expected_fragments=("0 errors",), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_dict_key_from_nested_call_when_compiling_then_unhashable_keys_are_located( + test_case: MacroDictKeyCompileE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "macros/python/values.py": _MACROS, + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n" + f"SELECT @py.show({test_case.argument}) AS shown\n" + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = compiled.stdout + compiled.stderr + assert compiled.returncode == test_case.expected_exit_code, output + assert "Traceback" not in output + assert all(fragment in output for fragment in test_case.expected_fragments), output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py new file mode 100644 index 0000000000..160caf3bdd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py @@ -0,0 +1,73 @@ +"""E2E coverage of Snowflake star outputs, which fold to upper case, against lowercase contracts.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + SnowflakeStarContractE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' + 'user = "builder"\n\n[defaults]\ndatabase = "ANALYTICS"\nschema = "MARTS"\n\n' + "[scopes]\nenforce_placement = false\n" +) +_CONTRACT: str = "contract enforced, columns (order_id (type INT), status (type VARCHAR))" + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStarContractE2ETestCase( + description="a star over the declared columns satisfies the lowercase contract", + upstream_columns="1 AS order_id, CAST('a' AS VARCHAR) AS status", + expected_exit_code=0, + expected_k005_columns=(), + ), + SnowflakeStarContractE2ETestCase( + description="an undeclared column a star adds is still reported", + upstream_columns="1 AS order_id, CAST('a' AS VARCHAR) AS status, 2 AS extra", + expected_exit_code=1, + expected_k005_columns=("EXTRA",), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_star_when_compiling_lowercase_contract_then_folded_names_match( + test_case: SnowflakeStarContractE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": ( + f"MODEL (description 'Test model orders.');\nSELECT {test_case.upstream_columns}\n" + ), + "models/order_rows.sql": ( + f"MODEL (description 'Test model order_rows.', {_CONTRACT});\n" + 'SELECT o.* FROM __ref("orders") AS o\n' + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + report: dict[str, Any] = json.loads(compiled.stdout) + codes: list[str] = [item["code"] for item in report["diagnostics"]] + messages: list[str] = [item["message"] for item in report["diagnostics"]] + assert compiled.returncode == test_case.expected_exit_code, compiled.stdout + compiled.stderr + assert codes.count("K005") == len(test_case.expected_k005_columns), messages + assert set(messages) >= { + f"column '{name}' is not declared in enforced contract for model 'order_rows'" + for name in test_case.expected_k005_columns + }, test_case.description diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py new file mode 100644 index 0000000000..2f9750e3d5 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py @@ -0,0 +1,59 @@ +"""E2E coverage that a warm compile logs why it could not keep cached SQL test artifacts.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + SqlTestCacheMissE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + COMPILE_CACHE_REGION_ENV_VAR, + append_to_compiled_sql_tests, + flip_compiled_sql_test_bytes, + run_installed_sqb, +) + +_MISS_LINE: str = "SQL test artifact cache missed" +_ENV: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + + +@pytest.mark.parametrize( + "test_case", + [ + SqlTestCacheMissE2ETestCase( + description="resized artifacts log size_changed", + change=append_to_compiled_sql_tests, + expected_debug_line=f"{_MISS_LINE}: size_changed=", + ), + SqlTestCacheMissE2ETestCase( + description="same-size rewritten artifacts log content_changed", + change=flip_compiled_sql_test_bytes, + expected_debug_line=f"{_MISS_LINE}: content_changed=", + ), + ], + ids=lambda case: case.description, +) +def test_given_changed_sql_test_artifacts_when_compiling_warm_then_debug_log_names_the_miss( + cache_invalidation_project: Path, test_case: SqlTestCacheMissE2ETestCase +) -> None: + project_dir: Path = cache_invalidation_project + debug_compile: tuple[str, ...] = ("--debug", "compile") + cold: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=debug_compile, env=_ENV + ) + warm: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=debug_compile, env=_ENV + ) + + test_case.change(project_dir) + changed: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=debug_compile, env=_ENV + ) + + assert (cold.returncode, warm.returncode, changed.returncode) == (0, 0, 0), changed.stderr + assert _MISS_LINE not in warm.stderr + assert test_case.expected_debug_line in changed.stderr, changed.stderr[-2000:] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py index 2ff3651c63..ac342829ef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py @@ -130,3 +130,13 @@ class DbtTargetConcurrencyE2ETestCase: target_settings: str command: tuple[str, ...] expected_fragment: str + + +@dataclass(frozen=True) +class DbtJsonOptionE2ETestCase: + """A `sqb dbt` command given `--json`, and whether it is rejected before dbt runs.""" + + description: str + command: tuple[str, ...] + expected_returncode: int + expected_stderr_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py new file mode 100644 index 0000000000..7ed4894e24 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py @@ -0,0 +1,45 @@ +"""E2E coverage that `sqb dbt run` and `sqb dbt build` reject `--json`, which they cannot honour.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.dbt._test_types import DbtJsonOptionE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.main.dbt.helpers import prepare_dbt_interop_project +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + DbtJsonOptionE2ETestCase( + description="dbt run rejects --json", + command=("dbt", "run", "--json", "--select", "local_only"), + expected_returncode=1, + expected_stderr_fragment="error[C238]: sqb dbt run does not support --json", + ), + DbtJsonOptionE2ETestCase( + description="dbt build rejects a leading global --json", + command=("--json", "dbt", "build"), + expected_returncode=1, + expected_stderr_fragment="error[C238]: sqb dbt build does not support --json", + ), + ], + ids=lambda case: case.description, +) +def test_given_json_option_when_running_dbt_interop_then_it_is_rejected_before_dbt_runs( + test_case: DbtJsonOptionE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert test_case.expected_stderr_fragment in result.stderr, test_case.description + assert "sqb dbt plan --json" in result.stderr + assert not (project_dir / "dbt_interop.duckdb").exists() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py index 21adbee791..17b823732c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py @@ -2,7 +2,9 @@ from __future__ import annotations +import sqlite3 import subprocess +from contextlib import closing from pathlib import Path import pytest @@ -248,3 +250,67 @@ def test_given_dlt_rest_api_source_when_loading_then_table_is_materialized_and_q sql="SELECT order_id, amount FROM order_totals ORDER BY order_id", ) assert tuple(model_rows) == test_case.expected_model_rows + + +@pytest.mark.parametrize( + "test_case", + [ + DltLocalSourceE2ETestCase( + description="loads local jsonl filesystem source into sqlite", + expected_loaded_rows=((1, 10), (2, 20), (3, 30)), + expected_model_rows=((2, 20), (3, 30)), + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlite_project_when_loading_dlt_filesystem_source_then_models_read_it( + tmp_path: Path, test_case: DltLocalSourceE2ETestCase +) -> None: + project_root: Path = tmp_path / "dlt_sqlite_project" + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="dlt_sqlite_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "dlt_sqlite_project"\nadapter = "sqlite"\n\n' + '[connection]\npath = "warehouse.sqlite"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + "sources/raw.yml": ( + "dlt_sources:\n" + " - type: filesystem\n" + " config:\n" + f' bucket_url: "file://{project_root / "data"}"\n' + ' file_glob: "orders.jsonl"\n' + " resources:\n" + " - name: raw_orders\n" + " description: Test source raw_orders.\n" + " reader: jsonl\n" + " write_disposition: replace\n" + ), + "data/orders.jsonl": ( + '{"order_id": 1, "amount": 10}\n' + '{"order_id": 2, "amount": 20}\n' + '{"order_id": 3, "amount": 30}\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Test model order_totals.', materialized table);\n\n" + 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' + ), + }, + ) + + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert built.returncode == 0, built.stdout + built.stderr + with closing(sqlite3.connect(project_dir / "warehouse.sqlite")) as warehouse: + loaded_rows: list[tuple[object, ...]] = warehouse.execute( + "SELECT order_id, amount FROM raw_orders ORDER BY order_id" + ).fetchall() + model_rows: list[tuple[object, ...]] = warehouse.execute( + "SELECT order_id, amount FROM order_totals ORDER BY order_id" + ).fetchall() + assert tuple(loaded_rows) == test_case.expected_loaded_rows + assert tuple(model_rows) == test_case.expected_model_rows diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 4388eba3f8..354f07f82e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -243,3 +243,13 @@ class PostgresLongModelNameE2ETestCase: model_columns: str expected_exit_code: int expected_scenario_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PostgresTemporaryTestFunctionsE2ETestCase: + """A `sqb test` of an edited function, with or without temporary function copies.""" + + description: str + connection_option: str + expected_exit_code: int + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py new file mode 100644 index 0000000000..d974db8467 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py @@ -0,0 +1,95 @@ +"""E2E coverage of the PostgreSQL option that turns off temporary function copies in `sqb test`.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( + PostgresTemporaryTestFunctionsE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( + build_postgres_project_toml, + build_unique_schema_name, + cleanup_postgres_schema, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_FUNCTION_PATH: str = "functions/sql/is_completed_order.sql" +_FUNCTION_HEADER: str = ( + "FUNCTION (description 'Whether an order status counts as completed.', " + "arguments (order_status VARCHAR), returns BOOLEAN);\n" +) +_TEST: str = ( + 'TEST (name "orders_case");\nWITH\n' + "__ref__order_statuses AS (SELECT 'completed' AS order_status),\n" + "__expected__orders AS (SELECT 'completed' AS order_status, TRUE AS is_completed)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresTemporaryTestFunctionsE2ETestCase( + description="by default the test runs the edited definition", + connection_option="", + expected_exit_code=1, + expected_stdout_fragment="FAIL=1", + ), + PostgresTemporaryTestFunctionsE2ETestCase( + description="with copies off the test runs the built function", + connection_option="temporary_test_functions = false\n", + expected_exit_code=0, + expected_stdout_fragment="PASS=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_edited_function_when_testing_then_option_chooses_the_definition( + test_case: PostgresTemporaryTestFunctionsE2ETestCase, + tmp_path: Path, + postgres_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_temp_functions") + project_toml: str = build_postgres_project_toml( + project_name="orders_temporary_functions", + schema_name=schema_name, + config=postgres_e2e_config, + ).replace("[connection]\n", f"[connection]\n{test_case.connection_option}", 1) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_functions", + repo_files={ + "sqlbuild_project.toml": project_toml, + "models/order_statuses.sql": ( + "MODEL (description 'Test model order_statuses.', materialized table);\n" + "SELECT 'completed' AS order_status\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n" + 'SELECT order_status, __udf("is_completed_order")(order_status) AS is_completed ' + 'FROM __ref("order_statuses")\n' + ), + "tests/unit/orders_case.sql": _TEST, + _FUNCTION_PATH: _FUNCTION_HEADER + "SELECT order_status = 'completed'", + }, + ) + try: + built: subprocess.CompletedProcess[str] = run_sqb( + command=("build", "--no-tests", "--no-audits"), project_dir=project_dir + ) + _ = (project_dir / _FUNCTION_PATH).write_text( + _FUNCTION_HEADER + "SELECT order_status = 'shipped'", encoding="utf-8" + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + finally: + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + assert built.returncode == 0, built.stdout + built.stderr + assert tested.returncode == test_case.expected_exit_code, tested.stdout + tested.stderr + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index d017b36f43..ab99b5aa0e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -289,3 +289,13 @@ class LayoutRepairDryRunE2ETestCase: files: dict[str, str] arguments: tuple[str, ...] expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class PythonNamedMoveE2ETestCase: + """A move of a model a Python check names, and whether `sqb mv` carries it out.""" + + description: str + destination: str + expected_exit_code: int + expected_model_path: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py new file mode 100644 index 0000000000..e0bfe88366 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py @@ -0,0 +1,70 @@ +"""CLI e2e coverage for `sqb mv` on a model that Python code names.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + PythonNamedMoveE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_CHECK: str = ( + "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n\n\n" + '@check(depends_on=model("orders"))\n' + "def orders_present(ctx):\n" + " '''Test check orders_present.'''\n" + " return ctx.pass_()\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonNamedMoveE2ETestCase( + description="a move to a folder keeps the name and moves", + destination="models/marts/", + expected_exit_code=0, + expected_model_path="models/marts/orders.sql", + ), + PythonNamedMoveE2ETestCase( + description="a move to a file of the same name moves", + destination="models/marts/orders.sql", + expected_exit_code=0, + expected_model_path="models/marts/orders.sql", + ), + PythonNamedMoveE2ETestCase( + description="a move that renames the model is refused", + destination="models/marts/fct_orders.sql", + expected_exit_code=1, + expected_model_path="models/orders.sql", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_named_by_python_when_moving_then_only_renames_are_refused( + test_case: PythonNamedMoveE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", + "python/checks/orders.py": _CHECK, + }, + ) + + moved: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "mv", "orders", test_case.destination), project_dir=project_dir + ) + + assert moved.returncode == test_case.expected_exit_code, moved.stdout + moved.stderr + assert (project_dir / test_case.expected_model_path).is_file() + assert (project_dir / "python/checks/orders.py").read_text(encoding="utf-8") == _CHECK diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 654eaa77cc..bd6089a214 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -1,7 +1,7 @@ """Test case types for Rules performance guards.""" from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path @@ -336,3 +336,23 @@ class ModuleStateAttributionCase: files: tuple[tuple[str, str], ...] edited_model: str expected_misses_after_edit: dict[str, int] + + +@dataclass(frozen=True) +class RulePositionE2ETestCase: + """A model file whose query starts somewhere or expands macros, and where a rule must point.""" + + description: str + model_sql: str + expected_location: str + rule: str = "SQBRMODEL102" + macro_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ClickHouseCommentE2ETestCase: + """A ClickHouse model with a line comment holding a quote, and the finding rules report.""" + + description: str + comment: str + expected_location: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py new file mode 100644 index 0000000000..0cc24a9f8b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py @@ -0,0 +1,59 @@ +"""E2E coverage that ClickHouse `#` and `//` line comments are comments to SQLBuild's scanners.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + ClickHouseCommentE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "clickhouse"\n\n[connection]\nhost = "localhost"\n' + 'database = "default"\n\n[defaults]\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseCommentE2ETestCase( + description="a hash comment with an apostrophe is skipped", + comment="# don't keep the star", + expected_location="models/orders.sql:4:8 [SQBRMODEL102]", + ), + ClickHouseCommentE2ETestCase( + description="a double-slash comment with an apostrophe is skipped", + comment="// don't keep the star", + expected_location="models/orders.sql:4:8 [SQBRMODEL102]", + ), + ], + ids=lambda case: case.description, +) +def test_given_clickhouse_line_comment_when_running_rules_then_it_is_not_scanned_as_code( + test_case: ClickHouseCommentE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n" + f"WITH o AS (SELECT 1 AS id)\n{test_case.comment}\nSELECT * FROM o\n" + ), + }, + ) + + ran: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "rules", "run", "SQBRMODEL102"), project_dir=project_dir + ) + + output: str = ran.stdout + ran.stderr + assert "P001" not in output, output + assert test_case.expected_location in output, test_case.description diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py new file mode 100644 index 0000000000..80ce39b09b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py @@ -0,0 +1,95 @@ +"""E2E coverage that model rule findings point at authored file positions, not query ones.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RulePositionE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_HEADER: str = 'MODEL (description "Test model orders.", materialized table);' +_CONTRACT_HEADER: str = ( + 'MODEL (\n description "Test model orders.",\n contract enforced,\n' + " columns (a (type VARCHAR), b (type VARCHAR), c (type VARCHAR)),\n);\n" +) +_BLANK_TO_NULL: dict[str, str] = { + "macros/sql/blank_to_null.sql": ( + 'MACRO (description "Blank to null.", arguments (x VARCHAR), returns VARCHAR);\n\n' + "CASE\n WHEN TRIM(@x) = ''\n THEN NULL\n ELSE\n TRIM(@x)\nEND\n" + ) +} + + +@pytest.mark.parametrize( + "test_case", + [ + RulePositionE2ETestCase( + description="a query on the header's line is offset by the header", + model_sql=f"{_HEADER} WITH o AS (SELECT 1 AS id) SELECT * FROM o\n", + expected_location="models/orders.sql:1:97 [SQBRMODEL102]", + ), + RulePositionE2ETestCase( + description="a query starting on the header's line keeps later lines unshifted", + model_sql=f"{_HEADER} WITH o AS (SELECT 1 AS id)\nSELECT * FROM o\n", + expected_location="models/orders.sql:2:8 [SQBRMODEL102]", + ), + RulePositionE2ETestCase( + description="a query below the header counts file lines", + model_sql=f"{_HEADER}\n-- orders\nWITH o AS (SELECT 1 AS id) SELECT * FROM o\n", + expected_location="models/orders.sql:3:35 [SQBRMODEL102]", + ), + RulePositionE2ETestCase( + description="a star below multi-line macro expansions points at its authored line", + model_sql=( + f"{_HEADER}\nWITH o AS (\n SELECT\n @sql.blank_to_null('a') AS a,\n" + " @sql.blank_to_null('b') AS b\n)\nSELECT * FROM o\n" + ), + macro_files=_BLANK_TO_NULL, + expected_location="models/orders.sql:7:8 [SQBRMODEL102]", + ), + RulePositionE2ETestCase( + description="a star after a macro call on its line points at its authored column", + model_sql=( + f"{_HEADER}\nWITH o AS (SELECT 'a' AS a)\n" + "SELECT @sql.blank_to_null('a') AS x, * FROM o\n" + ), + macro_files=_BLANK_TO_NULL, + expected_location="models/orders.sql:3:38 [SQBRMODEL102]", + ), + RulePositionE2ETestCase( + description="the third of three macro call outputs points at its authored line", + model_sql=( + f"{_CONTRACT_HEADER}\nSELECT\n @sql.blank_to_null('a') AS a,\n" + " @sql.blank_to_null('b') AS b,\n @sql.blank_to_null('c') AS c\n" + ), + rule="SQBRCONTRACT105", + macro_files=_BLANK_TO_NULL, + expected_location="models/orders.sql:10:3 [SQBRCONTRACT105]", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_query_layout_when_running_rules_then_findings_use_file_positions( + test_case: RulePositionE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "models/orders.sql": test_case.model_sql, + **test_case.macro_files, + }, + ) + + ran: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "rules", "run", test_case.rule), project_dir=project_dir + ) + + assert test_case.expected_location in ran.stdout + ran.stderr, ran.stdout + ran.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py index 067698c193..060d9a9f14 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py @@ -326,6 +326,7 @@ class LocalVariantScenarioTestCase: repo_files: dict[str, str] scenario_name: str expected_stdout_fragment: str + expected_returncode: int = 0 @dataclass(frozen=True) @@ -340,3 +341,14 @@ class SnapshotHelperEditTestCase: edited_to: str expected_exit_code: int expected_stdout_fragment: str + + +@dataclass(frozen=True) +class CapturedVariantTextScenarioTestCase: + """A Snowflake-captured VARIANT stored as JSON text, compared with a PARSE_JSON expected.""" + + description: str + captured_payload_text: str + expected_payload_json: str + expected_returncode: int + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index a0f92daa25..c028a31c81 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -1137,7 +1137,12 @@ def build_connect_by_scenario_project_files() -> dict[str, str]: } -def build_local_variant_scenario_project_files() -> dict[str, str]: +def build_local_variant_scenario_project_files( + *, + output: str = "o.payload IS NOT NULL AS has_payload", + expected: str = "TRUE AS has_payload", + fixture: str = '\'{"qty": 2, "tags": ["a"]}\'::JSON::VARIANT', +) -> dict[str, str]: """Build an in-memory DuckDB scenario whose source fixture holds a VARIANT column.""" return { @@ -1152,15 +1157,15 @@ def build_local_variant_scenario_project_files() -> dict[str, str]: ), "models/stg_orders.sql": ( "MODEL (description 'Test model stg_orders.');\n" - "SELECT o.order_id, o.payload IS NOT NULL AS has_payload\n" + f"SELECT o.order_id, {output}\n" 'FROM __source("raw_orders") AS o\n' ), "tests/scenarios/orders__keep_payload.sql": ( "SCENARIO (description 'Test scenario orders__keep_payload.');\n" "WITH\n" "__source__raw_orders AS (\n" - " SELECT 1 AS order_id, '{\"qty\": 2}'::JSON::VARIANT AS payload\n),\n" - "__expected__stg_orders AS (SELECT 1 AS order_id, TRUE AS has_payload)\n" + f" SELECT 1 AS order_id,\n {fixture} AS payload\n),\n" + f"__expected__stg_orders AS (SELECT 1 AS order_id, {expected})\n" "SELECT 1\n" ), } @@ -1192,3 +1197,90 @@ def build_helper_fixture_scenario_project_files() -> dict[str, str]: "SELECT 1\n" ), } + + +SNOWFLAKE_VARIANT_FIXTURE_SQL: str = ( + 'SELECT 1 AS order_id, \'a\' AS status, PARSE_JSON(\'{"qty": 2, "note": "x"}\') AS payload' +) + + +def build_snowflake_variant_scenario_project_files(*, expected_payload_json: str) -> dict[str, str]: + """Build an offline Snowflake project whose scenario compares a VARIANT with PARSE_JSON.""" + + return { + "sqlbuild_project.toml": build_offline_snowflake_project_toml(), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n columns:\n" + ' - {name: order_id, type: "NUMBER(38,0)"}\n' + " - {name: status, type: VARCHAR}\n" + " - {name: payload, type: VARIANT}\n" + ), + "models/stg_payloads.sql": ( + 'MODEL (description "Order payloads.");\n' + 'SELECT order_id, payload FROM __source("raw_orders")\n' + ), + "tests/scenarios/payloads__variant_compare.sql": ( + "SCENARIO (description 'Payload VARIANT values compare equal locally.');\n" + "WITH\n" + f"__source__raw_orders AS ({SNOWFLAKE_VARIANT_FIXTURE_SQL}),\n" + "__expected__stg_payloads AS (SELECT 1 AS order_id, " + f"PARSE_JSON('{expected_payload_json}') AS payload)\n" + "SELECT 1\n" + ), + } + + +def write_snowflake_variant_snapshot( + *, project_dir: Path, scenario_name: str, payload_text: str +) -> None: + """Write the snapshot a Snowflake capture stores: the VARIANT as Snowflake's JSON text.""" + + snapshot_root: Path = project_dir / "tests" / "_scenario_snapshots" / scenario_name + (snapshot_root / "sources").mkdir(parents=True) + row: str = json.dumps({"ORDER_ID": 1, "PAYLOAD": payload_text, "STATUS": "a"}) + "\n" + (snapshot_root / "sources" / "raw_orders.jsonl").write_text(row, encoding="utf-8") + manifest: dict[str, object] = { + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-25T11:10:54Z", + "format": "jsonl", + "input_fingerprint": build_scenario_snapshot_input_fingerprint( + scenario_name=scenario_name, + input_specs=( + ScenarioSnapshotInputSpec( + kind=ScenarioArtifactKind.SOURCE, + logical_name="raw_orders", + file_path=Path("sources") / "raw_orders.jsonl", + capture_sql=SNOWFLAKE_VARIANT_FIXTURE_SQL, + ), + ), + capture_adapter="snowflake", + capture_dialect="snowflake", + ), + "relations": [ + { + "bytes": len(row.encode("utf-8")), + "columns": [ + {"local_type": local, "name": name, "warehouse_type": warehouse} + for name, local, warehouse in ( + ("order_id", "DECIMAL(1, 0)", "NUMBER(1,0)"), + ("status", "TEXT(1)", "VARCHAR(1)"), + ("payload", "VARIANT", "VARIANT"), + ) + ], + "file": "sources/raw_orders.jsonl", + "kind": "source", + "logical_name": "raw_orders", + "row_count": 1, + } + ], + "scenario_name": scenario_name, + "sqlbuild_version": "0.177.0", + "total_bytes": len(row.encode("utf-8")), + "total_rows": 1, + "version": 1, + } + (snapshot_root / "scenario.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py index 82ab7d6f4a..6f8f81b177 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py @@ -8,10 +8,13 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( + CapturedVariantTextScenarioTestCase, LocalVariantScenarioTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( build_local_variant_scenario_project_files, + build_snowflake_variant_scenario_project_files, + write_snowflake_variant_snapshot, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -25,6 +28,54 @@ scenario_name="orders__keep_payload", expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", ), + LocalVariantScenarioTestCase( + description="an expected VARIANT object compares equal to the replayed fixture", + repo_files=build_local_variant_scenario_project_files( + output="o.payload", + expected='\'{"qty": 2, "tags": ["a"]}\'::JSON::VARIANT AS payload', + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + LocalVariantScenarioTestCase( + description="a cast-built VARIANT object compares equal to the replayed fixture", + repo_files=build_local_variant_scenario_project_files( + fixture="{'qty': 2, 'note': 'x'}::VARIANT", + output="o.payload", + expected="{'note': 'x', 'qty': 2}::VARIANT AS payload", + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + LocalVariantScenarioTestCase( + description="a cast-built VARIANT scalar compares equal to the replayed fixture", + repo_files=build_local_variant_scenario_project_files( + fixture="2::VARIANT", output="o.payload", expected="2::VARIANT AS payload" + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + LocalVariantScenarioTestCase( + description="a quoted VARIANT output column compares by its JSON value", + repo_files=build_local_variant_scenario_project_files( + fixture="{'qty': 2, 'note': 'x'}::VARIANT", + output='o.payload AS "Payload"', + expected='\'{"note": "x", "qty": 2}\'::JSON::VARIANT AS "Payload"', + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + LocalVariantScenarioTestCase( + description="a different cast-built VARIANT value still fails the comparison", + repo_files=build_local_variant_scenario_project_files( + fixture="{'qty': 2}::VARIANT", + output="o.payload", + expected="{'qty': 3}::VARIANT AS payload", + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="mismatched=1 missing=1", + expected_returncode=1, + ), ], ids=lambda case: case.description, ) @@ -52,6 +103,65 @@ def test_given_captured_variant_fixture_when_replaying_locally_then_the_scenario project_dir=project_dir, ) - assert replayed.returncode == 0, replayed.stdout + replayed.stderr + assert replayed.returncode == test_case.expected_returncode, replayed.stdout + replayed.stderr assert "X699" not in replayed.stdout assert test_case.expected_stdout_fragment in replayed.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + CapturedVariantTextScenarioTestCase( + description="pretty-printed JSON text in another key order equals the expected value", + captured_payload_text='{\n "note": "x",\n "qty": 2\n}', + expected_payload_json='{"qty": 2, "note": "x"}', + expected_returncode=0, + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + CapturedVariantTextScenarioTestCase( + description="nested objects in another key order equal the expected value", + captured_payload_text='{\n "a": [\n 1,\n {\n "d": 2,\n "c": 3\n }\n ],\n "b": null\n}', + expected_payload_json='{"b": null, "a": [1, {"c": 3, "d": 2}]}', + expected_returncode=0, + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + CapturedVariantTextScenarioTestCase( + description="a different value still fails the comparison", + captured_payload_text='{\n "note": "x",\n "qty": 3\n}', + expected_payload_json='{"qty": 2, "note": "x"}', + expected_returncode=1, + expected_stdout_fragment="mismatched=1 missing=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_variant_text_snapshot_when_replaying_locally_then_json_values_compare_semantically( + test_case: CapturedVariantTextScenarioTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="snowflake_variant", + repo_files=build_snowflake_variant_scenario_project_files( + expected_payload_json=test_case.expected_payload_json + ), + ) + write_snowflake_variant_snapshot( + project_dir=project_dir, + scenario_name="payloads__variant_compare", + payload_text=test_case.captured_payload_text, + ) + + replayed: subprocess.CompletedProcess[str] = run_sqb( + command=( + "--no-color", + "scenario", + "test", + "--local", + "--strict", + "payloads__variant_compare", + ), + project_dir=project_dir, + ) + + assert replayed.returncode == test_case.expected_returncode, replayed.stdout + replayed.stderr + assert test_case.expected_stdout_fragment in replayed.stdout, replayed.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py index 6020e57fa0..3173523ee5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py @@ -96,3 +96,24 @@ class SqliteLoaderE2ETestCase: relations: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] expected_declared_types: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class TextTimestampCursorE2ETestCase: + """A daily model whose timestamp cursor is a text expression, on one adapter.""" + + description: str + adapter: str + connection: str + day_expression: str + expected_exit_code: int + expected_fragment: str + + +@dataclass(frozen=True) +class SqliteTemporalCastE2ETestCase: + """A SQLite model selecting one expression, and the B104 warnings compile reports for it.""" + + description: str + expression: str + expected_warning_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py new file mode 100644 index 0000000000..b8dc12dc73 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py @@ -0,0 +1,73 @@ +"""E2E coverage of the compile warning for date and time casts, which SQLite turns into numbers.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteTemporalCastE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteTemporalCastE2ETestCase( + description="a timestamp cast warns once at the cast", + expression="CAST(o.ordered_at AS TIMESTAMP)", + expected_warning_count=1, + ), + SqliteTemporalCastE2ETestCase( + description="a date cast around a text cast warns once", + expression="CAST(CAST(o.ordered_at AS TEXT) AS DATE)", + expected_warning_count=1, + ), + SqliteTemporalCastE2ETestCase( + description="a SQLite date function does not warn", + expression="datetime(o.ordered_at)", + expected_warning_count=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_sqlite_temporal_cast_when_compiling_then_b104_warns( + test_case: SqliteTemporalCastE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "sqlite"\n\n[connection]\npath = "shop.sqlite"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n" + "SELECT 1 AS order_id, '2026-01-01 10:00:00' AS ordered_at\n" + ), + "models/order_times.sql": ( + "MODEL (description 'Test model order_times.');\n" + f'SELECT o.order_id, {test_case.expression} AS ordered_at FROM __ref("orders") AS o\n' + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + report: dict[str, Any] = json.loads(compiled.stdout) + located: list[tuple[str, str, int, int]] = [ + (item["code"], item["path"], item["line"], item["column"]) for item in report["diagnostics"] + ] + assert located.count(("B104", "models/order_times.sql", 2, 20)) == ( + test_case.expected_warning_count + ), test_case.description + assert [item[0] for item in located].count("B104") == test_case.expected_warning_count diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py new file mode 100644 index 0000000000..e97a7139db --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py @@ -0,0 +1,83 @@ +"""E2E coverage of text columns as timestamp cursors, which SQLite stores timestamps as.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + TextTimestampCursorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_TWO_ORDERS: str = ( + "MODEL (description 'Test model orders.');\n" + "SELECT 1 AS order_id, '2026-01-01 09:30:00' AS ordered_at\n" + "UNION ALL SELECT 2 AS order_id, '2026-01-02 09:30:00' AS ordered_at\n" +) +_THREE_ORDERS: str = _TWO_ORDERS + ( + "UNION ALL SELECT 3 AS order_id, '2026-01-03 09:30:00' AS ordered_at\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + TextTimestampCursorE2ETestCase( + description="sqlite accepts a text timestamp cursor and builds incrementally", + adapter="sqlite", + connection='path = "shop.sqlite"', + day_expression="strftime('%Y-%m-%d 00:00:00', o.ordered_at)", + expected_exit_code=0, + expected_fragment="✓ Completed successfully PASS=2", + ), + TextTimestampCursorE2ETestCase( + description="duckdb still rejects a text timestamp cursor", + adapter="duckdb", + connection='database = "shop.duckdb"', + day_expression="CAST(o.ordered_at AS VARCHAR)", + expected_exit_code=1, + expected_fragment="cursor_type TIMESTAMP does not match column 'order_day' type", + ), + ], + ids=lambda case: case.description, +) +def test_given_text_timestamp_cursor_when_building_twice_then_only_sqlite_accepts_it( + test_case: TextTimestampCursorE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + f'name = "shop"\nadapter = "{test_case.adapter}"\n\n' + f"[connection]\n{test_case.connection}\n\n" + '[defaults]\nmaterialized = "table"\n\n[scopes]\nenforce_placement = false\n' + ), + "models/orders.sql": _TWO_ORDERS, + "models/daily_orders.sql": ( + "MODEL (description 'Test model daily_orders.', materialized incremental,\n" + " incremental_strategy delete_insert, cursor order_day, cursor_type timestamp,\n" + " cursor_grain day, cursor_inputs (orders ordered_at));\n" + f"SELECT o.order_id, {test_case.day_expression} AS order_day\n" + 'FROM __ref("orders") AS o\n' + ), + }, + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + _ = (project_dir / "models/orders.sql").write_text(_THREE_ORDERS, encoding="utf-8") + + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + output: str = first.stdout + first.stderr + second.stdout + second.stderr + assert (first.returncode, second.returncode) == ( + test_case.expected_exit_code, + test_case.expected_exit_code, + ), output + assert test_case.expected_fragment in output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index e48ffb8404..f047989255 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -593,3 +593,23 @@ class MacroMockAliasTestCase: command: tuple[str, ...] expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class ParenthesisedJoinMockTestCase: + """A model reading mocked references through a parenthesised join, and the test outcome.""" + + description: str + model_from: str + expected_exit_code: int + expected_fragment: str + + +@dataclass(frozen=True) +class PythonUdfConnectionsE2ETestCase: + """Unit tests calling a DuckDB Python UDF over several test connections to one database.""" + + description: str + database: str + concurrency: int + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py new file mode 100644 index 0000000000..2eb13148af --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py @@ -0,0 +1,71 @@ +"""E2E coverage of SQL-test mocks for references read through a parenthesised join.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + ParenthesisedJoinMockTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" +) +_TEST: str = ( + "TEST ();\n\n" + "WITH\n" + "__ref__orders AS (SELECT 7 AS order_id),\n" + "__ref__customers AS (SELECT 7 AS order_id),\n" + "__expected__order_customers AS (SELECT 7 AS order_id)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + ParenthesisedJoinMockTestCase( + description="both sides of a parenthesised join use their mocks", + model_from=( + '(__ref("orders") AS o JOIN __ref("customers") AS c ON o.order_id = c.order_id)' + ), + expected_exit_code=0, + expected_fragment="PASS=1 FAIL=0", + ), + ParenthesisedJoinMockTestCase( + description="an unparenthesised join uses its mocks", + model_from='__ref("orders") AS o JOIN __ref("customers") AS c ON o.order_id = c.order_id', + expected_exit_code=0, + expected_fragment="PASS=1 FAIL=0", + ), + ], + ids=lambda case: case.description, +) +def test_given_parenthesised_join_when_testing_then_both_references_use_their_mocks( + test_case: ParenthesisedJoinMockTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": 'MODEL (description "Orders.");\n\nSELECT 1 AS order_id\n', + "models/customers.sql": 'MODEL (description "Customers.");\n\nSELECT 1 AS order_id\n', + "models/order_customers.sql": ( + 'MODEL (description "Orders with customers.");\n\n' + f"SELECT o.order_id FROM {test_case.model_from}\n" + ), + "tests/unit/test_order_customers.sql": _TEST, + }, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + + assert tested.returncode == test_case.expected_exit_code, tested.stdout + tested.stderr + assert test_case.expected_fragment in tested.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py new file mode 100644 index 0000000000..a22a256835 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py @@ -0,0 +1,80 @@ +"""E2E coverage that every `sqb test` connection can call the project's DuckDB Python UDFs.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + PythonUdfConnectionsE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_UDF: str = ( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"order_status": "STRING"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(order_status: str | None) -> bool:\n" + ' """Test function main."""\n' + ' return order_status == "completed"\n' +) +_TEST: str = ( + "TEST ();\nWITH\n" + "__ref__order_statuses AS (SELECT 'completed' AS order_status),\n" + "__expected__orders AS (SELECT 'completed' AS order_status, TRUE AS is_completed)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUdfConnectionsE2ETestCase( + description="in-memory databases get the UDF on every connection", + database=":memory:", + concurrency=4, + expected_stdout_fragment="PASS=8 FAIL=0 TOTAL=8", + ), + PythonUdfConnectionsE2ETestCase( + description="connections to one database file share the UDF", + database="orders.duckdb", + concurrency=4, + expected_stdout_fragment="PASS=8 FAIL=0 TOTAL=8", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_udf_when_testing_on_several_connections_then_every_test_can_call_it( + test_case: PythonUdfConnectionsE2ETestCase, tmp_path: Path +) -> None: + tests: dict[str, str] = {f"tests/unit/test_orders_{index}.sql": _TEST for index in range(1, 9)} + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_python_udf", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders_python_udf"\nadapter = "duckdb"\n\n' + f'[connection]\ndatabase = "{test_case.database}"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "functions/python/is_completed_order.py": _UDF, + "models/order_statuses.sql": ( + "MODEL (description 'Test model order_statuses.');\n" + "SELECT 'completed' AS order_status\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n" + 'SELECT s.order_status, __udf("is_completed_order")(s.order_status) AS is_completed\n' + 'FROM __ref("order_statuses") AS s\n' + ), + **tests, + }, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test", "--concurrency", str(test_case.concurrency)), + project_dir=project_dir, + ) + + assert tested.returncode == 0, tested.stdout + tested.stderr + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py index cbb700083b..4db0cacd83 100644 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py +++ b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py @@ -1,6 +1,8 @@ from __future__ import annotations +import sqlite3 import subprocess +from contextlib import closing from pathlib import Path import pytest @@ -79,6 +81,57 @@ def test_given_postgres_source_when_loading_with_ingestr_then_duckdb_target_has_ ) +@pytest.mark.parametrize( + "test_case", + [ + IngestrE2ETestCase( + description="loads a sqlite source into a sqlite destination", + expected_rows=((1, "new"), (2, "paid")), + expected_stdout_fragments=("ingestr execution", "raw_sqlite_orders", "OK"), + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlite_project_when_loading_with_ingestr_then_target_has_rows( + tmp_path: Path, + test_case: IngestrE2ETestCase, +) -> None: + source_path: Path = tmp_path / "source.db" + with closing(sqlite3.connect(source_path)) as source, source: + source.execute("CREATE TABLE orders (order_id INTEGER, status TEXT)") + source.execute("INSERT INTO orders VALUES (1, 'new'), (2, 'paid')") + project_dir: Path = tmp_path / "sqlite_to_sqlite" + write_project_files( + project_dir=project_dir, + files={ + "sqlbuild_project.toml": ( + 'name = "sqlite_to_sqlite"\nadapter = "sqlite"\n\n' + '[connection]\npath = "target.sqlite"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_sqlite_orders\n description: Test source raw_sqlite_orders.\n" + " ingestr:\n" + f' source_uri: "sqlite:///{source_path}"\n' + " source_table: orders\n" + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( + command=("--no-color", "load"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout + with closing(sqlite3.connect(project_dir / "target.sqlite")) as target: + rows: list[tuple[object, ...]] = target.execute( + "SELECT order_id, status FROM raw_sqlite_orders ORDER BY order_id" + ).fetchall() + assert tuple(rows) == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json b/tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json new file mode 100644 index 0000000000..3de33d2fe5 --- /dev/null +++ b/tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json @@ -0,0 +1,294 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "B002", + "message": "Unknown column 'missing' in stg_orders", + "resource_type": "model", + "resource_name": "customer_totals", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 64, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 64, + "end_line": 5, + "end_column": 71 + }, + "notes": [ + "stg_orders has: amount, customer_id, order_id, status" + ] + }, + { + "phase": "compile", + "severity": "warning", + "code": "B104", + "message": "SQLite casts to TIMESTAMP with NUMERIC affinity and keeps only the leading number, so '2026-01-01 10:00:00' becomes 2026", + "resource_type": "model", + "resource_name": "customer_totals", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 21, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 21, + "end_line": 5, + "end_column": 45 + }, + "help": "keep the ISO-8601 text without a cast, or convert it with datetime(), date() or strftime()" + } + ] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, CAST(amount AS TIMESTAMP) AS total_amount, missing", + "FROM __ref(\"stg_orders\")", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py b/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py index 80e42963d6..ca01171fc0 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py @@ -17,12 +17,12 @@ BuildAttachSqlTestCase( description="attaches database with path only", attach_entry={"path": "/tmp/other.duckdb"}, - expected_sql="ATTACH '/tmp/other.duckdb'", + expected_sql="ATTACH IF NOT EXISTS '/tmp/other.duckdb'", ), BuildAttachSqlTestCase( description="attaches database with alias", attach_entry={"path": "/tmp/other.duckdb", "alias": "other"}, - expected_sql="ATTACH '/tmp/other.duckdb' AS other", + expected_sql="ATTACH IF NOT EXISTS '/tmp/other.duckdb' AS other", ), BuildAttachSqlTestCase( description="attaches database with type and read_only", @@ -32,17 +32,17 @@ "type": "sqlite", "read_only": True, }, - expected_sql="ATTACH 'sqlite.db' AS sqlite_db (TYPE sqlite, READ_ONLY)", + expected_sql="ATTACH IF NOT EXISTS 'sqlite.db' AS sqlite_db (TYPE sqlite, READ_ONLY)", ), BuildAttachSqlTestCase( description="attaches database with read_only false omits option", attach_entry={"path": "/tmp/data.duckdb", "read_only": False}, - expected_sql="ATTACH '/tmp/data.duckdb'", + expected_sql="ATTACH IF NOT EXISTS '/tmp/data.duckdb'", ), BuildAttachSqlTestCase( description="attaches database with type only", attach_entry={"path": "pg.db", "type": "postgres"}, - expected_sql="ATTACH 'pg.db' (TYPE postgres)", + expected_sql="ATTACH IF NOT EXISTS 'pg.db' (TYPE postgres)", ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 64cc99889a..c88a32de38 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -24,6 +24,8 @@ EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() EMITTED_TAIL: bytes = b"after the warning\n" MEBIBYTE: int = 1024 * 1024 +HANG_SECONDS: int = 600 +HANG_DUMP_GRACE_SECONDS: int = 30 EMPTY_RESULT: bytes = b'{"rows": []}' NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) PROJECT_TOML: str = ( @@ -108,22 +110,36 @@ def write_project(project_dir: Path, *, extra_toml: str = "", targets: int = 1) def environment(**overrides: str) -> dict[str, str]: - """The process environment for one CLI run.""" + """The process environment for one CLI run; a hang dump prints every thread's stack.""" - return {**os.environ, "NO_COLOR": "1", **overrides} + return {**os.environ, "NO_COLOR": "1", "PYTHONFAULTHANDLER": "1", **overrides} def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: - """Run the Python CLI once in the project and capture exact streams.""" + """Run the Python CLI once in the project and capture exact streams, or fail with its stacks.""" + + process: subprocess.Popen[bytes] = start(project_dir, *args, **env) + try: + stdout, stderr = process.communicate(timeout=HANG_SECONDS) + except subprocess.TimeoutExpired: + raise AssertionError( + f"sqlbuild {shlex.join(args)} hung for {HANG_SECONDS}s:\n{hang_dump(process)}" + ) from None + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) - return subprocess.run( - [str(PYTHON_CLI), *args], - cwd=project_dir, - capture_output=True, - check=False, - env=environment(**env), - timeout=600, - ) + +def hang_dump(process: subprocess.Popen[bytes]) -> str: + """Abort a hung CLI's process group so it prints every thread's stack, and return its stderr.""" + + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGABRT) + try: + _, stderr = process.communicate(timeout=HANG_DUMP_GRACE_SECONDS) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + _, stderr = process.communicate() + return stderr.decode(errors="replace") def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: @@ -267,15 +283,22 @@ def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list ] reads: int = 0 unexpected_failures: int = 0 - while any(build.poll() is None for build in builds): + deadline: float = time.monotonic() + HANG_SECONDS + while any(build.poll() is None for build in builds) and time.monotonic() < deadline: reader: subprocess.CompletedProcess[bytes] = run( project_dir, "recall", "query", "--json", "SELECT COUNT(*) FROM invocations" ) reads += 1 unexpected_failures += int(reader.returncode != 0 and b"[C971]" not in reader.stderr) + hung: list[str] = list(map(hang_dump, filter(_still_running, builds))) + assert not hung, f"{len(hung)} builds still ran after {HANG_SECONDS}s:\n" + "\n".join(hung) return [build.wait(timeout=600) for build in builds], reads, unexpected_failures +def _still_running(process: subprocess.Popen[bytes]) -> bool: + return process.poll() is None + + def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]: """A build with the seconds until its first stderr byte and until it exited.""" diff --git a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py index b8ae9adfd3..398b3d8e02 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py @@ -37,10 +37,10 @@ class UnparsedLineageTestCase: @dataclass(frozen=True) -class ParserPanicLineageTestCase: - """A model whose SQL panics the native parser, failing lineage as an internal error.""" +class TruncatedSqlLineageTestCase: + """A model whose SQL ends mid-expression, which fast lineage leaves out.""" description: str dialect: str query_sql: str - expected_message: str + expected_has_model: bool diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index e9c5cec76c..3dcaebd13c 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -11,7 +11,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( ProjectColumnLineage, compiled_model_record, @@ -29,7 +28,7 @@ from tests.integration.src.sqlbuild.compiler.lineage._test_types import ( FormerlyDeferredLineageTestCase, GeneratedLineageParityTestCase, - ParserPanicLineageTestCase, + TruncatedSqlLineageTestCase, UnparsedLineageTestCase, ) from tests.integration.src.sqlbuild.compiler.lineage.helpers import ( @@ -241,20 +240,17 @@ def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_rais @pytest.mark.parametrize( "test_case", [ - ParserPanicLineageTestCase( - description="truncated T-SQL the parser panics on, which crashes the wheel's process", + TruncatedSqlLineageTestCase( + description="truncated T-SQL the parser once panicked on is left out", dialect="tsql", query_sql="SELECT IF(region > 1, re", - expected_message=( - "NativeCompilerError: native SQL compilation panicked " - "(native fast lineage of request model 0)" - ), + expected_has_model=False, ), ], ids=lambda case: case.description, ) -def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compiler_error( - test_case: ParserPanicLineageTestCase, tmp_path: Path +def test_given_truncated_sql_when_building_fast_lineage_then_omits_the_model( + test_case: TruncatedSqlLineageTestCase, tmp_path: Path ) -> None: project: CompiledProjectView = compiled_project( project_dir=tmp_path / "project", @@ -272,10 +268,12 @@ def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compil ), ) - with pytest.raises(native_module.NativeCompilerError) as raised: - _ = fast_column_lineage(project=broken, dialect=test_case.dialect, model_names=None) + native: ProjectColumnLineage | None = fast_column_lineage( + project=broken, dialect=test_case.dialect, model_names=None + ) - assert str(raised.value) == test_case.expected_message + assert native is not None + assert native.has_model(project.models[0].name) is test_case.expected_has_model if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index 17fd7216d5..c7047fc3ae 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -53,10 +53,10 @@ class OptOutTestCase: @dataclass(frozen=True) -class NativePanicTestCase: - """A project whose native assembly panics in the parser.""" +class TruncatedHookTestCase: + """A project whose hook SQL ends mid-expression, which assembly reports as invalid input.""" description: str files: dict[str, str] dialect: str - expected_message: str + expected_message_fragment: str diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index e3bc480ef9..e8f15d0d8e 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -10,7 +10,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -26,8 +25,8 @@ AssemblyDeferralTestCase, EdgeAssemblyTestCase, GeneratedAssemblyParityTestCase, - NativePanicTestCase, OptOutTestCase, + TruncatedHookTestCase, WindowsEnvironmentTestCase, ) from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( @@ -255,8 +254,8 @@ def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejecti @pytest.mark.parametrize( "test_case", [ - NativePanicTestCase( - description="a T-SQL hook the parser panics on", + TruncatedHookTestCase( + description="a T-SQL hook the parser once panicked on", files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/orders.sql": ( @@ -265,25 +264,24 @@ def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejecti ), }, dialect="tsql", - expected_message=( - "NativeCompilerError: native SQL compilation panicked (native project assembly)" - ), + expected_message_fragment="Expected ')' to close the IF condition", ), ], ids=lambda case: case.description, ) -def test_given_native_assembly_panics_when_assembling_then_raises_without_python_retry( - test_case: NativePanicTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_given_truncated_hook_sql_when_assembling_then_reports_it_without_python_retry( + test_case: TruncatedHookTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: inputs: CompileInputsView = project_inputs( project_dir=tmp_path / "project", files=test_case.files ) calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) - with pytest.raises(native_module.NativeCompilerError) as raised: + with pytest.raises(CompileInputError) as raised: _ = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect=test_case.dialect), ) - assert (str(raised.value), calls) == (test_case.expected_message, Counter()) + assert test_case.expected_message_fragment in str(raised.value) + assert calls == Counter() diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index 97d0d5d5d4..e00348448c 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -18,6 +18,7 @@ "adapters/bigquery/_helpers/relation_usage.py", "adapters/bigquery/classes/bigquery_adapter.py", "adapters/databricks/classes/databricks_adapter.py", + "adapters/databricks/constants.py", "adapters/postgres/classes/postgres_adapter.py", "adapters/snowflake/_helpers/metadata_types.py", "adapters/snowflake/_helpers/show_metadata.py", diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index 4f82e53f10..3f61d0ccf0 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -189,3 +189,14 @@ class DatabricksRelationUsageFailureTestCase: message: str sqlstate: str | None expected_error_match: str + + +@dataclass(frozen=True) +class DatabricksSchemaExistsTestCase: + """A schema existence check with or without a catalog, and what it must query and answer.""" + + description: str + database: str | None + rows: list[tuple[object, ...]] + expected_exists: bool + expected_query_fragment: str diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index 046e6ad499..122d6bce96 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -27,6 +27,7 @@ DatabricksRenderDurableCloneTestCase, DatabricksRenderPythonFunctionTestCase, DatabricksRenderTableFunctionTestCase, + DatabricksSchemaExistsTestCase, DatabricksStringTypeCastRenderingTestCase, DatabricksTableFreshnessBatchTestCase, DatabricksTableFreshnessFallbackTestCase, @@ -752,3 +753,46 @@ def test_given_candidate_relations_when_adding_age_metadata_then_databricks_read str(enriched[0].created_at), str(enriched[0].last_altered_at), ) == test_case.expected_timestamps + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksSchemaExistsTestCase( + description="a schema in the session's current catalog is found without a catalog", + database=None, + rows=[(1,)], + expected_exists=True, + expected_query_fragment="SELECT 1 FROM information_schema.schemata", + ), + DatabricksSchemaExistsTestCase( + description="a schema missing from the current catalog is reported missing", + database=None, + rows=[], + expected_exists=False, + expected_query_fragment="SELECT 1 FROM information_schema.schemata", + ), + DatabricksSchemaExistsTestCase( + description="an explicit catalog is queried through its information schema", + database="main", + rows=[(1,)], + expected_exists=True, + expected_query_fragment="SELECT 1 FROM `main`.information_schema.schemata", + ), + ], + ids=lambda case: case.description, +) +def test_given_databricks_schema_when_checking_existence_then_current_catalog_is_used_without_one( + test_case: DatabricksSchemaExistsTestCase, +) -> None: + cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor(rows=test_case.rows) + + exists: bool = DatabricksAdapter().schema_exists( + connection=cast(Any, FakeDatabricksMetadataConnection((cursor,))), + database=test_case.database, + schema="analytics", + ) + + assert exists is test_case.expected_exists, test_case.description + assert cursor.executed_sql is not None + assert test_case.expected_query_fragment in cursor.executed_sql, test_case.description diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index f3acc3ef7a..caaa9c3b86 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -100,11 +100,11 @@ def plan_dbt_interop_from_project( "test_case", [ DbtExecutionWrapperTestCase( - description="dbt run strips local json and verbose flags before execution", + description="dbt run strips the local verbose flag before execution", command_name="run", - args=("--json", "--verbose", "--select", "tag:nightly"), + args=("--verbose", "--select", "tag:nightly"), expected_forwarded_args=("--select", "tag:nightly"), - expected_progress_stream=sys.stderr, + expected_progress_stream=sys.stdout, ), DbtExecutionWrapperTestCase( description="dbt build keeps human output on stdout", diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 9c59b2342d..d67ac5307a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -409,3 +409,14 @@ def order_status_callee_test_plan(*, callee_body_sql: str, test_count: int) -> P ), function_locations={**plan.function_locations, "status_matches": callee_destination}, ) + + +def describe_lifecycle_events(events: list[LifecycleEvent]) -> list[str]: + """Describe events for an assertion message, with the invocation and run that produced them.""" + + return [ + f"{event.event_type} invocation={event.invocation_id} run={event.run_id} " + f"adapter={event.payload.get('adapter')} purpose={event.payload.get('purpose')} " + f"kind={event.payload.get('statement_kind')} resource={event.resource_id}" + for event in events + ] diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index 2a276b5d77..d6fdebf610 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -43,6 +43,7 @@ ScenarioPipelinePlanBuilder, ScenarioPipelineTestAdapter, build_scenario_pipeline_result, + describe_lifecycle_events, local_snapshot_loader_for_test_case, ) @@ -139,7 +140,7 @@ def complete_scenario( "resource_attempt_completed", "resource_attempt_started", "resource_attempt_failed", - ) + ), describe_lifecycle_events(lifecycle_events) assert tuple(event.resource_id for event in resource_events) == ( "sql_scenario:passing_scenario", "sql_scenario:passing_scenario", diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py index 63e0cc8ae0..bd4f84044e 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py @@ -192,3 +192,22 @@ class LocalScenarioSqlTestCase: source_dialect: str | None sql: str expected_sql: str + + +@dataclass(frozen=True) +class ScenarioLocalVariantLoadTestCase: + """A Snowflake-captured VARIANT JSONL value and the variant type it must load as.""" + + description: str + value: object + expected_variant_type: str + expected_matches_parsed_json: bool + + +@dataclass(frozen=True) +class CanonicalJsonTestCase: + """JSON text and the canonical text equal values of it share.""" + + description: str + text: str | None + expected_canonical: str | None diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py new file mode 100644 index 0000000000..73dae70228 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py @@ -0,0 +1,46 @@ +"""Tests for the canonical JSON text scenario comparisons use for VARIANT and JSON values.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.executor.scenario._helpers.lifecycle.canonical_json import canonical_json +from tests.unit.src.sqlbuild.executor.scenario._helpers._test_types import CanonicalJsonTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + CanonicalJsonTestCase( + description="pretty-printed keys in another order", + text='{\n "qty": 2,\n "note": "x"\n}', + expected_canonical='{"note":"x","qty":2}', + ), + CanonicalJsonTestCase( + description="nested objects are sorted and arrays keep their order", + text='{"b": [3, {"z": 1, "a": null}], "a": true}', + expected_canonical='{"a":true,"b":[3,{"a":null,"z":1}]}', + ), + CanonicalJsonTestCase( + description="integral numbers lose trailing zeros", + text='{"a": 2.0, "b": 1E1}', + expected_canonical='{"a":2,"b":10}', + ), + CanonicalJsonTestCase( + description="fractional numbers keep exact digits", + text="[2.50, 0.0000001]", + expected_canonical="[2.5,0.0000001]", + ), + CanonicalJsonTestCase( + description="strings stay strings and keep non-ASCII text", + text='["2", "caf\\u00e9"]', + expected_canonical='["2","café"]', + ), + CanonicalJsonTestCase(description="NULL stays NULL", text=None, expected_canonical=None), + ], + ids=lambda case: case.description, +) +def test_given_json_text_when_canonicalising_then_equal_values_share_one_text( + test_case: CanonicalJsonTestCase, +) -> None: + assert canonical_json(test_case.text) == test_case.expected_canonical diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py index 891bf8bb83..5f9f20be74 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py @@ -21,6 +21,7 @@ from tests.unit.src.sqlbuild.executor.scenario._helpers._test_types import ( ScenarioLocalSnapshotLoadErrorTestCase, ScenarioLocalSnapshotLoadTestCase, + ScenarioLocalVariantLoadTestCase, ) from tests.unit.src.sqlbuild.executor.scenario._helpers.helpers import ( build_local_snapshot_load_manifest, @@ -178,3 +179,58 @@ def test_given_invalid_local_snapshot_when_loading_then_raises_coded_error( expected_fragment: str for expected_fragment in test_case.expected_error_fragments: assert expected_fragment in error_text + + +@pytest.mark.parametrize( + "test_case", + [ + ScenarioLocalVariantLoadTestCase( + description="driver JSON text of an object loads as that object", + value='{\n "qty": 2\n}', + expected_variant_type="OBJECT(qty)", + expected_matches_parsed_json=True, + ), + ScenarioLocalVariantLoadTestCase( + description="driver JSON text of a string loads as the string", + value='"placed"', + expected_variant_type="VARCHAR", + expected_matches_parsed_json=False, + ), + ScenarioLocalVariantLoadTestCase( + description="text that is not JSON loads as a string", + value="placed", + expected_variant_type="VARCHAR", + expected_matches_parsed_json=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_variant_text_when_loading_into_duckdb_then_it_loads_as_parsed_json( + tmp_path: Path, + test_case: ScenarioLocalVariantLoadTestCase, +) -> None: + manifest: ScenarioSnapshotManifest = build_local_snapshot_load_manifest( + input_fingerprint=INPUT_FINGERPRINT, + columns=(("payload", "VARIANT", "VARIANT"),), + ) + write_local_snapshot_load_test_files( + project_dir=tmp_path, manifest=manifest, rows=({"PAYLOAD": test_case.value},) + ) + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + + result: ScenarioLocalSnapshotLoadResult = load_scenario_snapshot_into_duckdb( + project_dir=tmp_path, + scenario_name=SCENARIO_NAME, + current_input_fingerprint=INPUT_FINGERPRINT, + connection=connection, + ) + loaded: tuple[object, ...] | None = connection.execute( + "SELECT variant_typeof(payload), " + "payload IN (SELECT CAST(CAST('{\"qty\": 2}' AS JSON) AS VARIANT)) " + f'FROM "{result.relations[0].table_name}"' + ).fetchone() + + assert loaded == ( + test_case.expected_variant_type, + test_case.expected_matches_parsed_json, + ), test_case.description diff --git a/website/src/content/docs/docs/cli/dbt.mdx b/website/src/content/docs/docs/cli/dbt.mdx index 49e6e0a693..1238052e98 100644 --- a/website/src/content/docs/docs/cli/dbt.mdx +++ b/website/src/content/docs/docs/cli/dbt.mdx @@ -33,6 +33,9 @@ Build dbt models first (including dbt tests), then SQLBuild models with audits. sqb dbt build [--select ...] [--exclude ...] ``` +`sqb dbt run` and `sqb dbt build` have no JSON output: they reject `--json` with `C238` before +dbt starts. Use `sqb dbt plan --json` for a machine-readable preview of the combined work. + ### Plan safety policies After dbt finishes, `sqb dbt run` and `sqb dbt build` apply the same plan safety policies as diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index 84f956f65d..f0d4911c2a 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -149,7 +149,7 @@ The command refuses, and writes nothing, when: - a reference is produced by a macro, or a column is read inside SQL that a macro generates; - a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; - a downstream model's `SELECT *` would silently rename its output, without `--cascade`; -- Python code or Python node SQL strings name the model or column; +- Python code or Python node SQL strings name a model being renamed, or a column being renamed. A move that keeps the model's name leaves Python references to it as they are; - a Python import of a moved Python file cannot be rewritten safely. A rename whose new name belongs to another layer's folder, under the layer-folder rules `SQBRPROJECT102` and `SQBRPROJECT105`, fails verification because `sqb rename` keeps the file where it is. When those rules are the only failures, the output says where the model belongs and prints the one-step [`sqb mv`](#rename-or-move) command, with that layer's folder and the current sub-folders filled in: diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index 90b0e70a7d..f09a229587 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -49,6 +49,13 @@ alias = "external" read_only = true ``` +A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`, +and a path starting with `~/` is relative to your home directory. This also applies to `attach` +entries that come from a dbt profile: they resolve from the SQLBuild project directory, not from +the dbt project or the working directory. +Every connection of a command attaches it once: connections to the same project database share the +attachment. + ## Table promotion mode DuckDB defaults to `staged` promotion: tables are materialized into a staging table, audited, then swapped into the target. This is configurable in `settings`: diff --git a/website/src/content/docs/docs/concepts/adapters/postgres.mdx b/website/src/content/docs/docs/concepts/adapters/postgres.mdx index e0bb8c2153..c38c0a2714 100644 --- a/website/src/content/docs/docs/concepts/adapters/postgres.mdx +++ b/website/src/content/docs/docs/concepts/adapters/postgres.mdx @@ -37,6 +37,7 @@ schema = "analytics_dev" | `user` | Database user | | `password` | Database password | | `dbname` | Database name | +| `temporary_test_functions` | `false` stops `sqb test` from creating session-temporary (`pg_temp`) copies of project SQL functions, so tests call the built functions. Use it behind a transaction-mode pooler such as PgBouncer, where later statements can run in a session without the copies (default: `true`) | Connection fields are passed to `psycopg.connect()`. See the [psycopg documentation](https://www.psycopg.org/psycopg3/docs/api/connections.html) for all available options. The target remains authoritative for database and schema qualification. @@ -67,4 +68,5 @@ schema = "dev" `sqb test` creates each SQL function the selected tests call in the session's `pg_temp` schema and calls it as `pg_temp.`, so tests use the current project definition without a build. Temporary functions belong to one server session: behind a transaction-mode pooler, run `sqb test` through a -direct connection or a session-mode pool. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). +direct connection or a session-mode pool, or set `temporary_test_functions = false` so tests call the +built functions. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index 48bdd38330..594b94f6fe 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -62,7 +62,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly its declared type; a computed column gets the type SQLBuild infers at compile time. - Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not produce timestamps in SQLite (they keep only the leading number), so write timestamps as text - literals or use SQLite date functions. SQLBuild never applies such a cast itself: source, + literals or use SQLite date functions. Compile warns with `B104` at each such cast in model SQL. SQLBuild never applies such a cast itself: source, contract and type-change casts to a date or time type keep the stored text, and source loaders write Python `datetime` values as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`. - Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the @@ -70,6 +70,9 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against `__cursor_start()` and `__cursor_end()` compare the stored text as is. +- A `cursor_type timestamp` column may be text, such as + `strftime('%Y-%m-%d 00:00:00', ordered_at)`, because SQLite stores timestamps as text. Other + adapters reject a text timestamp cursor with `B301`. - Booleans are stored as `0` and `1`. - JSON, arrays and objects are stored as JSON text. - `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)` @@ -93,6 +96,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly | `sqb diff`, `sqb clone` | Supported; clones are physical copies | | Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | | Statement time limits | Connection interrupt | +| `sqb load`: Python, dlt and ingestr loaders | Supported; dlt writes through its `sqlalchemy` destination | ## Scenarios diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index c4d5355e9e..dd93a67b34 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -22,8 +22,8 @@ def check_orders_exported(ctx: CheckContext): """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: - return ctx.fail("no orders exported") - return ctx.pass_("orders exported") + return ctx.fail(message="no orders exported") + return ctx.pass_(message="orders exported") ``` The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`. @@ -40,7 +40,7 @@ def orders_present(ctx): """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] - return ctx.pass_() if count else ctx.fail("no orders") + return ctx.pass_() if count else ctx.fail(message="no orders") ``` Do not hard-code relation names in check SQL; see [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). @@ -75,7 +75,7 @@ Returning `None` is not allowed - checks must be explicit. def orders_freshness(ctx): """Orders freshness.""" if stale(): - return ctx.fail("export is stale") # recorded as a warning, does not fail the build + return ctx.fail(message="export is stale") # recorded as a warning, does not fail the build return ctx.pass_() ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx index 6994e8fbee..0af21a86e8 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx @@ -68,7 +68,7 @@ def orders_pipeline(): @check(depends_on=export) def orders_export_check(ctx): """Check that the orders export is ready.""" - return ctx.pass_("export ready") + return ctx.pass_(message="export ready") return [load, export, orders_export_check] ``` diff --git a/website/src/content/docs/docs/concepts/scenarios.mdx b/website/src/content/docs/docs/concepts/scenarios.mdx index 0a70f20061..828b5835d8 100644 --- a/website/src/content/docs/docs/concepts/scenarios.mdx +++ b/website/src/content/docs/docs/concepts/scenarios.mdx @@ -381,5 +381,9 @@ See the [CLI reference](/docs/cli/scenario) for full command documentation. ## Limitations - Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error. +- Local replay compares VARIANT and JSON output columns by their JSON value, not their type: key + order, whitespace and number spelling do not matter, so a Snowflake capture's pretty-printed + JSON equals an expected `PARSE_JSON(...)`. Values of non-JSON types with the same JSON text also + compare equal, for example `DATE '2024-01-01'::VARIANT` and `'2024-01-01'::VARIANT`. - Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason. - Outside local replay, scenario fixture, expected and assertion SQL runs as authored: SQLBuild replaces `__ref`, `__source`, `__seed` and `__dbt_ref` calls with relation names and may wrap a partial fixture to add its missing columns, but never regenerates the query. Calls inside comments and quoted text are left unchanged. Quoted text and comments follow the project adapter's rules, including with `--local`: backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, and `//` line comments on Snowflake. A call naming a relation the scenario does not provide, or SQL with an unclosed string or comment, fails planning with `S511` naming the scenario file. diff --git a/website/src/content/docs/docs/concepts/semantic-compilation.mdx b/website/src/content/docs/docs/concepts/semantic-compilation.mdx index 0b6eefbe1b..1387a963ec 100644 --- a/website/src/content/docs/docs/concepts/semantic-compilation.mdx +++ b/website/src/content/docs/docs/concepts/semantic-compilation.mdx @@ -197,6 +197,7 @@ the notice never reduces them to a count alone. | B101 | E202 or SQLBuild | Unknown function, or a built-in spelling the dialect does not accept | | B102 | E203 or project declaration | Invalid function arity | | B103 | SQLBuild | Truth-value test the dialect does not accept | +| B104 | SQLBuild | Warning: a SQLite `CAST` to a date or time type, which keeps only the leading number | | B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | | B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | | B216 | E216 | Set-operation, subquery, or row arity | diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index c1e4b8cc4e..851049c889 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -111,7 +111,7 @@ Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_i ### Destination -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): +By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. On SQLite, dlt writes through its `sqlalchemy` destination into the project's database file (dataset `main`); the `sqlbuild[dlt]` extra installs the SQLAlchemy it needs. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): ```yaml dlt_sources: diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx index d57f6ecb41..7704837775 100644 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ b/website/src/content/docs/docs/integrations/ingestr.mdx @@ -128,6 +128,7 @@ SQLBuild automatically builds the ingestr destination URI from your adapter conn | Adapter | Destination URI format | |---------|----------------------| | DuckDB | `duckdb:///path/to/db.duckdb` | +| SQLite | `sqlite:///path/to/db.sqlite` | | MotherDuck | `motherduck://database` | | PostgreSQL | `postgresql://user:pass@host:port/db` | | Snowflake | `snowflake://user:pass@account/db/schema` |