diff --git a/Makefile b/Makefile index 124fe03891..9b29cb0610 100644 --- a/Makefile +++ b/Makefile @@ -73,6 +73,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_sql_hook_identity.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_change_tracking.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_query_propagation.py \ diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 6ecc7baca1..2e4535d748 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{ Bound, IntoPyObject, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use crate::configuration::main::load; use crate::constants::API_VERSION; @@ -38,19 +38,37 @@ fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult, + dialect: String, +} + #[pyfunction] fn prepare_lint_sql( py: Python<'_>, - expanded: &str, - before_expansion: &str, - prior_sites: Vec, + request: LintPreparationRequest, ) -> PyResult> { py.detach(|| { - crate::sql_lint::main::preparation::prepare(expanded, before_expansion, &prior_sites) + crate::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) }) .map_err(value_error) } +#[pyfunction] +fn lint_backtick_identifiers(dialect: &str) -> bool { + crate::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect) +} + #[pyfunction] fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { py.detach(|| crate::sql_lint::main::batch_engine::lint_batch_json(request_json)) @@ -325,6 +343,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; + module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs index ebcdd035c2..7e7688c379 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs @@ -202,7 +202,8 @@ fn classify_model(ctes: Vec, file: &str) -> Result { authored.push(cte); } else if let Some(value) = name.strip_prefix("__expected__") { expected_models.push(required(value, "__expected__", file)?); - validate_expected(&cte, file, "__expected__", true)?; + let label = cte.0.clone(); + validate_expected(&cte, file, &label, true)?; expected.push(cte); } else if let Some(value) = name.strip_prefix("__assert__") { assertion_names.push(required(value, "__assert__", file)?); @@ -431,7 +432,7 @@ fn validate_expected( "SQL test '{file}' must not use SELECT * in {label} CTEs" )); } - let branches = split_unions(&cte.1)?; + let branches = split_set_operations(&cte.1)?; let mut names: Vec> = Vec::new(); for branch in branches { names.push(projection_names(branch, file, label)?); @@ -844,10 +845,20 @@ fn contains_select_star(sql: &str) -> Result { Ok(false) } -fn split_unions(sql: &str) -> Result, String> { - let mut values: Vec<&str> = Vec::new(); - let mut start = 0; - let mut index = 0; +/// What a top-level scan does after visiting one code offset. +enum CodeStep { + Advance, + Resume(usize), + Stop(T), +} + +/// Visit each code offset outside quotes and comments with its depth after that byte's parenthesis. +fn scan_code( + sql: &str, + start: usize, + mut visit: impl FnMut(usize, isize) -> Result, String>, +) -> Result, String> { + let mut index = start; let mut depth = 0; while index < sql.len() { let next = skip_non_code(sql, index)?; @@ -860,83 +871,85 @@ fn split_unions(sql: &str) -> Result, String> { Some(b')') => depth -= 1, _ => {} } - if depth == 0 - && let Some(end) = consume_keyword(sql, index, "UNION") - { - let value = sql[start..index].trim(); - if !value.is_empty() { - values.push(value); - } - index = skip_ignorable(sql, end)?; - if let Some(quantifier_end) = consume_keyword(sql, index, "ALL") - .or_else(|| consume_keyword(sql, index, "DISTINCT")) - { - index = skip_ignorable(sql, quantifier_end)?; - } - start = index; - continue; + match visit(index, depth)? { + CodeStep::Advance => index += char_len(sql, index), + CodeStep::Resume(resume) => index = resume, + CodeStep::Stop(value) => return Ok(Some(value)), } - index += char_len(sql, index); } - let value = sql[start..].trim(); - if !value.is_empty() { - values.push(value); - } - Ok(values) + Ok(None) +} + +fn non_empty_trimmed(value: &str) -> Option<&str> { + Some(value.trim()).filter(|trimmed| !trimmed.is_empty()) } -fn split_top_level(sql: &str, separator: u8) -> Result, String> { +/// Split on top-level `UNION`, `INTERSECT` and `EXCEPT`, each with an optional `ALL`/`DISTINCT`. +pub(crate) fn split_set_operations(sql: &str) -> Result, String> { let mut values: Vec<&str> = Vec::new(); let mut start = 0; - let mut index = 0; - let mut depth = 0; - while index < sql.len() { - let next = skip_non_code(sql, index)?; - if next != index { - index = next; - continue; + scan_code::<()>(sql, 0, |index, depth| { + if depth != 0 { + return Ok(CodeStep::Advance); } - match byte_at(sql, index) { - Some(b'(') => depth += 1, - Some(b')') => depth -= 1, - Some(value) if value == separator && depth == 0 => { - let item = sql[start..index].trim(); - if !item.is_empty() { - values.push(item); - } - start = index + 1; - } - _ => {} + let Some(end) = set_operator_end(sql, index)? else { + return Ok(CodeStep::Advance); + }; + values.extend(non_empty_trimmed(&sql[start..index])); + let mut resume = skip_ignorable(sql, end)?; + if let Some(quantifier_end) = + consume_keyword(sql, resume, "ALL").or_else(|| consume_keyword(sql, resume, "DISTINCT")) + { + resume = skip_ignorable(sql, quantifier_end)?; } - index += char_len(sql, index); + start = resume; + Ok(CodeStep::Resume(resume)) + })?; + values.extend(non_empty_trimmed(&sql[start..])); + Ok(values) +} + +/// Return the end of a set operator at `index`; `* EXCEPT (...)` star modifiers are not operators. +fn set_operator_end(sql: &str, index: usize) -> Result, String> { + if let Some(end) = + consume_keyword(sql, index, "UNION").or_else(|| consume_keyword(sql, index, "INTERSECT")) + { + return Ok(Some(end)); } - let item = sql[start..].trim(); - if !item.is_empty() { - values.push(item); + match consume_keyword(sql, index, "EXCEPT") { + Some(end) if previous_code_byte(sql, index)? != Some(b'*') => Ok(Some(end)), + _ => Ok(None), } - Ok(values) } -fn find_top_level_keyword(sql: &str, start: usize, keyword: &str) -> Result, String> { - let mut index = start; - let mut depth = 0; - while index < sql.len() { - let next = skip_non_code(sql, index)?; - if next != index { - index = next; - continue; - } - match byte_at(sql, index) { - Some(b'(') => depth += 1, - Some(b')') => depth -= 1, - _ => {} - } - if depth == 0 && consume_keyword(sql, index, keyword).is_some() { - return Ok(Some(index)); +pub(crate) fn split_top_level(sql: &str, separator: u8) -> Result, String> { + let mut values: Vec<&str> = Vec::new(); + let mut start = 0; + scan_code::<()>(sql, 0, |index, depth| { + if depth == 0 && byte_at(sql, index) == Some(separator) { + values.extend(non_empty_trimmed(&sql[start..index])); + start = index + 1; } - index += char_len(sql, index); - } - Ok(None) + Ok(CodeStep::Advance) + })?; + values.extend(non_empty_trimmed(&sql[start..])); + Ok(values) +} + +pub(crate) fn find_top_level_keyword( + sql: &str, + start: usize, + keyword: &str, +) -> Result, String> { + scan_code(sql, start, |index, depth| { + Ok( + if depth == 0 && consume_keyword(sql, index, keyword).is_some() { + CodeStep::Stop(index) + } else { + CodeStep::Advance + }, + ) + }) } fn find_top_level_clause_keyword( diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs index 58f157ebad..058724762e 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs @@ -1056,15 +1056,7 @@ fn resolve_assertion_textual_sql( ); let body_sql = resolution.sql; let reached = resolution.reached; - let resolved_sql = if lifted_ctes.is_empty() { - body_sql.clone() - } else { - let cte_parts: Vec = lifted_ctes - .iter() - .map(|(name, body)| cte_definition_sql(name, body)) - .collect(); - format!("WITH {} {body_sql}", cte_parts.join(", ")) - }; + let resolved_sql = with_leading_ctes(&lifted_ctes, &body_sql); Ok(( TextualStep { resolved_sql, diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index 422ac14668..d72e8bf7ac 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -212,7 +212,7 @@ pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool { ), ( r#"{"tests":[{"sql":"WITH __source__raw_orders AS (SELECT 1 AS id), __expected__orders AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/orders.sql","mode":"model"}]}"#, - "SQL test 'tests/orders.sql' must alias every non-trivial __expected__ projection", + "SQL test 'tests/orders.sql' must alias every non-trivial __expected__orders projection", ), ]; for (request, expected_error) in cases { @@ -222,6 +222,92 @@ pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool { true } +fn extract_expected( + mode: &str, + actual: &str, + expected_name: &str, + expected_sql: &str, +) -> Result { + let sql = format!("WITH {actual}, {expected_name} AS ({expected_sql}) SELECT 1"); + extract_batch_json( + &json!({"tests": [{"sql": sql, "fileLabel": "tests/orders.sql", "mode": mode}]}) + .to_string(), + ) +} + +pub(crate) fn set_operation_expected_ctes_validate_every_branch() -> bool { + let kinds = [ + ( + "model", + "__source__raw_orders AS (SELECT 1 AS order_id)", + "__expected__orders", + ), + ( + "macro", + "__macro_actual__ AS (SELECT 1 AS order_id)", + "__macro_expected__", + ), + ( + "udf", + "__udf_actual__ AS (SELECT 1 AS order_id)", + "__udf_expected__", + ), + ( + "table_fn", + "__table_fn_actual__ AS (SELECT 1 AS order_id)", + "__table_fn_expected__", + ), + ]; + let accepted = [ + "SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + "SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", + "SELECT 1 AS order_id INTERSECT /* EXCEPT */ -- UNION\n SELECT 1 AS order_id", + ]; + for (mode, actual, expected_name) in kinds { + for expected_sql in accepted { + assert!( + extract_expected(mode, actual, expected_name, expected_sql).is_ok(), + "{mode}: {expected_sql}" + ); + } + let mismatch = format!( + "SQL test 'tests/orders.sql' must use the same {expected_name} projection names and order in every set-operation branch; branch 2 does not match branch 1" + ); + let third_mismatch = mismatch.replace("branch 2", "branch 3"); + let not_select = format!( + "SQL test 'tests/orders.sql' must define each {expected_name} set-operation branch as a SELECT query" + ); + let rejected = [ + ( + "SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", + &mismatch, + ), + ( + "SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + &mismatch, + ), + ( + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + &third_mismatch, + ), + ("SELECT 1 AS order_id EXCEPT VALUES (1)", ¬_select), + ]; + for (expected_sql, expected_error) in rejected { + assert_eq!( + extract_expected(mode, actual, expected_name, expected_sql).as_ref(), + Err(expected_error), + "{mode}: {expected_sql}" + ); + } + } + true +} + pub(crate) fn concurrent_requests_initialize_shared_template_once() -> bool { let templates = Arc::new(Mutex::new(HashMap::new())); let starts = Arc::new(Barrier::new(4)); @@ -657,6 +743,70 @@ pub(crate) fn plan_without_rendering_returns_executable_steps() -> bool { true } +pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_planning::plan_and_render_json( + &json!({ + "models": [ + { + "name": "stg_orders", + "querySql": "SELECT * FROM __source(\"raw_orders\")", + "modelDependencies": [] + }, + { + "name": "orders", + "querySql": "SELECT * FROM __ref(\"stg_orders\")", + "modelDependencies": ["stg_orders"] + } + ], + "tests": [{ + "name": "orders_case", + "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": [{ + "name": "__source__raw_orders", + "sqlBody": "SELECT 1 AS order_id" + }], + "expectedCtes": [], + "expectedModelNames": [], + "assertionCtes": [{ + "name": "__assert__no_negative_orders", + "sqlBody": "WITH negative_orders AS (SELECT * FROM __ref(\"orders\") WHERE order_id < 0) SELECT * FROM negative_orders" + }] + } + }], + "sqlAnalysisEnabled": false, + "sqlAnalysisDialect": "duckdb", + "renderSql": false + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + + let resolved_sql = response["artifacts"][0]["assertions"][0]["resolvedSql"] + .as_str() + .expect("resolved assertion SQL"); + assert_eq!( + resolved_sql, + concat!( + "WITH __ref__stg_orders AS (SELECT * FROM (SELECT 1 AS order_id)), ", + "__ref__orders AS (SELECT * FROM __ref__stg_orders), ", + "negative_orders AS (SELECT * FROM __ref__orders WHERE order_id < 0) ", + "SELECT * FROM negative_orders" + ) + ); + assert!( + polyglot_sql::Dialect::get(polyglot_sql::DialectType::DuckDB) + .parse(resolved_sql) + .is_ok(), + "{resolved_sql}" + ); + true +} + pub(crate) fn upstream_fallback_resolves() -> bool { let request = json!({ "models": [ diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs index 671caec8be..e5b777fb29 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs @@ -1,11 +1,15 @@ +use crate::compiler::_helpers::sql_tests::extraction::{ + find_top_level_keyword, split_set_operations, split_top_level, +}; use crate::compiler::tests::helpers::{ dependent_assertion_returns_authoritative_error, empty_model_fixture_marker_preserves_direct_mode_validation, expected_projection_errors_name_the_expected_cte, mixed_expanded_tests_preserve_order_and_payloads, quoted_ctes_and_implicit_alias_preserve_payload, + set_operation_expected_ctes_validate_every_branch, }; -use crate::compiler::tests::test_types::SqlTestExtractionTestCase; +use crate::compiler::tests::test_types::{SqlTestExtractionTestCase, TopLevelScanTestCase}; #[test] fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_holds() { @@ -35,6 +39,11 @@ fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_h run: expected_projection_errors_name_the_expected_cte, expected_success: true, }, + SqlTestExtractionTestCase { + description: "set-operation expected CTEs validate every branch", + run: set_operation_expected_ctes_validate_every_branch, + expected_success: true, + }, ]; for test_case in test_cases { @@ -46,3 +55,130 @@ fn given_sql_test_cases_when_extracting_native_payloads_then_expected_behavior_h ); } } + +#[test] +fn given_expected_cte_sql_when_scanning_top_level_then_set_operations_commas_and_keywords_are_pinned() + { + let unclosed_quote = "SQL test contains an unclosed quoted string".to_owned(); + let unclosed_comment = "SQL test contains an unclosed block comment".to_owned(); + let test_cases = [ + TopLevelScanTestCase { + description: "plain select", + sql: "SELECT a, b AS c FROM orders", + expected_unions: Ok(vec!["SELECT a, b AS c FROM orders"]), + expected_commas: Ok(vec!["SELECT a", "b AS c FROM orders"]), + expected_from: Ok(Some(17)), + }, + TopLevelScanTestCase { + description: "union quantifiers and comments between keywords", + sql: "SELECT 1 AS a UNION ALL SELECT 2 AS a UNION /* x */ DISTINCT -- y\n SELECT 3 AS a UNION SELECT 4 AS a", + expected_unions: Ok(vec![ + "SELECT 1 AS a", + "SELECT 2 AS a", + "SELECT 3 AS a", + "SELECT 4 AS a", + ]), + expected_commas: Ok(vec![ + "SELECT 1 AS a UNION ALL SELECT 2 AS a UNION /* x */ DISTINCT -- y\n SELECT 3 AS a UNION SELECT 4 AS a", + ]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "quotes comments and parentheses hide separators", + sql: "SELECT 'x, UNION (' AS a, /* UNION , FROM */ \"b)\", `c,d`, f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t", + expected_unions: Ok(vec![ + "SELECT 'x, UNION (' AS a, /* UNION , FROM */ \"b)\", `c,d`, f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t", + ]), + expected_commas: Ok(vec![ + "SELECT 'x, UNION (' AS a", + "/* UNION , FROM */ \"b)\"", + "`c,d`", + "f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t", + ]), + expected_from: Ok(Some(107)), + }, + TopLevelScanTestCase { + description: "keyword boundaries and empty pieces", + sql: ",SELECT reunion, union_id,, x FROMAGE UNION", + expected_unions: Ok(vec![",SELECT reunion, union_id,, x FROMAGE"]), + expected_commas: Ok(vec!["SELECT reunion", "union_id", "x FROMAGE UNION"]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "unbalanced close paren goes negative", + sql: "SELECT a) UNION SELECT b, c FROM t", + expected_unions: Ok(vec!["SELECT a) UNION SELECT b, c FROM t"]), + expected_commas: Ok(vec!["SELECT a) UNION SELECT b, c FROM t"]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "multibyte text", + sql: "SELECT 'é' AS ñ, ü FROM t UNION SELECT 1, 2", + expected_unions: Ok(vec!["SELECT 'é' AS ñ, ü FROM t", "SELECT 1, 2"]), + expected_commas: Ok(vec!["SELECT 'é' AS ñ", "ü FROM t UNION SELECT 1", "2"]), + expected_from: Ok(Some(22)), + }, + TopLevelScanTestCase { + description: "intersect and except quantifiers mixed with union", + sql: "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a UNION SELECT 4 AS a EXCEPT SELECT 5 AS a", + expected_unions: Ok(vec![ + "SELECT 1 AS a", + "SELECT 2 AS a", + "SELECT 3 AS a", + "SELECT 4 AS a", + "SELECT 5 AS a", + ]), + expected_commas: Ok(vec![ + "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a UNION SELECT 4 AS a EXCEPT SELECT 5 AS a", + ]), + expected_from: Ok(None), + }, + TopLevelScanTestCase { + description: "star except modifier and identifier boundaries are not set operations", + sql: "SELECT t.* EXCEPT (status), exceptional, intersection FROM t", + expected_unions: Ok(vec![ + "SELECT t.* EXCEPT (status), exceptional, intersection FROM t", + ]), + expected_commas: Ok(vec![ + "SELECT t.* EXCEPT (status)", + "exceptional", + "intersection FROM t", + ]), + expected_from: Ok(Some(54)), + }, + TopLevelScanTestCase { + description: "unclosed quote", + sql: "SELECT 'a, b FROM t", + expected_unions: Err(unclosed_quote.clone()), + expected_commas: Err(unclosed_quote.clone()), + expected_from: Err(unclosed_quote), + }, + TopLevelScanTestCase { + description: "unclosed block comment after union", + sql: "SELECT 1 UNION /* never closed", + expected_unions: Err(unclosed_comment.clone()), + expected_commas: Err(unclosed_comment.clone()), + expected_from: Err(unclosed_comment), + }, + ]; + for test_case in test_cases { + assert_eq!( + split_set_operations(test_case.sql), + test_case.expected_unions, + "unions: {}", + test_case.description + ); + assert_eq!( + split_top_level(test_case.sql, b','), + test_case.expected_commas, + "commas: {}", + test_case.description + ); + assert_eq!( + find_top_level_keyword(test_case.sql, 0, "FROM"), + test_case.expected_from, + "FROM: {}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs index b343728391..eb6f3a9359 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs @@ -5,7 +5,7 @@ use crate::compiler::tests::helpers::{ mock_read_through_helper_brings_its_mock_dependencies_into_scope, model_test_batch_returns_ordered_artifact, plan_without_rendering_returns_executable_steps, scoped_helper_named_like_generated_cte_is_rejected, - shared_textual_chain_renders_each_model_once, + shared_textual_chain_renders_each_model_once, textual_assertion_with_clause_merges_lifted_ctes, unicode_cte_after_leading_with_preserves_identifier, unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves, }; @@ -91,6 +91,11 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be run: scoped_helper_named_like_generated_cte_is_rejected, expected_success: true, }, + SqlTestPlanningTestCase { + description: "textual assertion WITH clause merges lifted CTEs", + run: textual_assertion_with_clause_merges_lifted_ctes, + expected_success: true, + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs index ee2a9052a0..895efdcd05 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs @@ -34,3 +34,11 @@ pub(crate) struct ExpectedColumnsTestCase { pub(crate) sql: &'static str, pub(crate) expected_columns: Option<&'static [&'static str]>, } + +pub(crate) struct TopLevelScanTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_unions: Result, String>, + pub(crate) expected_commas: Result, String>, + pub(crate) expected_from: Result, String>, +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs index 03d28d174c..7b904bff6f 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs @@ -479,6 +479,48 @@ fn given_dialect_quoting_around_table_functions_when_evaluating_rules_then_model Ok(()) } +#[test] +fn given_comment_apostrophes_when_generic_fallback_normalizes_rules_sql_then_model_parses() +-> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let test_cases = [ + test_types::DialectEvaluationTestCase { + description: "line comment apostrophe before a colon path", + dialect: "postgres", + query_sql: "-- don't drop the customer path\nSELECT payload:customer_id AS customer_id FROM orders", + expected_code: "SQBRCONTRACT101", + }, + test_types::DialectEvaluationTestCase { + description: "block comment apostrophe before a colon path", + dialect: "postgres", + query_sql: "/* customer's path */ SELECT payload:customer_id AS customer_id FROM orders", + expected_code: "SQBRCONTRACT101", + }, + test_types::DialectEvaluationTestCase { + description: "colon inside a line comment stays commented", + dialect: "postgres", + query_sql: "SELECT payload:customer_id AS customer_id -- note: it's nested\nFROM orders", + expected_code: "SQBRCONTRACT101", + }, + ]; + + for test_case in test_cases { + let config = json!({"select": ["SQBRCONTRACT101"], "cache": {"enabled": false}}); + let mut request: Value = serde_json::from_str(&helpers::request(&project_dir, &config)) + .map_err(|error| error.to_string())?; + request["dialect"] = json!(test_case.dialect); + request["models"][0]["query_sql"] = json!(test_case.query_sql); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + assert_eq!( + result["faults"][0]["code"], test_case.expected_code, + "{}: {result}", + test_case.description + ); + } + Ok(()) +} + #[test] fn given_enforced_contract_outputs_when_evaluating_explicit_type_rule_then_returns_expected_faults() -> Result<(), String> { diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs index baea4d051b..f2e5c1b7b4 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_sql_scanning.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers::sql_references::extraction::extract; -use crate::engine::tests::test_types::SqlScannerTestCase; +use crate::engine::tests::test_types::{LintDialectSiteTestCase, SqlScannerTestCase}; use crate::rules::_helpers::evaluation::{normalize_rules_sql, rules_quote_policy}; use crate::rules::_helpers::numeric_decisions::compact_sql; use crate::sql_lint::_helpers::preparation::prepare; @@ -282,7 +282,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann test_case.description ); let lint_sql = format!("SELECT @m({fragment} x) AS y"); - let lint_site = prepare(&lint_sql, &lint_sql, &[])?.map(|prepared| { + let lint_site = prepare(&lint_sql, &lint_sql, &[], "postgres")?.map(|prepared| { prepared .1 .first() @@ -304,3 +304,58 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann } Ok(()) } + +#[test] +fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quoting() +-> Result<(), String> { + let test_cases = [ + LintDialectSiteTestCase { + description: "databricks backtick containing close paren", + dialect: "databricks", + fragment: "`a)b`", + expected_lint_site: "@m(`a)b` x)", + }, + LintDialectSiteTestCase { + description: "bigquery doubled backtick", + dialect: "bigquery", + fragment: "`a``)b`", + expected_lint_site: "@m(`a``)b` x)", + }, + LintDialectSiteTestCase { + description: "databricks single quote inside backticks", + dialect: "databricks", + fragment: "`it's)`", + expected_lint_site: "@m(`it's)` x)", + }, + LintDialectSiteTestCase { + description: "databricks backslash in single quote still escapes", + dialect: "databricks", + fragment: "'a\\')'", + expected_lint_site: "@m('a\\')' x)", + }, + LintDialectSiteTestCase { + description: "postgres backtick remains code", + dialect: "postgres", + fragment: "`a)b`", + expected_lint_site: "@m(`a)", + }, + LintDialectSiteTestCase { + description: "sqlserver backtick remains code", + dialect: "tsql", + fragment: "`a)b`", + expected_lint_site: "@m(`a)", + }, + ]; + for test_case in test_cases { + let lint_sql = format!("SELECT @m({} x) AS y", test_case.fragment); + let lint_site = prepare(&lint_sql, &lint_sql, &[], test_case.dialect)? + .and_then(|prepared| prepared.1.first().map(|site| site.5.clone())) + .unwrap_or_default(); + assert_eq!( + lint_site, test_case.expected_lint_site, + "SQL lint macro site: {}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs index 9c319a4f50..ab954a9371 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs @@ -169,6 +169,13 @@ pub(crate) struct EmptyInputMinimumTestsTestCase { } /// One SQL fragment scanned by every native quote- and comment-aware scanner. +pub(crate) struct LintDialectSiteTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) fragment: &'static str, + pub(crate) expected_lint_site: &'static str, +} + pub(crate) struct SqlScannerTestCase { pub(crate) description: &'static str, pub(crate) fragment: &'static str, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs index 6732ebc948..01407def13 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs @@ -13,7 +13,6 @@ use crate::rules::models::{ }; use crate::sql_scan::main::matching_paren::matching_paren; use crate::sql_scan::main::non_code_end::non_code_end; -use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::QuotePolicy; use globset::{Glob, GlobSetBuilder}; use sqlparser::ast::{ @@ -273,12 +272,13 @@ fn normalize_generic_fallback(sql: &str) -> String { while index < bytes.len() { let byte = bytes[index]; let next = bytes.get(index + 1).copied(); - if policy.is_quote(byte) { - let Ok(end) = quote_end(&bytes, index, policy) else { - break; - }; - index = end; - continue; + match non_code_end(&bytes, index, policy) { + Ok(Some(end)) => { + index = end; + continue; + } + Ok(None) => {} + Err(_) => break, } if byte == b'-' && next == Some(b'>') { bytes[index] = b','; diff --git a/crates/sqlbuild-rules-native/src/rules/main/mod.rs b/crates/sqlbuild-rules-native/src/rules/main/mod.rs index 3ecc50b539..605292c0ac 100644 --- a/crates/sqlbuild-rules-native/src/rules/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/rules/main/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod catalogue; pub(crate) mod evaluate; pub(crate) mod evaluate_project; pub(crate) mod fingerprint; +pub(crate) mod quote_policy; pub(crate) mod resolve_threshold_overrides; pub(crate) mod select; pub(crate) mod selected_codes; diff --git a/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs b/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs new file mode 100644 index 0000000000..a33a47e140 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/rules/main/quote_policy.rs @@ -0,0 +1,4 @@ +/// Return the quoting policy rules use when scanning SQL for `dialect_name`. +pub(crate) fn quote_policy(dialect_name: &str) -> crate::sql_scan::models::QuotePolicy { + crate::rules::_helpers::evaluation::rules_quote_policy(dialect_name) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs index 872d157226..e0d6f65181 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/preparation.rs @@ -5,14 +5,22 @@ use std::sync::LazyLock; use regex::Regex; +use crate::rules::main::quote_policy::quote_policy; use crate::sql_lint::types::{InterpolationSite, PreparedSql}; use crate::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; use crate::sql_scan::models::QuotePolicy; -static SITES: LazyLock> = LazyLock::new(|| { - Regex::new( - r#"--[^\n]*(?:\n|\z)|/\*[\s\S]*?(?:\*/|\z)|'(?:\\.|''|[^'\\])*(?:'|\z)|"(?:\\.|""|[^"\\])*(?:"|\z)|(?P@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()"#, - ) +const SITE_PATTERN: &str = + r#"(?P@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()"#; +const NON_CODE_PATTERN: &str = r#"--[^\n]*(?:\n|\z)|/\*[\s\S]*?(?:\*/|\z)|'(?:\\.|''|[^'\\])*(?:'|\z)|"(?:\\.|""|[^"\\])*(?:"|\z)"#; +const BACKTICK_PATTERN: &str = r#"`(?:``|[^`])*(?:`|\z)"#; + +static SITES: LazyLock> = + LazyLock::new(|| Regex::new(&format!("{NON_CODE_PATTERN}|{SITE_PATTERN}"))); +static BACKTICK_SITES: LazyLock> = LazyLock::new(|| { + Regex::new(&format!( + "{NON_CODE_PATTERN}|{BACKTICK_PATTERN}|{SITE_PATTERN}" + )) }); static CTES: LazyLock> = LazyLock::new(|| { Regex::new( @@ -24,15 +32,27 @@ static WORDS: LazyLock> = static OPAQUE_CTE: LazyLock> = LazyLock::new(|| Regex::new(r"(?i)\b[A-Za-z_][A-Za-z0-9_]*\s+AS\s*\(\s*(?:--[^\n]*\n\s*)?$")); +/// Return whether SQL lint treats backticks as identifier quotes for `dialect`, as rules do. +pub(crate) fn backtick_identifiers(dialect: &str) -> bool { + quote_policy(dialect).backtick_identifiers +} + pub(crate) fn prepare( expanded: &str, before_expansion: &str, prior_sites: &[usize], + dialect: &str, ) -> Result, String> { if !expanded.is_ascii() || !before_expansion.is_ascii() { return Ok(None); } - let site_pattern = SITES.as_ref().map_err(|error| error.to_string())?; + let policy = QuotePolicy::SQL_LINT.with_backtick_identifiers(backtick_identifiers(dialect)); + let sites_regex = if policy.backtick_identifiers { + &BACKTICK_SITES + } else { + &SITES + }; + let site_pattern = sites_regex.as_ref().map_err(|error| error.to_string())?; let cte_pattern = CTES.as_ref().map_err(|error| error.to_string())?; let word_pattern = WORDS.as_ref().map_err(|error| error.to_string())?; let opaque_pattern = OPAQUE_CTE.as_ref().map_err(|error| error.to_string())?; @@ -47,7 +67,7 @@ pub(crate) fn prepare( if start < copied_to { continue; } - let Some(end) = site_end(expanded, start) else { + let Some(end) = site_end(expanded, start, policy) else { continue; }; text.push_str(&expanded[copied_to..start]); @@ -96,7 +116,7 @@ pub(crate) fn prepare( Ok(Some((text, sites, referenced))) } -fn site_end(text: &str, start: usize) -> Option { +fn site_end(text: &str, start: usize, policy: QuotePolicy) -> Option { let bytes = text.as_bytes(); match bytes[start] { b'@' if bytes.get(start + 1) == Some(&b'@') => Some(scan_name(bytes, start + 2, true)), @@ -110,7 +130,7 @@ fn site_end(text: &str, start: usize) -> Option { } let end = scan_name(bytes, start + 1, false); if bytes.get(end) == Some(&b'(') { - matching_paren(bytes, end) + matching_paren(bytes, end, policy) } else { Some(end) } @@ -119,7 +139,7 @@ fn site_end(text: &str, start: usize) -> Option { b'_' => { let end = scan_name(bytes, start, false); (bytes.get(end) == Some(&b'(')) - .then(|| matching_paren(bytes, end)) + .then(|| matching_paren(bytes, end, policy)) .flatten() } _ => None, @@ -137,8 +157,8 @@ fn scan_name(bytes: &[u8], mut index: usize, interpolation: bool) -> usize { index } -fn matching_paren(bytes: &[u8], open: usize) -> Option { - match scan_matching_paren(bytes, open, QuotePolicy::SQL_LINT) { +fn matching_paren(bytes: &[u8], open: usize, policy: QuotePolicy) -> Option { + match scan_matching_paren(bytes, open, policy) { Ok(close) => Some(close + 1), Err(_) => None, } diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs new file mode 100644 index 0000000000..f0cc7a2a75 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_lint/main/backtick_identifiers.rs @@ -0,0 +1,4 @@ +/// Return whether SQL lint treats backticks as identifier quotes for `dialect`. +pub(crate) fn backtick_identifiers(dialect: &str) -> bool { + crate::sql_lint::_helpers::preparation::backtick_identifiers(dialect) +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs index 506a01d172..f5c37dd789 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod backtick_identifiers; pub(crate) mod batch_engine; pub(crate) mod batch_formatter; pub(crate) mod catalogue; diff --git a/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs b/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs index 6a3aa9e3ec..a1b2710ffd 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/main/preparation.rs @@ -6,6 +6,12 @@ pub(crate) fn prepare( expanded: &str, before_expansion: &str, prior_sites: &[usize], + dialect: &str, ) -> Result, String> { - crate::sql_lint::_helpers::preparation::prepare(expanded, before_expansion, prior_sites) + crate::sql_lint::_helpers::preparation::prepare( + expanded, + before_expansion, + prior_sites, + dialect, + ) } diff --git a/crates/sqlbuild-rules-native/src/sql_scan/models.rs b/crates/sqlbuild-rules-native/src/sql_scan/models.rs index 74e0f5a908..4f508b195c 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/models.rs @@ -27,13 +27,21 @@ impl QuotePolicy { double_quote_backslash_escapes: false, }; - /// SQL lint policy, mirroring its site lexer: backslashes escape, backticks are code. + /// SQL lint base policy: backslashes escape; lint enables backticks per dialect. pub(crate) const SQL_LINT: Self = Self { backtick_identifiers: false, single_quote_backslash_escapes: true, double_quote_backslash_escapes: true, }; + /// Return this policy with backtick-delimited identifiers enabled or disabled. + pub(crate) const fn with_backtick_identifiers(self, backtick_identifiers: bool) -> Self { + Self { + backtick_identifiers, + ..self + } + } + /// Return whether `byte` opens quoted text under this policy. pub(crate) fn is_quote(self, byte: u8) -> bool { match byte { diff --git a/docs/execution-history.md b/docs/execution-history.md index f382e05495..37258df52d 100644 --- a/docs/execution-history.md +++ b/docs/execution-history.md @@ -13,7 +13,7 @@ contract. The public types, filters, pages, records, errors, and helper function The concrete SQLite and PostgreSQL classes additionally provide transactional `append_and_project`, `reconcile`, and `check_health` operations. -`EventLogStorage` remains a deprecated compatibility alias for `LifecycleEventLogStorage`. Storage +The deprecated `EventLogStorage` alias has been removed; use `LifecycleEventLogStorage`. Storage tables use the lifecycle-specific names `lifecycle_event_log` in SQLite and `sqlbuild_lifecycle_event_log` in PostgreSQL; the former `event_log` and `sqlbuild_event_log` names are retired. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 264de8d6fb..f4a8b177bd 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,5 +1,13 @@ """Private SQLBuild native engine bindings.""" +from typing import TypedDict + +class LintPreparationRequest(TypedDict): + expanded: str + before_expansion: str + prior_sites: list[int] + dialect: str + def catalogue_json() -> str: ... def evaluate_json(request_json: str) -> str: ... def finalize_rule_findings_json(request_json: str) -> str: ... @@ -8,8 +16,9 @@ def format_sql_batch_json(request_json: str) -> str: ... def lint_sql_json(request_json: str) -> str: ... def lint_sql_batch_json(request_json: str) -> str: ... def prepare_lint_sql( - expanded: str, before_expansion: str, prior_sites: list[int] + request: LintPreparationRequest, ) -> tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None: ... +def lint_backtick_identifiers(dialect: str) -> bool: ... def load_config_json(project_dir: str) -> str: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index afca25076e..d63e1249c6 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -303,24 +303,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: return () return tuple(str(column[0]) for column in description) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -727,23 +709,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return False - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -759,45 +724,9 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) - def _render_query_with_cursor_bounds_impl( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - quoted_cursor: str = self.render_identifier(cursor_column) - start_literal: str = self.render_cursor_bound_literal( - value=cursor_start, cursor_type=cursor_type - ) - end_literal: str = self.render_cursor_bound_literal( - value=cursor_end, cursor_type=cursor_type - ) - return ( - f"SELECT * FROM ({sql}) AS __sqlbuild_cursor_bounded " - f"WHERE {quoted_cursor} >= {start_literal} AND {quoted_cursor} < {end_literal}" - ) - def _render_seed_select_before_cursor_impl( self, *, @@ -812,20 +741,6 @@ def _render_seed_select_before_cursor_impl( ) return f"SELECT * FROM {origin} WHERE {quoted_cursor} < {end_literal}" - def _render_seed_select_after_cursor_impl( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - quoted_cursor: str = self.render_identifier(cursor_column) - start_literal: str = self.render_cursor_bound_literal( - value=cursor_start_exclusive, cursor_type=cursor_type - ) - return f"SELECT * FROM {origin} WHERE {quoted_cursor} > {start_literal}" - def _relation_names_match_impl(self, *, left: str, right: str) -> bool: return left.replace('"', "") == right.replace('"', "") @@ -1470,24 +1385,6 @@ def diff_rows( ) -> RowDiffResult: raise AdapterUserError(message="diff_rows requires an engine-specific implementation") - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - where_clause: str = "" - if cursor_column and start_cursor: - where_clause = f" WHERE {cursor_column} >= '{start_cursor.value}'" - if end_cursor: - where_clause += f" AND {cursor_column} < '{end_cursor.value}'" - cursor: Any = connection.execute(f"SELECT COUNT(*) FROM {relation}{where_clause}") - result: Any = cursor.fetchone() - return int(result[0]) - def _inspect_row_diff_coverage( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/diff.py b/src/sqlbuild/adapter/contract/classes/diff.py index 852a3d438c..ed0fa770cc 100644 --- a/src/sqlbuild/adapter/contract/classes/diff.py +++ b/src/sqlbuild/adapter/contract/classes/diff.py @@ -62,19 +62,6 @@ def diff_unkeyed_rows( """Compare exact full-row multiplicities without a unique key.""" ... - @abstractmethod - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - """Return the row count for a relation, optionally bounded by cursor.""" - ... - @abstractmethod def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 35690b7107..95623ea398 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -937,21 +937,6 @@ def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> return f"TIMESTAMP '{value}'" return f"'{value}'" - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def connect(self, config: dict[str, Any]) -> Any: """Open a DuckDB connection from the resolved connection config.""" @@ -1020,24 +1005,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: cursor: Any = self.execute(connection=connection, sql=f"DESCRIBE {sql}") return tuple(str(row[0]) for row in cursor.fetchall()) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -1561,23 +1528,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return True - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -2078,26 +2028,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: Any = self.execute(connection=connection, sql=query).fetchone() - return int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index f7524f205f..a58341aa7f 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -159,17 +159,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: """Return column names produced by a SQL query.""" ... - @abstractmethod - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - ... - @abstractmethod def render_max_cursor_at_or_before( self, @@ -366,19 +355,6 @@ def supports_transactional_ddl(self) -> bool: """Return whether renames and state inserts can commit or roll back together.""" ... - @abstractmethod - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - """Render a query wrapped with adapter-correct cursor bounds.""" - ... - @abstractmethod def render_seed_select_before_cursor( self, @@ -391,18 +367,6 @@ def render_seed_select_before_cursor( """Render a seed-select query that keeps rows before a cursor bound.""" ... - @abstractmethod - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - """Render a seed-select query that keeps rows after a cursor bound.""" - ... - @abstractmethod def relation_names_match(self, *, left: str, right: str) -> bool: """Return whether two relation name strings identify the same adapter relation.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 68e116ffea..2025669fc2 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -33,7 +33,6 @@ class CursorKind(StrEnum): class PromotionStrategy(StrEnum): ATOMIC_SWAP = "atomic_swap" ATOMIC_REPLACE = "atomic_replace" - CREATE_NEW = "create_new" class MigrationTransfer(StrEnum): diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 73b6943b25..7965c8c81c 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1754,23 +1754,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return False - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -1786,21 +1769,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -2078,24 +2046,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: return () return tuple(str(column[0]) for column in description) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -2506,28 +2456,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: tuple[Any, ...] | None = self.execute(connection=connection, sql=query).fetchone() - if result is None: - raise AdapterUserError(message="BigQuery count query returned no result") - return self._to_int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 245a4f7a29..592bbccdfe 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -686,33 +686,6 @@ def _get_delta_history_freshness_metadata( ) return results - def _get_table_type( - self, - *, - connection: Any, - database: str, - schema: str, - name: str, - ) -> str: - query: str = ( - f"SELECT table_type FROM {self._information_schema(database)}.tables " - f"WHERE table_schema = {self._string_literal(schema)} " - f"AND table_name = {self._string_literal(name)}" - ) - cursor: Any = connection.cursor() - try: - cursor.execute(query) - row: tuple[Any, ...] | None = cursor.fetchone() - finally: - cursor.close() - if row is None: - raise AdapterUserError( - message=( - f"Databricks table freshness metadata not found for {database}.{schema}.{name}" - ) - ) - return str(row[0]) - def maximum_identifier_length(self) -> int: """Return the maximum unqualified identifier length supported by the adapter.""" @@ -1788,23 +1761,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return False - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -1820,21 +1776,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -2234,26 +2175,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: finally: cursor.close() - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = connection.cursor() - try: - cursor.execute(f"SELECT max({quoted_cursor}) FROM {relation}") - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - finally: - cursor.close() - def render_max_cursor_at_or_before( self, *, @@ -2508,28 +2429,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: tuple[Any, ...] | None = self.execute(connection=connection, sql=query).fetchone() - if result is None: - raise AdapterUserError(message="Databricks count query returned no result") - return self._to_int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index e4760b67bc..ed09d3e30a 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -193,24 +193,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: return () return tuple(str(column[0]) for column in description) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -631,23 +613,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return True - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -663,21 +628,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -1048,24 +998,6 @@ def delete_insert_cursor( for stmt in statements: self.execute(connection=connection, sql=stmt) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - where_clause: str = "" - if cursor_column and start_cursor: - where_clause = f" WHERE {cursor_column} >= '{start_cursor.value}'" - if end_cursor: - where_clause += f" AND {cursor_column} < '{end_cursor.value}'" - cursor: Any = connection.execute(f"SELECT COUNT(*) FROM {relation}{where_clause}") - result: Any = cursor.fetchone() - return int(result[0]) - def validate_row_diff_keys( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index b62a05ef61..4857c838db 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -2277,23 +2277,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return False - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -2309,21 +2292,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) @@ -2451,26 +2419,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: finally: cursor.close() - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = connection.cursor() - try: - cursor.execute(f"SELECT max({quoted_cursor}) FROM {relation}") - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - finally: - cursor.close() - def render_max_cursor_at_or_before( self, *, @@ -2723,26 +2671,6 @@ def diff_rows( sampling=sampling, ) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: Any | None = None, - end_cursor: Any | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - query: str = f"SELECT COUNT(*) FROM {relation}" - if cursor_filter: - query += f" WHERE {cursor_filter}" - result: Any = self.execute(connection=connection, sql=query).fetchone() - return int(result[0]) - def sample_unequal_rows( self, *, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 36d8af3132..f39656e348 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -231,24 +231,6 @@ def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: ) return tuple(str(row[2]) for row in cursor.fetchall() if row[2] is not None) - def get_relation_max_cursor( - self, - *, - connection: Any, - relation: str, - cursor_column: str, - ) -> object | None: - """Return the maximum cursor value currently present in a relation.""" - - quoted_cursor: str = self.render_identifier(cursor_column) - cursor: Any = self.execute( - connection=connection, sql=f"SELECT max({quoted_cursor}) FROM {relation}" - ) - row: Any | None = cursor.fetchone() - if row is None: - return None - return row[0] - def render_max_cursor_at_or_before( self, *, @@ -1111,25 +1093,6 @@ def durable_clone( for stmt in statements: self.execute(connection=connection, sql=stmt) - def count_rows( - self, - *, - connection: Any, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - cursor_filter: str = self.build_cursor_filter( - cursor_column=cursor_column, - start_cursor=start_cursor, - end_cursor=end_cursor, - ) - where_clause: str = f" WHERE {cursor_filter}" if cursor_filter else "" - cursor: Any = connection.execute(f"SELECT COUNT(*) FROM {relation}{where_clause}") - result: Any = cursor.fetchone() - return int(result[0]) - def create_function( self, *, @@ -1816,23 +1779,6 @@ def capture_dependent_view_rebinds( def supports_transactional_ddl(self) -> bool: return True - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str, - cursor_type: str | None, - ) -> str: - return self._render_query_with_cursor_bounds_impl( - sql=sql, - cursor_column=cursor_column, - cursor_start=cursor_start, - cursor_end=cursor_end, - cursor_type=cursor_type, - ) - def render_seed_select_before_cursor( self, *, @@ -1848,21 +1794,6 @@ def render_seed_select_before_cursor( cursor_type=cursor_type, ) - def render_seed_select_after_cursor( - self, - *, - origin: str, - cursor_column: str, - cursor_start_exclusive: str, - cursor_type: str | None, - ) -> str: - return self._render_seed_select_after_cursor_impl( - origin=origin, - cursor_column=cursor_column, - cursor_start_exclusive=cursor_start_exclusive, - cursor_type=cursor_type, - ) - def relation_names_match(self, *, left: str, right: str) -> bool: return self._relation_names_match_impl(left=left, right=right) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py index 5fb090c963..0f7a8f1f63 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py @@ -58,12 +58,6 @@ def resolve_env_default_concurrency(explicit_concurrency: int | None) -> int | N return concurrency -def read_selector_files(paths: list[str]) -> tuple[str, ...]: - """Read newline-delimited selectors.""" - - return read_selector_file_inputs(paths).selectors - - def read_selector_file_inputs(paths: list[str]) -> SelectorInputs: """Read newline-delimited selectors while retaining file provenance.""" diff --git a/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py b/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py index a86722a0b4..51d1f3fe00 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py @@ -8,7 +8,7 @@ from rich.cells import cell_len from sqlbuild.lint.constants import TAB_CHARACTER, VIOLATION_SEVERITY_FAULT -from sqlbuild.lint.models import FixRunResult, LintRunResult, LintViolation +from sqlbuild.lint.models import LintRunResult, LintViolation from sqlbuild.presentation.classes.cli_style import CliStyle _REPORT_LINE_WIDTH: int = 100 @@ -17,7 +17,7 @@ def format_lint_diagnostics( - *, result: LintRunResult | FixRunResult, root: Path, use_color: bool + *, result: LintRunResult, root: Path, use_color: bool ) -> tuple[str, ...]: """Format every lint violation as a source-pointing diagnostic block.""" @@ -31,7 +31,7 @@ def format_lint_diagnostics( def _format_violation( *, violation: LintViolation, - result: LintRunResult | FixRunResult, + result: LintRunResult, root: Path, style: CliStyle, ) -> str: diff --git a/src/sqlbuild/cli/commands/_helpers/lint/runs.py b/src/sqlbuild/cli/commands/_helpers/lint/runs.py index 0e04b72938..0a7b14259e 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/runs.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/runs.py @@ -13,19 +13,14 @@ ADAPTER_DIALECT_TRANSLATIONS, DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES, - FIX_STATUS_APPLIED, - FIX_STATUS_SKIPPED, FORMAT_SECTION_KEY, LINE_WIDTH_KEY, MAX_DESCRIPTION_LINES_KEY, PROJECT_CONFIG_FILENAME_KEY, ) from sqlbuild.lint.models import ( - FixRunResult, LintConfig, - LintFixRecord, LintRunResult, - LintViolation, ) from sqlbuild.presentation.classes.cli_style import CliStyle @@ -133,90 +128,6 @@ def render_lint_result_json(*, result: LintRunResult) -> None: ) -def render_fix_result(*, result: FixRunResult, root: Path, use_color: bool, preview: bool) -> None: - """Render applied/skipped repairs followed by remaining diagnostics.""" - - style: CliStyle = CliStyle(use_color=use_color) - for record in result.fixes: - label: str = f"{record.status}[{record.code}]" - styled: str = ( - style.success_strong(label) - if record.status == FIX_STATUS_APPLIED - else style.warning_strong(label) - ) - location: str = _relative_path(path=record.file_path, root=root) - suffix: str = f": {record.reason}" if record.reason is not None else "" - print(f"{styled}: {location}:{record.line}:{record.column}{suffix}") - if result.fixes: - print() - if result.violations: - print("\n\n".join(format_lint_diagnostics(result=result, root=root, use_color=use_color))) - print() - fixed_count: int = sum(record.status == FIX_STATUS_APPLIED for record in result.fixes) - skipped_count: int = sum(record.status == FIX_STATUS_SKIPPED for record in result.fixes) - action: str = "WOULD_FIX" if preview else "FIXED" - summary: str = ( - f"Completed. {action}={fixed_count} SKIPPED={skipped_count} " - f"REMAINING={len(result.violations)} FILES={result.files_checked}" - ) - if result.violations or (preview and result.changed_files): - print(style.warning_strong(summary)) - else: - print(style.success_strong(summary)) - - -def render_fix_result_json(*, result: FixRunResult, preview: bool) -> None: - """Render a stable machine-readable fix result.""" - - print( - json.dumps( - { - "changed_files": [str(path) for path in result.changed_files], - "files_checked": result.files_checked, - "fixes": [_fix_record_json(record=record) for record in result.fixes], - "mode": "check" if preview else "write", - "remaining": [_violation_json(violation=entry) for entry in result.violations], - }, - sort_keys=True, - separators=(",", ":"), - ) - ) - - -def _fix_record_json(*, record: LintFixRecord) -> dict[str, object]: - return { - "code": record.code, - "column": record.column, - "file": str(record.file_path), - "line": record.line, - "reason": record.reason, - "status": record.status, - } - - -def _violation_json(*, violation: LintViolation) -> dict[str, object]: - return { - "code": violation.code, - "column": violation.column, - "end_column": violation.end_column, - "end_line": violation.end_line, - "engine": violation.engine, - "file": str(violation.file_path), - "fix_unavailable_reason": violation.fix_unavailable_reason, - "line": violation.line, - "message": violation.message, - "remediation": violation.remediation, - "severity": violation.severity, - } - - -def _relative_path(*, path: Path, root: Path) -> str: - try: - return path.relative_to(root).as_posix() - except ValueError: - return path.as_posix() - - def _styled_summary(*, summary: str, result: LintRunResult, use_color: bool) -> str: style: CliStyle = CliStyle(use_color=use_color) if result.faults: diff --git a/src/sqlbuild/cli/commands/_helpers/skills/update.py b/src/sqlbuild/cli/commands/_helpers/skills/update.py index 6f899e5441..2072fb1691 100644 --- a/src/sqlbuild/cli/commands/_helpers/skills/update.py +++ b/src/sqlbuild/cli/commands/_helpers/skills/update.py @@ -110,10 +110,6 @@ def normalize_skill_targets(*, targets: tuple[str, ...], source: str) -> tuple[s return tuple(normalized_targets) -def load_project_skill_targets(*, project_dir: Path) -> tuple[str, ...]: - return load_project_skill_settings(project_dir=project_dir).targets - - def load_project_skill_settings(*, project_dir: Path) -> SkillSettings: config_path: Path = project_dir / PROJECT_CONFIG_FILENAME if not config_path.exists(): diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index c33697d784..56cf87f724 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -13,7 +13,6 @@ "dbt_project_dir": None, "dbt_profiles_dir": None, "dbt_target": None, - "dbt_target_path": None, "dbt_profile": None, "sqb_output_dir": None, "dry_run": False, @@ -187,7 +186,6 @@ class CliNamespace: dbt_project_dir: str | None dbt_profiles_dir: str | None dbt_target: str | None - dbt_target_path: str | None dbt_profile: str | None sqb_output_dir: str | None dry_run: bool diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 0d44f9df82..941a51f346 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -82,7 +82,6 @@ COST_DEFAULT_HISTORY_LIMIT: int = 10 ISO_DATE_LENGTH: int = 10 COST_NARROW_TERMINAL_WIDTH: int = 120 -COST_TABLE_COLUMN_COUNT: int = 6 COST_HISTORY_NUMERIC_COLUMNS: frozenset[int] = frozenset({4, 5}) COST_RESOURCE_NUMERIC_START_COLUMN: int = 2 BYTE_SCALE: int = 1024 diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py index e594579651..724c45bed9 100644 --- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py +++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py @@ -625,15 +625,6 @@ def _build_projection_is_partial( ) -def _projected_asset_summary(*, assets: tuple[dict[str, object], ...]) -> dict[str, int]: - return { - "success_count": sum(1 for asset in assets if asset.get("status") == _SUCCESS_STATUS), - "failure_count": sum(1 for asset in assets if asset.get("status") == _FAILED_STATUS), - "skipped_count": sum(1 for asset in assets if asset.get("status") == _SKIPPED_STATUS), - "warning_count": sum(1 for asset in assets if asset.get("status") == _WARNING_STATUS), - } - - def _format_model_assets( *, results: tuple[ModelExecutionResult, ...], plan: PlanOutput | None ) -> tuple[dict[str, object], ...]: diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py index 2e895c203f..2c207f0727 100644 --- a/src/sqlbuild/cli/output/models.py +++ b/src/sqlbuild/cli/output/models.py @@ -579,29 +579,6 @@ class WrittenTarget: target_dir: Path diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - def summary_line(self) -> str: - """Build a human-readable summary line.""" - - parts: list[str] = [] - if self.model_count: - model_label: str = "model" if self.model_count == 1 else "models" - parts.append(f"{self.model_count} {model_label}") - if self.seed_count: - seed_label: str = "seed" if self.seed_count == 1 else "seeds" - parts.append(f"{self.seed_count} {seed_label}") - if self.function_count: - function_label: str = "function" if self.function_count == 1 else "functions" - parts.append(f"{self.function_count} {function_label}") - if self.audit_count: - audit_label: str = "audit" if self.audit_count == 1 else "audits" - parts.append(f"{self.audit_count} {audit_label}") - if self.test_count: - test_label: str = "test" if self.test_count == 1 else "tests" - parts.append(f"{self.test_count} {test_label}") - if not parts: - return "Compiled 0 resources" - return f"Compiled {', '.join(parts)}" - @dataclass(frozen=True) class SkillInstallTarget: diff --git a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py index 1d88465d0a..fd4b4d9365 100644 --- a/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py +++ b/src/sqlbuild/cli/progress/classes/nested_command_progress_callbacks.py @@ -53,7 +53,6 @@ def __init__( self._is_tty: bool = hasattr(stream, "isatty") and stream.isatty() self._counter: int = 0 self._current_group: str = "" - self._active_group: str = "" self._active_name: str = "" self._active_group_is_new: bool = False self._spinner_frame_index: int = 0 @@ -74,7 +73,6 @@ def on_item_start( self._projector.expect_resource_enrichment( resource_name=canonical_resource_name or item_name ) - self._active_group = group_name self._active_name = item_name self._active_group_is_new = group_name != self._current_group if self._active_group_is_new: diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py index 36d1ccf721..0cb65058e7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py @@ -20,9 +20,6 @@ ) from sqlbuild.compiler.compile.constants import ( DECIMAL_SQL_TYPE_NAME, - FULL_JOIN_SIDE, - LEFT_JOIN_SIDE, - RIGHT_JOIN_SIDE, SQL_QUALIFIER_SEPARATOR_TOKEN, SQL_WILDCARD_TOKEN, ) @@ -447,61 +444,6 @@ def _analyze_columns_and_lineage_from_polyglot_ast( return tuple(columns), tuple(lineage_columns), has_star -def _extract_polyglot_lineage_facts( - *, - parsed: Any, - references: tuple[CompileSqlReference, ...], -) -> tuple[tuple[CompiledLineageColumnFact, ...], bool]: - if str(getattr(parsed, "kind", "")) != _POLYGLOT_KIND_SELECT: - return (), False - alias_map: dict[str, tuple[CompiledResourceType, str]] = _polyglot_reference_alias_map( - parsed=parsed, - references=references, - ) - unqualified_resource: tuple[CompiledResourceType, str] | None = _single_alias_resource( - alias_map - ) - lineage_columns: list[CompiledLineageColumnFact] = [] - has_star: bool = False - projection: Any - for projection in getattr(parsed, "expressions", ()): - projection = _unwrap_polyglot_annotations(projection) - if bool(getattr(projection, "is_star", False)): - has_star = True - continue - inner: Any = ( - projection.this - if str(getattr(projection, "kind", "")) == _POLYGLOT_KIND_ALIAS - else projection - ) - if bool(getattr(inner, "is_star", False)): - has_star = True - continue - output_column: str = str(getattr(projection, "output_name", "") or "") - if not output_column or output_column == SQL_WILDCARD_TOKEN: - continue - upstream_columns, confidence = _polyglot_lineage_upstream_columns( - projection=projection, - alias_map=alias_map, - unqualified_resource=unqualified_resource, - ) - transform_kind: ColumnTransformKind = _polyglot_lineage_transform_kind( - expression=inner, - has_upstream=bool(upstream_columns), - ) - lineage_columns.append( - CompiledLineageColumnFact( - output_column=output_column, - upstream_columns=upstream_columns, - transform_kind=transform_kind, - confidence=confidence - if upstream_columns or transform_kind == ColumnTransformKind.CONSTANT - else ColumnLineageConfidence.UNKNOWN, - ) - ) - return tuple(lineage_columns), has_star - - def _polyglot_reference_alias_map( *, parsed: Any, references: tuple[CompileSqlReference, ...] ) -> dict[str, tuple[CompiledResourceType, str]]: @@ -893,27 +835,6 @@ def _infer_polyglot_column_nullability( return InferredNullability.UNKNOWN -def _polyglot_columns_in_expression(expression: Any) -> tuple[Any, ...]: - if str(getattr(expression, "kind", "")) == _POLYGLOT_KIND_COLUMN: - return (expression,) - columns: list[Any] = [] - seen: set[int] = set() - - def visit(*, node: Any, visited: set[int], found: list[Any]) -> tuple[set[int], list[Any]]: - node_id: int = id(node) - if node_id in visited: - return visited, found - visited = visited | {node_id} - if str(getattr(node, "kind", "")) == _POLYGLOT_KIND_COLUMN: - return visited, [*found, node] - for child in _polyglot_child_expressions(node): - visited, found = visit(node=child, visited=visited, found=found) - return visited, found - - seen, columns = visit(node=expression, visited=seen, found=columns) - return tuple(columns) - - def _polyglot_column_table_name(column: Any) -> str: payload: object = column.to_dict().get(_POLYGLOT_PAYLOAD_COLUMN, {}) if not isinstance(payload, dict): @@ -925,18 +846,6 @@ def _polyglot_column_table_name(column: Any) -> str: return raw_name if isinstance(raw_name, str) else "" -def _polyglot_child_expressions(expression: Any) -> tuple[Any, ...]: - children: list[Any] = [] - for attr_name in ("this", "expression", "left", "right"): - child: Any | None = getattr(expression, attr_name, None) - if child is not None and str(getattr(child, "kind", "")): - children.append(child) - for child in getattr(expression, "expressions", ()) or (): - if str(getattr(child, "kind", "")): - children.append(child) - return tuple(children) - - def _polyglot_expression_args(expression: Any) -> tuple[Any, ...]: args: list[Any] = [] primary_arg: Any | None = getattr(expression, "this", None) @@ -1012,13 +921,6 @@ def _polyglot_alias_nullability_from_select( return alias_nullability -def _polyglot_table_alias_or_name(table: Any) -> str: - alias_or_name: str = str(getattr(table, "alias_or_name", "") or "") - if alias_or_name: - return alias_or_name - return str(getattr(table, "name", "") or "") - - def _polyglot_table_payload_alias_and_name(table_payload: dict[str, object]) -> tuple[str, str]: table_name: str = _polyglot_name_payload_value(table_payload.get(_POLYGLOT_PAYLOAD_NAME)) alias: str = ( @@ -1111,102 +1013,6 @@ def _replace_table_function_calls_with_stubs( return "".join(parts) -def _find_outermost_select(*, parsed: Any, expressions_module: Any) -> Any | None: - """Find the outermost SELECT statement from a parsed expression.""" - - union_type: type[Any] = expressions_module.Union - select_type: type[Any] = expressions_module.Select - intersect_type: type[Any] = expressions_module.Intersect - except_type: type[Any] = expressions_module.Except - - if isinstance(parsed, (union_type, intersect_type, except_type)): - return parsed.find(select_type) - if isinstance(parsed, select_type): - return parsed - - body: Any | None = getattr(parsed, "this", None) - if body is None: - return None - if isinstance(body, (union_type, intersect_type, except_type)): - return body.find(select_type) - if isinstance(body, select_type): - return body - return parsed.find(select_type) - - -def _is_set_operation(*, parsed: Any, expressions_module: Any) -> bool: - union_type: type[Any] = expressions_module.Union - intersect_type: type[Any] = expressions_module.Intersect - except_type: type[Any] = expressions_module.Except - if isinstance(parsed, (union_type, intersect_type, except_type)): - return True - body: Any | None = getattr(parsed, "this", None) - return isinstance(body, (union_type, intersect_type, except_type)) - - -def _extract_columns_from_select( - *, - select: Any, - expressions_module: Any, - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - infer_nullability: bool, - inference_profile: ExpressionInferenceProfile, -) -> tuple[InferredColumn, ...]: - """Extract output column names and types from a SELECT's projection list.""" - - column_nullability_by_table = dict(column_nullability_by_table) - star_type: type[Any] = expressions_module.Star - annotated_type: type[Any] = expressions_module.Annotated - alias_type: type[Any] = expressions_module.Alias - column_type: type[Any] = expressions_module.Column - cast_type: type[Any] = expressions_module.Cast - try_cast_type: type[Any] = expressions_module.TryCast - alias_nullability: dict[str, InferredNullability] = _alias_nullability_from_select( - select=select, - expressions_module=expressions_module, - column_nullability_by_table=column_nullability_by_table, - ) - - projection_list: list[Any] = select.args.get("expressions", []) - columns: list[InferredColumn] = [] - - expression: Any - for expression in projection_list: - while isinstance(expression, annotated_type): - expression = expression.this - if isinstance(expression, star_type): - continue - - name: str - inner: Any - if isinstance(expression, alias_type): - name = expression.alias - inner = expression.this - elif isinstance(expression, column_type): - name = expression.name - inner = expression - else: - continue - - col_type: str | None = None - if isinstance(inner, (cast_type, try_cast_type)): - col_type = inner.to.sql() - - nullability: InferredNullability = InferredNullability.UNKNOWN - if infer_nullability: - nullability = _infer_expression_nullability( - expression=inner, - expressions_module=expressions_module, - alias_nullability=alias_nullability, - column_nullability_by_table=column_nullability_by_table, - inference_profile=inference_profile, - ) - - columns.append(InferredColumn(name=name, type=col_type, nullability=nullability)) - - return tuple(columns) - - def _infer_expression_nullability( *, expression: Any, @@ -1346,59 +1152,6 @@ def _expression_function_args(expression: Any) -> tuple[Any, ...]: return tuple(args) -def _alias_nullability_from_select( - *, - select: Any, - expressions_module: Any, - column_nullability_by_table: dict[str, dict[str, InferredNullability]], -) -> dict[str, InferredNullability]: - table_type: type[Any] = expressions_module.Table - alias_nullability: dict[str, InferredNullability] = {} - current_aliases: set[str] = set() - - from_expression: Any | None = select.args.get("from_") - from_table: Any | None = getattr(from_expression, "this", None) - if isinstance(from_table, table_type): - alias: str = _table_alias_or_name(from_table) - current_aliases.add(alias) - alias_nullability[alias] = InferredNullability.UNKNOWN - _copy_table_facts_to_alias( - alias=alias, - table_name=from_table.name, - column_nullability_by_table=column_nullability_by_table, - ) - - for join in select.args.get("joins") or []: - joined_table: Any | None = join.this - if not isinstance(joined_table, table_type): - continue - joined_alias: str = _table_alias_or_name(joined_table) - side: str = str(join.args.get("side") or "").upper() - if side == LEFT_JOIN_SIDE: - alias_nullability[joined_alias] = InferredNullability.NULLABLE - elif side == RIGHT_JOIN_SIDE: - for alias in current_aliases: - alias_nullability[alias] = InferredNullability.NULLABLE - alias_nullability[joined_alias] = InferredNullability.UNKNOWN - elif side == FULL_JOIN_SIDE: - for alias in current_aliases: - alias_nullability[alias] = InferredNullability.NULLABLE - alias_nullability[joined_alias] = InferredNullability.NULLABLE - else: - alias_nullability[joined_alias] = InferredNullability.UNKNOWN - current_aliases.add(joined_alias) - _copy_table_facts_to_alias( - alias=joined_alias, - table_name=joined_table.name, - column_nullability_by_table=column_nullability_by_table, - ) - return alias_nullability - - -def _table_alias_or_name(table: Any) -> str: - return str(table.alias_or_name or table.name) - - def _copy_table_facts_to_alias( *, alias: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py index 569ca031a9..bf65ecd3bc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py @@ -221,32 +221,6 @@ def infer_columns_with_sql_analysis( return None -def analyze_columns_with_polyglot( - *, - query_sql: str, - placeholders: dict[str, str] | None = None, - column_nullability_by_table: dict[str, dict[str, InferredNullability]] | None = None, - inference_profile: ExpressionInferenceProfile | None = None, -) -> tuple[InferredColumn, ...] | None | bool: - """Infer columns with one Polyglot parse, returning False when unavailable.""" - - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() - cleaned_sql: str = _replace_refs_with_stubs( - query_sql=query_sql, - dialect=profile.sql_analysis_dialect, - ) - if placeholders: - cleaned_sql = substitute_placeholder_defaults( - query_sql=cleaned_sql, placeholders=placeholders - ) - return _infer_columns_with_polyglot( - cleaned_sql=cleaned_sql, - dialect=profile.sql_analysis_dialect, - column_nullability_by_table=column_nullability_by_table or {}, - inference_profile=profile, - ) - - def analyze_columns_and_lineage_with_polyglot( *, query_sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py index 43294b01a3..c56f1ce44b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/tests.py @@ -7,21 +7,18 @@ from sqlbuild.compiler.compile.constants import ( POLYGLOT_COLUMN_EXPRESSION_NAME, POLYGLOT_SELECT_EXPRESSION_NAME, - POLYGLOT_UNION_EXPRESSION_NAME, + POLYGLOT_SET_OPERATION_EXPRESSION_NAMES, POLYGLOT_WRAPPER_EXPRESSION_NAMES, - SQL_CLOSE_PAREN_TOKEN, - SQL_IDENTIFIER_EXTRA_TOKEN, - SQL_OPEN_PAREN_TOKEN, - SQL_QUOTE_TOKENS, - SQL_SET_OPERATION_KEYWORDS, - SQL_UNION_ALL_KEYWORD, - SQL_UNION_KEYWORD, ) from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( + split_set_operation_branches, +) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _POLYGLOT_VALUES_EXPRESSION_NAME: str = "Values" _POLYGLOT_VALUES_SET_ALIAS: str = "_values" +_SCAN_CONTEXT: str = "SQL test" def extract_expected_branch_column_names_with_sql_analysis( @@ -34,25 +31,37 @@ def extract_expected_branch_column_names_with_sql_analysis( try: parsed_expression: Any = polyglot_module.parse_one(sql, dialect="generic") except polyglot_module.PolyglotError: - branches: tuple[str, ...] = _split_set_operation_branches(sql) - if len(branches) > 1: - return tuple( - _extract_branch_names( - expression=polyglot_module.parse_one(branch_sql, dialect="generic"), - file_label=file_label, - label=label, - )[0] - for branch_sql in branches - ) - return None + return _extract_unparsed_branch_names( + sql=sql, polyglot_module=polyglot_module, file_label=file_label, label=label + ) return _extract_branch_names(expression=parsed_expression, file_label=file_label, label=label) +def _extract_unparsed_branch_names( + *, sql: str, polyglot_module: Any, file_label: str, label: str +) -> tuple[tuple[str, ...], ...] | None: + """Parse each top-level set-operation branch separately when the whole query cannot parse.""" + + branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_SCAN_CONTEXT) + if len(branches) <= 1: + return None + try: + parsed_branches: tuple[Any, ...] = tuple( + polyglot_module.parse_one(branch_sql, dialect="generic") for branch_sql in branches + ) + except polyglot_module.PolyglotError: + return None + return tuple( + _extract_branch_names(expression=expression, file_label=file_label, label=label)[0] + for expression in parsed_branches + ) + + def _extract_branch_names( *, expression: Any, file_label: str, label: str ) -> tuple[tuple[str, ...], ...]: expression = _unwrap_expression(expression=expression) - if expression.__class__.__name__ == POLYGLOT_UNION_EXPRESSION_NAME: + if expression.__class__.__name__ in POLYGLOT_SET_OPERATION_EXPRESSION_NAMES: left_expression: Any = expression.args["left"] right_expression: Any = expression.args["right"] return ( @@ -113,70 +122,3 @@ def _extract_select_names(*, expression: Any, file_label: str, label: str) -> tu f"SQL test '{file_label}' must project at least one column in {label}" ) return tuple(names) - - -def _split_set_operation_branches(sql: str) -> tuple[str, ...]: - branches: list[str] = [] - branch_start: int = 0 - depth: int = 0 - quote: str | None = None - index: int = 0 - while index < len(sql): - character: str = sql[index] - if quote is not None: - if character == quote: - quote = None - index += 1 - continue - if character in SQL_QUOTE_TOKENS: - quote = character - index += 1 - continue - if character == SQL_OPEN_PAREN_TOKEN: - depth += 1 - index += 1 - continue - if character == SQL_CLOSE_PAREN_TOKEN: - depth = max(0, depth - 1) - index += 1 - continue - operation_end: int | None = _consume_set_operation(sql=sql, start=index) - if depth == 0 and operation_end is not None: - branch_sql: str = sql[branch_start:index].strip() - if branch_sql: - branches.append(branch_sql) - branch_start = operation_end - index = operation_end - continue - index += 1 - final_branch_sql: str = sql[branch_start:].strip() - if final_branch_sql: - branches.append(final_branch_sql) - return tuple(branches) - - -def _consume_set_operation(*, sql: str, start: int) -> int | None: - keyword: str - for keyword in SQL_SET_OPERATION_KEYWORDS: - end: int = start + len(keyword) - if sql[start:end].upper() != keyword: - continue - if start > 0 and (sql[start - 1].isalnum() or sql[start - 1] == SQL_IDENTIFIER_EXTRA_TOKEN): - continue - if end < len(sql) and (sql[end].isalnum() or sql[end] == SQL_IDENTIFIER_EXTRA_TOKEN): - continue - index: int = _skip_whitespace(sql=sql, start=end) - if keyword == SQL_UNION_KEYWORD and sql[index : index + 3].upper() == SQL_UNION_ALL_KEYWORD: - all_end: int = index + 3 - if all_end == len(sql) or not ( - sql[all_end].isalnum() or sql[all_end] == SQL_IDENTIFIER_EXTRA_TOKEN - ): - index = _skip_whitespace(sql=sql, start=all_end) - return index - return None - - -def _skip_whitespace(*, sql: str, start: int) -> int: - while start < len(sql) and sql[start].isspace(): - start += 1 - return start diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 95cea9d4b5..2bf965757d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -24,7 +24,6 @@ from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile._helpers.render.arguments import ( render_parameterized_sql, - render_sql_argument_value, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion @@ -94,7 +93,6 @@ def remember_scoped_declarations( self.scoped_declarations[key] = declarations -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} @@ -676,43 +674,6 @@ def render_generic_audit_sql( ) -def render_generic_audit_argument( - *, - argument_name: str, - arguments: dict[str, object], - owner_file: Path, - definition_name: str, - quoted: bool, -) -> str: - """Render one generic attached-audit parameter value into SQL text.""" - - return render_parameterized_sql( - sql=f"@'{argument_name}'" if quoted else f"@{argument_name}", - arguments=arguments, - owner_label=str(owner_file), - definition_label=f"generic audit '{definition_name}'", - ) - - -def render_generic_audit_argument_value( - *, - argument_value: object, - owner_file: Path, - definition_name: str, - argument_name: str, - quoted: bool, -) -> str: - """Render one generic attached-audit argument value using raw or literal SQL rules.""" - - return render_sql_argument_value( - argument_value=argument_value, - owner_label=str(owner_file), - definition_label=f"audit '{definition_name}'", - argument_name=argument_name, - quoted=quoted, - ) - - def resolve_audit_severity( *, instance_severity: str | AuditSeverity | None, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 189916b29d..b8ef635ced 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -1115,33 +1115,6 @@ def build_model_config(*, request: ModelConfigBuildRequest) -> CompileModelConfi ) -def expand_model_hook_macros( - *, - values: dict[str, object], - file_path: Path, - effective_vars: dict[str, object], - context_values: dict[str, str | None], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - declaration_expansion: DeclarationExpansionContext, - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] | None = None, - consumer: ResourceIdentity | None = None, -) -> dict[str, object]: - """Expand SQL interpolation and macros within executable hook SQL strings.""" - - return expand_model_hook_macros_result( - values=values, - file_path=file_path, - effective_vars=effective_vars, - context_values=context_values, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=declaration_expansion, - sql_hook_definitions=sql_hook_definitions, - consumer=consumer, - ).values - - def expand_model_hook_macros_result( *, values: dict[str, object], diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index cfb32eba0e..a48b3c9dfb 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -80,7 +80,6 @@ class _PythonFunctionBuildContext: _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index ea7c56e8c0..863629bf07 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -521,29 +521,6 @@ def _resolve_model_schema( return declaration -def _merge_schema_tags( - *, config: CompileModelConfig, schema_entry: SchemaModelEntry -) -> CompileModelConfig: - """Union schema.yml tags into model config values.""" - - if not schema_entry.tags: - return config - merged_values: dict[str, object] = _merged_with_tag_union( - base=dict(config.values), - overlay={"tags": list(schema_entry.tags)}, - ) - return CompileModelConfig( - values=merged_values, - model_header_keys=config.model_header_keys, - matched_path_default=config.matched_path_default, - logical_schema=config.logical_schema, - layer_schema=config.layer_schema, - logical_database=config.logical_database, - time_travel_retention=config.time_travel_retention, - table_type=config.table_type, - ) - - def find_matching_path_default( *, model_file: DiscoveredSqlModelFile, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py index 1e76b40573..6a1da58202 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py @@ -20,7 +20,6 @@ from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py index 3383ed4e17..55ca9f3889 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py @@ -39,7 +39,6 @@ ) _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index f11dd9b130..26a0eb28b2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -77,7 +77,6 @@ from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind, UsageKind _HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_LEGACY_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hook", "post_hook"}) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( {"ctx", "context", "_ctx", "hook_context"} diff --git a/src/sqlbuild/compiler/compile/_helpers/config/deprecation.py b/src/sqlbuild/compiler/compile/_helpers/config/deprecation.py deleted file mode 100644 index 1b078c6bd1..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/deprecation.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Build compile diagnostics for deprecated model configuration.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic - - -def build_cursor_alias_diagnostics( - *, model_inputs: tuple[CompileModelInput, ...] -) -> tuple[CompilerDiagnostic, ...]: - """Build one project-level deprecation warning for the legacy cursor alias.""" - - return () diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index f150d64e1c..d847acff0c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -474,34 +474,6 @@ def _validate_no_inherited_column_overrides( ) -def expand_declaration_references( - *, - sql: str, - file_path: Path, - enums: dict[str, EnumDeclaration], - constants: dict[str, ConstantDeclaration], - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - inaccessible_enums: dict[str, DeclarationRecord] | None = None, - inaccessible_constants: dict[str, DeclarationRecord] | None = None, -) -> str: - """Resolve enum-member and constant references to SQL scalar literals.""" - - result: DeclarationExpansionResult = expand_declaration_references_result( - sql=sql, - file_path=file_path, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - declarations=DeclarationResolutionContext( - enums=enums, - constants=constants, - inaccessible_enums=inaccessible_enums or {}, - inaccessible_constants=inaccessible_constants or {}, - ), - ) - return result.sql - - def expand_declaration_references_with_spans( *, sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 561c891295..2d4dd10672 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -55,51 +55,6 @@ _NATIVE_RESULT_LENGTH: int = 2 -def validate_var_macro_collision( - *, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], -) -> None: - """Raise if any project var name collides with a macro name.""" - - collisions: set[str] = set(effective_vars) & set(loaded_macros) - if collisions: - names: str = ", ".join(sorted(collisions)) - raise CompileInputError( - f"project variable names collide with macro names: {names}. " - f"Rename the variable or macro to avoid ambiguity." - ) - - -def expand_authored_sql( # noqa: PLR0913 - *, - sql: str, - file_path: Path, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, - context_values: Mapping[str, str | None] | None = None, - declarations: DeclarationResolutionContext | None = None, - declaration_resolver: DeclarationScopeResolver | None = None, -) -> str: - """Apply SQL interpolation and macro expansion to authored SQL text.""" - - return expand_authored_sql_result( - sql=sql, - file_path=file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - context_values=context_values, - declarations=declarations, - declaration_resolver=declaration_resolver, - ).sql - - def expand_authored_sql_result( # noqa: PLR0913 *, sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py deleted file mode 100644 index 795fef6f12..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/cache.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Versioned project-local cache for SQL test CTE extraction.""" - -from __future__ import annotations - -import hashlib -import hmac -import json -import sqlite3 -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from pathlib import Path -from types import TracebackType -from typing import Any, cast - -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - classify_sql_test_ctes, - extract_unclassified_sql_test_ctes, -) -from sqlbuild.compiler.compile.models import CompileSqlTestCte, CompileSqlTestCtes -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.profiling.main.record import record_compile_timing - -_CACHE_VERSION: int = 1 -_CACHE_DATABASE_NAME: str = "sql-test-ctes.sqlite3" -_MAX_CACHE_ENTRY_BYTES: int = 10_000_000 -_SQLITE_TIMEOUT_SECONDS: float = 0.1 -_CACHE_CTE_ROW_FIELD_COUNT: int = 2 -_CREATE_CACHE_TABLE_SQL: str = """ -CREATE TABLE IF NOT EXISTS sql_test_cte ( - cache_key TEXT PRIMARY KEY, - payload TEXT NOT NULL -) -""" - - -class _SqlTestCteCache: - """Reuse exact CTE boundaries while retaining classification and validation.""" - - def __init__(self, *, root: Path | None) -> None: - self._root: Path | None = root - self._database_path: Path | None = None - self._connection: sqlite3.Connection | None = None - self._pending_by_key: dict[str, str] = {} - - def __enter__(self) -> _SqlTestCteCache: - if self._root is None: - return self - self._database_path = self._root / f"sql-test-ctes-v{_CACHE_VERSION}" / _CACHE_DATABASE_NAME - if not self._database_path.is_file(): - return self - try: - self._connection = sqlite3.connect( - self._database_path, - timeout=_SQLITE_TIMEOUT_SECONDS, - ) - except (OSError, sqlite3.DatabaseError): - self._disable() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exc_value, traceback - connection: sqlite3.Connection | None = self._connection - self._connection = None - try: - if exc_type is None: - with record_compile_timing("cache_publication_ms"): - connection = self._write_pending(connection=connection) - elif connection is not None: - connection.rollback() - except (OSError, sqlite3.DatabaseError): - pass - finally: - self._pending_by_key.clear() - if connection is not None: - try: - connection.close() - except sqlite3.DatabaseError: - pass - - def extract(self, *, sql: str, file_label: str, mode: SqlTestMode) -> CompileSqlTestCtes: - """Return cached raw CTEs or scan and record this exact expanded SQL.""" - - cache_key: str = hashlib.sha256(sql.encode()).hexdigest() - ctes: tuple[CompileSqlTestCte, ...] | None = self._pending_ctes(cache_key=cache_key) - if ctes is None: - ctes = self._read_ctes(cache_key=cache_key) - if ctes is None: - ctes = extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label) - if self._database_path is not None: - contents: str = _cache_contents(cache_key=cache_key, ctes=ctes) - if len(contents.encode()) <= _MAX_CACHE_ENTRY_BYTES: - self._pending_by_key[cache_key] = contents - return classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode) - - def _pending_ctes(self, *, cache_key: str) -> tuple[CompileSqlTestCte, ...] | None: - contents: str | None = self._pending_by_key.get(cache_key) - return ( - None - if contents is None - else _ctes_from_contents(contents=contents, cache_key=cache_key) - ) - - def _read_ctes(self, *, cache_key: str) -> tuple[CompileSqlTestCte, ...] | None: - connection: sqlite3.Connection | None = self._connection - if connection is None: - return None - try: - row: tuple[str] | None = connection.execute( - "SELECT payload FROM sql_test_cte WHERE cache_key = ?", - (cache_key,), - ).fetchone() - except sqlite3.DatabaseError: - self._disable() - return None - return None if row is None else _ctes_from_contents(contents=row[0], cache_key=cache_key) - - def _write_pending(self, *, connection: sqlite3.Connection | None) -> sqlite3.Connection | None: - if not self._pending_by_key or self._database_path is None: - return connection - if connection is None: - self._database_path.parent.mkdir(parents=True, exist_ok=True) - connection = sqlite3.connect(self._database_path, timeout=_SQLITE_TIMEOUT_SECONDS) - _ = connection.execute(_CREATE_CACHE_TABLE_SQL) - _ = connection.executemany( - "INSERT OR REPLACE INTO sql_test_cte (cache_key, payload) VALUES (?, ?)", - self._pending_by_key.items(), - ) - connection.commit() - return connection - - def _disable(self) -> None: - connection: sqlite3.Connection | None = self._connection - self._connection = None - if connection is not None: - try: - connection.close() - except sqlite3.DatabaseError: - pass - - -def _cache_contents(*, cache_key: str, ctes: tuple[CompileSqlTestCte, ...]) -> str: - serialized: str = json.dumps( - { - "v": _CACHE_VERSION, - "k": cache_key, - "c": [[cte.name, cte.sql_body] for cte in ctes], - }, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - return f"{_cache_digest(cache_key=cache_key, serialized=serialized)}\n{serialized}" - - -def _ctes_from_contents( - *, contents: object, cache_key: str -) -> tuple[CompileSqlTestCte, ...] | None: - if not isinstance(contents, str) or len(contents.encode()) > _MAX_CACHE_ENTRY_BYTES: - return None - try: - digest, separator, serialized = contents.partition("\n") - if not separator or not hmac.compare_digest( - digest, - _cache_digest(cache_key=cache_key, serialized=serialized), - ): - return None - payload: object = json.loads(serialized) - if not isinstance(payload, dict) or payload.get("v") != _CACHE_VERSION: - return None - if payload.get("k") != cache_key or not isinstance(payload.get("c"), list): - return None - rows: list[object] = cast(list[object], payload["c"]) - if not all( - isinstance(row, list) - and len(row) == _CACHE_CTE_ROW_FIELD_COUNT - and isinstance(row[0], str) - and isinstance(row[1], str) - for row in rows - ): - return None - return tuple( - CompileSqlTestCte(name=cast(list[str], row)[0], sql_body=cast(list[str], row)[1]) - for row in rows - ) - except (ValueError, TypeError, KeyError, RecursionError, json.JSONDecodeError): - return None - - -def _cache_digest(*, cache_key: str, serialized: str) -> str: - digest: Any = hashlib.sha256(cache_key.encode()) - digest.update(b"\0") - digest.update(serialized.encode()) - return digest.hexdigest() - - -@contextmanager -def cached_sql_test_cte_extractor( - *, root: Path | None -) -> Iterator[Callable[[str, str, SqlTestMode], CompileSqlTestCtes]]: - """Yield an exact cached SQL test CTE extractor for one compile invocation.""" - - with _SqlTestCteCache(root=root) as cache: - yield lambda sql, file_label, mode: cache.extract( - sql=sql, - file_label=file_label, - mode=mode, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 9887de4642..3b49726ee3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -63,6 +63,9 @@ from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import ( skip_quoted_text, ) +from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( + split_set_operation_branches, +) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _CONTEXT: str = "SQL test" @@ -479,6 +482,7 @@ def _classify_model_sql_test_ctes( _validate_expected_cte_query( cte=cte, file_label=file_label, + label=cte.name, allow_empty_fixture=True, ) expected_ctes.append(cte) @@ -702,42 +706,13 @@ def _extract_expected_branch_column_names( ) if sql_analysis_column_names is not None: return sql_analysis_column_names - branches: tuple[str, ...] = _split_set_operation_branches(sql) + branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_CONTEXT) return tuple( _extract_expected_select_column_names(branch_sql=branch, file_label=file_label, label=label) for branch in branches ) -def _split_set_operation_branches(sql: str) -> tuple[str, ...]: - branches: list[str] = [] - branch_start: int = 0 - resume: int = 0 - index: int - depth: int - for index, depth in iter_code_positions(sql=sql, context=_CONTEXT): - if index < resume or depth != 0: - continue - union_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="UNION") - if union_end is None: - continue - branch_sql: str = sql[branch_start:index].strip() - if branch_sql: - branches.append(branch_sql) - resume = _skip_ignorable(sql=sql, start=union_end) - quantifier_end: int | None = _try_consume_keyword( - sql=sql, start=resume, keyword="ALL" - ) or _try_consume_keyword(sql=sql, start=resume, keyword="DISTINCT") - if quantifier_end is not None: - resume = _skip_ignorable(sql=sql, start=quantifier_end) - branch_start = resume - - final_branch_sql: str = sql[branch_start:].strip() - if final_branch_sql: - branches.append(final_branch_sql) - return tuple(branches) - - def _extract_expected_select_column_names( *, branch_sql: str, file_label: str, label: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 8eb5364e16..552d76a929 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -19,10 +19,7 @@ CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" -CURSOR_FILTER_INPUTS_CONFIG_KEY: str = "cursor_filter_inputs" -CURSOR_WATERMARK_INPUTS_CONFIG_KEY: str = "cursor_watermark_inputs" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" -MICROBATCH_LIMIT_CONFIG_KEY: str = "microbatch_limit" MICROBATCH_LIMIT_MAX_BATCHES_KEY: str = "max_batches" MICROBATCH_LIMIT_ACTION_KEY: str = "action" WATERMARK_CURSOR_INPUT_BLOCK_KEYS: frozenset[str] = frozenset({"column", "roles"}) @@ -49,19 +46,15 @@ SQL_QUALIFIER_SEPARATOR_TOKEN: str = "." SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) SQL_IDENTIFIER_EXTRA_TOKEN: str = "_" -SQL_UNION_KEYWORD: str = "UNION" -SQL_UNION_ALL_KEYWORD: str = "ALL" -SQL_SET_OPERATION_KEYWORDS: tuple[str, ...] = ("UNION", "INTERSECT", "EXCEPT") SQL_WITH_KEYWORD: str = "WITH" SQL_CEREMONIAL_SELECT_VALUE: str = "1" UNKNOWN_SQL_TYPE_NAME: str = "UNKNOWN" DECIMAL_SQL_TYPE_NAME: str = "DECIMAL" RESOLVED_SOURCE_CONFIDENCE: str = "resolved" -LEFT_JOIN_SIDE: str = "LEFT" -RIGHT_JOIN_SIDE: str = "RIGHT" -FULL_JOIN_SIDE: str = "FULL" -POLYGLOT_UNION_EXPRESSION_NAME: str = "Union" +POLYGLOT_SET_OPERATION_EXPRESSION_NAMES: frozenset[str] = frozenset( + {"Union", "Intersect", "Except"} +) POLYGLOT_SELECT_EXPRESSION_NAME: str = "Select" POLYGLOT_COLUMN_EXPRESSION_NAME: str = "Column" POLYGLOT_WRAPPER_EXPRESSION_NAMES: frozenset[str] = frozenset({"Subquery", "Paren"}) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index c70da831c5..4ec1ba2245 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -31,7 +31,6 @@ from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) -from sqlbuild.compiler.compile._helpers.config.deprecation import build_cursor_alias_diagnostics from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id from sqlbuild.compiler.compile._helpers.render.declarations import ( build_public_declaration_indexes, @@ -309,8 +308,5 @@ def _build_models_with_declarations( public_enums=declarations.enums, public_constants=declarations.constants, ), - diagnostics=( - *build_cursor_alias_diagnostics(model_inputs=model_inputs), - *build_audit_factory_orphan_diagnostics(discovered_inputs=discovered_inputs), - ), + diagnostics=build_audit_factory_orphan_diagnostics(discovered_inputs=discovered_inputs), ) diff --git a/src/sqlbuild/compiler/compile/main/_cursor_roles.py b/src/sqlbuild/compiler/compile/main/_cursor_roles.py index 06762b8d57..fc5b437af6 100644 --- a/src/sqlbuild/compiler/compile/main/_cursor_roles.py +++ b/src/sqlbuild/compiler/compile/main/_cursor_roles.py @@ -47,7 +47,6 @@ def resolve_cursor_input_roles(*, model: CompiledModel) -> CursorInputRoles: watermark_inputs=watermark_inputs, filter_field=CURSOR_INPUTS_CONFIG_KEY, watermark_field=CURSOR_INPUTS_CONFIG_KEY, - uses_legacy_alias=False, ) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index f8fd4c18cd..d54e22ac83 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -220,7 +220,6 @@ class CursorInputRoles: watermark_inputs: dict[str, str] filter_field: str watermark_field: str - uses_legacy_alias: bool @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/contracts/models.py b/src/sqlbuild/compiler/contracts/models.py index 4040f2076d..70d9863a9d 100644 --- a/src/sqlbuild/compiler/contracts/models.py +++ b/src/sqlbuild/compiler/contracts/models.py @@ -12,9 +12,3 @@ class ContractValidationResult: """Diagnostics emitted by model contract validation.""" diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - - @property - def has_errors(self) -> bool: - """Return whether contract validation found errors.""" - - return any(diagnostic.is_error for diagnostic in self.diagnostics) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index ebc6b82114..d02b078ab4 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -450,7 +450,6 @@ def _discover_model_file( model_name=file_path.stem, relative_path=relative_path, ), - output_column_locations_extracted=extract_output_column_locations, extract_implicit_alias_columns=extract_implicit_alias_columns, ) @@ -894,46 +893,6 @@ def discover_materialization_files( return tuple(discovered_files) -def discover_loader_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredLoaderFunction, ...]: - """Discover decorated source loader functions under loaders/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).loaders - ) - - -def discover_task_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredTaskFunction, ...]: - """Discover decorated task functions under tasks/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).tasks - ) - - -def discover_asset_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredAssetFunction, ...]: - """Discover decorated asset functions under assets/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).assets - ) - - -def discover_check_functions( - *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () -) -> tuple[DiscoveredCheckFunction, ...]: - """Discover decorated check functions under checks/.""" - - return tuple( - _discover_python_node_functions(project_dir=project_dir, providers=providers).checks - ) - - def discover_python_node_functions( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> DiscoveredPythonNodeFunctions: diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 3cad384065..5ba5185769 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -10,7 +10,6 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.auditing.types import ThresholdOperator from sqlbuild.compiler.discovery.exceptions import ( DiscoveryError, ModelHeaderSyntaxError, @@ -436,24 +435,6 @@ def _location_for_projection_range( ) -def _location_for_header_token( - *, - contents: str, - header_start: int, - token: _ModelHeaderToken, - relative_path: Path, - line_starts: tuple[int, ...], -) -> SourceLocation: - absolute_position: int = header_start + token.position - return _location_for_absolute_span( - contents=contents, - start=absolute_position, - end=absolute_position + len(token.value), - relative_path=relative_path, - line_starts=line_starts, - ) - - def _location_for_absolute_span( *, contents: str, start: int, end: int, relative_path: Path, line_starts: tuple[int, ...] ) -> SourceLocation: @@ -485,38 +466,6 @@ def _line_starts(contents: str) -> tuple[int, ...]: return tuple(starts) -def _find_top_level_keyword(*, sql: str, keyword: str, start: int) -> int | None: - depth: int = 0 - index: int = start - in_quote: str | None = None - while index < len(sql): - character: str = sql[index] - if in_quote is not None: - if character == _MODEL_HEADER_ESCAPE_CHARACTER: - index += 2 - continue - if character == in_quote: - in_quote = None - index += 1 - continue - if character in _MODEL_HEADER_QUOTE_NAMES: - in_quote = character - index += 1 - continue - if character == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - index += 1 - continue - if character == _MODEL_HEADER_CLOSE_PAREN: - depth = max(0, depth - 1) - index += 1 - continue - if depth == 0 and _keyword_at(sql=sql, keyword=keyword, index=index): - return index - index += 1 - return None - - def _split_top_level_select_items(*, sql: str, start: int, end: int) -> tuple[tuple[int, int], ...]: items: list[tuple[int, int]] = [] depth: int = 0 @@ -737,298 +686,6 @@ def _project_native_header_value(value: object) -> object: raise ModelHeaderSyntaxError(f"Native MODEL header parser returned invalid '{kind}' marker") -class _ModelHeaderParser: - def __init__(self, *, header: str) -> None: - self._tokens: list[_ModelHeaderToken] = _tokenize_model_header_for_spans(header) - self._index: int = 0 - - def parse(self) -> dict[str, object]: - if self._peek().kind == _MODEL_HEADER_END_TOKEN: - return {} - values: dict[str, object] = self._parse_map(end_symbol=None) - self._expect_end() - return values - - def _parse_map( - self, - *, - end_symbol: str | None, - threshold_policy: bool = False, - allow_outside_threshold: bool = False, - column_entries: bool = False, - column_metadata: bool = False, - ) -> dict[str, object]: - values: dict[str, object] = {} - while not self._is_at_end_symbol(end_symbol): - if self._match_symbol(_MODEL_HEADER_COMMA): - continue - key: str = self._consume_key() - if key in values: - raise ModelHeaderSyntaxError(f"duplicate key '{key}'") - if self._match_symbol(_MODEL_HEADER_KEY_VALUE_SEPARATOR): - raise ModelHeaderSyntaxError( - f"unexpected ':' after key '{key}'; use SQLBuild syntax '{key} value'" - ) - if self._is_at_end_symbol(end_symbol) or self._peek().kind == _MODEL_HEADER_END_TOKEN: - raise ModelHeaderSyntaxError( - f"unexpected token '{key}' without a value; quote values with spaces" - ) - if allow_outside_threshold and key == ThresholdOperator.OUTSIDE: - values[key] = self._parse_outside_threshold() - elif key in _MODEL_HEADER_HOOK_FIELD_NAMES: - values[key] = self._parse_hook_field_value(key) - elif column_metadata and key == _MODEL_HEADER_TYPE_KEY: - values[key] = self._parse_type_value() - else: - values[key] = self._parse_value( - threshold_policy=key == _MODEL_HEADER_THRESHOLDS_KEY, - allow_outside_threshold=( - threshold_policy and key in _MODEL_HEADER_THRESHOLD_BOUND_KEYS - ), - nested_column_entries=key == _MODEL_HEADER_COLUMNS_KEY, - nested_column_metadata=column_entries, - ) - self._match_symbol(_MODEL_HEADER_COMMA) - if end_symbol is not None: - self._consume_symbol(end_symbol) - return values - - def _parse_outside_threshold(self) -> tuple[object, object]: - """Parse the directional threshold shorthand ``outside LOW HIGH``.""" - - lower: object = self._parse_value() - if self._is_at_end_symbol(_MODEL_HEADER_CLOSE_PAREN): - raise ModelHeaderSyntaxError("outside threshold requires lower and upper values") - upper: object = self._parse_value() - return lower, upper - - def _parse_type_value(self) -> object: - """Parse parameterized SQL types as scalar header values.""" - - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_WORD_TOKEN: - return self._parse_value() - if not ( - self._tokens[self._index + 1].kind == _MODEL_HEADER_SYMBOL_TOKEN - and self._tokens[self._index + 1].value == _MODEL_HEADER_OPEN_PAREN - ): - return self._parse_value() - - parts: list[str] = [self._advance().value, self._advance().value] - depth: int = 1 - while depth > 0: - parameter_token: _ModelHeaderToken = self._peek() - if parameter_token.kind == _MODEL_HEADER_END_TOKEN: - raise ModelHeaderSyntaxError(f"unterminated SQL type at position {token.position}") - if parameter_token.kind == _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError( - f"quoted SQL type parameters require the whole type to be quoted " - f"at position {parameter_token.position}" - ) - self._advance() - parts.append(parameter_token.value) - if parameter_token.kind != _MODEL_HEADER_SYMBOL_TOKEN: - continue - if parameter_token.value == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - elif parameter_token.value == _MODEL_HEADER_CLOSE_PAREN: - depth -= 1 - return "".join(parts) - - def _parse_value( - self, - *, - threshold_policy: bool = False, - allow_outside_threshold: bool = False, - nested_column_entries: bool = False, - nested_column_metadata: bool = False, - ) -> object: - token: _ModelHeaderToken = self._peek() - if token.kind == _MODEL_HEADER_STRING_TOKEN: - self._advance() - return token.value - if token.kind == _MODEL_HEADER_WORD_TOKEN: - self._advance() - if ( - self._peek().kind == _MODEL_HEADER_SYMBOL_TOKEN - and self._peek().value == _MODEL_HEADER_OPEN_PAREN - ): - self._advance() - if token.value in _MODEL_HEADER_RELATION_CALL_NAMES: - return self._parse_relation_call(token.value) - if token.value in _MODEL_HEADER_HOOK_CALL_NAMES: - return self._parse_hook_call(token.value) - if token.value == _MODEL_HEADER_TYPED_CONSTANT_CALL: - return AuthoredSqlValueCall( - arguments=tuple( - self._parse_map(end_symbol=_MODEL_HEADER_CLOSE_PAREN).items() - ) - ) - return { - token.value: self._parse_map( - end_symbol=_MODEL_HEADER_CLOSE_PAREN, - threshold_policy=threshold_policy, - allow_outside_threshold=allow_outside_threshold, - ) - } - return _parse_word_value(token.value) - if self._match_symbol(_MODEL_HEADER_OPEN_BRACKET): - return self._parse_list() - if self._match_symbol(_MODEL_HEADER_OPEN_BRACE): - return AuthoredSqlSet(values=tuple(self._parse_sequence(_MODEL_HEADER_CLOSE_BRACE))) - if self._match_symbol(_MODEL_HEADER_OPEN_PAREN): - return self._parse_map( - end_symbol=_MODEL_HEADER_CLOSE_PAREN, - threshold_policy=threshold_policy, - allow_outside_threshold=allow_outside_threshold, - column_entries=nested_column_entries, - column_metadata=nested_column_metadata, - ) - raise ModelHeaderSyntaxError(f"expected value at position {token.position}") - - def _parse_list(self) -> list[object]: - return self._parse_sequence(_MODEL_HEADER_CLOSE_BRACKET) - - def _parse_sequence(self, end_symbol: str) -> list[object]: - values: list[object] = [] - while not self._is_at_end_symbol(end_symbol): - if self._match_symbol(_MODEL_HEADER_COMMA): - continue - values.append(self._parse_value()) - self._match_symbol(_MODEL_HEADER_COMMA) - self._consume_symbol(end_symbol) - return values - - def _parse_hook_field_value(self, field_name: str) -> list[object]: - if not self._match_symbol(_MODEL_HEADER_OPEN_BRACKET): - token: _ModelHeaderToken = self._peek() - raise ModelHeaderSyntaxError( - f"{field_name} must be a list of typed inline_sql(...), sql(...), or " - "python(...) hook entries " - f"at position {token.position}" - ) - values: list[object] = [] - while not self._is_at_end_symbol(_MODEL_HEADER_CLOSE_BRACKET): - if self._match_symbol(_MODEL_HEADER_COMMA): - continue - values.append(self._parse_hook_list_entry(field_name)) - self._match_symbol(_MODEL_HEADER_COMMA) - self._consume_symbol(_MODEL_HEADER_CLOSE_BRACKET) - return values - - def _parse_hook_list_entry(self, field_name: str) -> object: - token: _ModelHeaderToken = self._peek() - if token.kind == _MODEL_HEADER_STRING_TOKEN: - self._advance() - return token.value - if token.kind not in {_MODEL_HEADER_WORD_TOKEN, _MODEL_HEADER_STRING_TOKEN}: - raise ModelHeaderSyntaxError( - f"{field_name} entries must use typed inline_sql(...), sql(...), or " - "python(...) hook syntax" - ) - self._advance() - if ( - self._peek().kind != _MODEL_HEADER_SYMBOL_TOKEN - or self._peek().value != _MODEL_HEADER_OPEN_PAREN - ): - raise ModelHeaderSyntaxError( - f"{field_name} entries must use typed inline_sql(...), sql(...), or " - "python(...) hook syntax" - ) - self._advance() - if token.value not in _MODEL_HEADER_HOOK_CALL_NAMES: - raise ModelHeaderSyntaxError( - f"{field_name} entries must use typed inline_sql(...), sql(...), or " - "python(...) hook syntax" - ) - return self._parse_hook_call(token.value) - - def _parse_relation_call(self, name: str) -> str: - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError(f"{name}(...) requires a double-quoted relation name") - self._advance() - relation_name: str = token.value.replace('"', '\\"') - self._consume_symbol(_MODEL_HEADER_CLOSE_PAREN) - return f'{name}("{relation_name}")' - - def _parse_hook_call(self, name: str) -> SqlHookEntry | NamedSqlHookEntry | PythonHookEntry: - if name == _MODEL_HEADER_INLINE_SQL_HOOK_CALL: - statement_token: _ModelHeaderToken = self._peek() - if statement_token.kind != _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError("inline_sql(...) requires a quoted SQL string") - self._advance() - if self._match_symbol(_MODEL_HEADER_COMMA): - raise ModelHeaderSyntaxError("inline_sql(...) does not accept additional arguments") - self._consume_symbol(_MODEL_HEADER_CLOSE_PAREN) - return SqlHookEntry(statement=statement_token.value) - - hook_name_token: _ModelHeaderToken = self._peek() - if hook_name_token.kind != _MODEL_HEADER_STRING_TOKEN: - raise ModelHeaderSyntaxError(f"{name}(...) requires a quoted hook name") - self._advance() - if not hook_name_token.value.strip(): - raise ModelHeaderSyntaxError(f"{name}(...) requires a non-empty hook name") - kwargs: dict[str, object] = self._parse_hook_kwargs() - if name == _MODEL_HEADER_NAMED_SQL_HOOK_CALL: - return NamedSqlHookEntry(name=hook_name_token.value, kwargs=kwargs) - return PythonHookEntry(name=hook_name_token.value, kwargs=kwargs) - - def _parse_hook_kwargs(self) -> dict[str, object]: - kwargs: dict[str, object] = {} - while self._match_symbol(_MODEL_HEADER_COMMA): - if self._is_at_end_symbol(_MODEL_HEADER_CLOSE_PAREN): - break - key: str = self._consume_key() - if key in kwargs: - raise ModelHeaderSyntaxError(f"duplicate hook argument '{key}'") - self._consume_symbol(_MODEL_HEADER_KEY_VALUE_SEPARATOR) - kwargs[key] = self._parse_value() - self._consume_symbol(_MODEL_HEADER_CLOSE_PAREN) - return kwargs - - def _consume_key(self) -> str: - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_WORD_TOKEN: - raise ModelHeaderSyntaxError(f"expected key at position {token.position}") - self._advance() - return token.value - - def _is_at_end_symbol(self, symbol: str | None) -> bool: - token: _ModelHeaderToken = self._peek() - if symbol is None: - return token.kind == _MODEL_HEADER_END_TOKEN - return token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == symbol - - def _match_symbol(self, symbol: str) -> bool: - token: _ModelHeaderToken = self._peek() - if token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == symbol: - self._advance() - return True - return False - - def _consume_symbol(self, symbol: str) -> None: - if self._match_symbol(symbol): - return - token: _ModelHeaderToken = self._peek() - raise ModelHeaderSyntaxError(f"expected '{symbol}' at position {token.position}") - - def _expect_end(self) -> None: - token: _ModelHeaderToken = self._peek() - if token.kind != _MODEL_HEADER_END_TOKEN: - raise ModelHeaderSyntaxError( - f"unexpected token '{token.value}' at position {token.position}" - ) - - def _peek(self) -> _ModelHeaderToken: - return self._tokens[self._index] - - def _advance(self) -> _ModelHeaderToken: - token: _ModelHeaderToken = self._tokens[self._index] - self._index += 1 - return token - - def _model_header_parse(header: str) -> _ModelHeaderTokenization: tokenization: _ModelHeaderTokenization | None = _MODEL_HEADER_TOKEN_CACHE.get(header) if tokenization is None: diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 530fd3a6be..b2c4ea0166 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -77,7 +77,6 @@ class DiscoveredSqlModelFile: query_sql: str enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - output_column_locations_extracted: bool = True extract_implicit_alias_columns: bool = True diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index 2add2ac5c5..7ca63166e8 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -27,25 +27,7 @@ NODE_TYPE_TABLE_FN: str = "table_fn" FUNCTION_NODE_TYPES: tuple[str, str] = (NODE_TYPE_UDF, NODE_TYPE_TABLE_FN) NODE_TYPE_SEED: str = "seed" -NODE_TYPE_LOADER: str = "loader" -NODE_TYPE_TASK: str = "task" -NODE_TYPE_ASSET: str = "asset" -NODE_TYPE_CHECK: str = "check" NODE_TYPE_HOOK: str = "hook" NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" -FINGERPRINT_COLUMNS: tuple[str, ...] = ( - COLUMN_NODE_TYPE, - COLUMN_NODE_NAME, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_SCHEMA, - COLUMN_TARGET_NAME, - COLUMN_RUN_ID, - COLUMN_DEFINITION_HASH, - COLUMN_VERSION_HASH, - COLUMN_SCHEMA_FINGERPRINT, - COLUMN_DEFINITION_B64, - COLUMN_METADATA_JSON_B64, - COLUMN_TIMESTAMP, -) AUDIT_GATE_METADATA_KEY: str = "audit_gate" diff --git a/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py b/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py index ed09b462e4..63b5c8e5e2 100644 --- a/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py +++ b/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py @@ -518,39 +518,6 @@ def _polyglot_column_table_name(column: Any) -> str: return raw_name if isinstance(raw_name, str) else "" -def _polyglot_columns_in_expression(expression: Any) -> tuple[Any, ...]: - if str(getattr(expression, "kind", "")) == POLYGLOT_KIND_COLUMN: - return (expression,) - columns: list[Any] = [] - seen: set[int] = set() - - def visit(*, node: Any, visited: set[int], found: list[Any]) -> tuple[set[int], list[Any]]: - node_id: int = id(node) - if node_id in visited: - return visited, found - visited = visited | {node_id} - if str(getattr(node, "kind", "")) == POLYGLOT_KIND_COLUMN: - return visited, [*found, node] - for child in _polyglot_child_expressions(node): - visited, found = visit(node=child, visited=visited, found=found) - return visited, found - - seen, columns = visit(node=expression, visited=seen, found=columns) - return tuple(columns) - - -def _polyglot_child_expressions(expression: Any) -> tuple[Any, ...]: - children: list[Any] = [] - for attr_name in ("this", "expression", "left", "right"): - child: Any | None = getattr(expression, attr_name, None) - if child is not None and str(getattr(child, "kind", "")): - children.append(child) - for child in getattr(expression, "expressions", ()) or (): - if str(getattr(child, "kind", "")): - children.append(child) - return tuple(children) - - def _polyglot_classify_transform( *, expression: Any, diff --git a/src/sqlbuild/compiler/lineage/models.py b/src/sqlbuild/compiler/lineage/models.py index 9c812a48e9..23b65bd473 100644 --- a/src/sqlbuild/compiler/lineage/models.py +++ b/src/sqlbuild/compiler/lineage/models.py @@ -61,31 +61,6 @@ def as_qualified_column(self) -> QualifiedLineageColumn: ) -@dataclass(frozen=True) -class ColumnLineageNode: - """One internal SQL analysis lineage graph node.""" - - id: str - name: str - expression_sql: str | None = None - source_sql: str | None = None - resource_type: CompiledResourceType | str | None = None - resource_name: str | None = None - scope_name: str | None = None - - def __post_init__(self) -> None: - if self.resource_type is not None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True) -class InternalColumnLineageEdge: - """One edge in the internal SQL analysis lineage graph.""" - - upstream_node_id: str - downstream_node_id: str - - @dataclass(frozen=True) class ColumnLineageEdge: """One collapsed SQLBuild lineage graph edge.""" @@ -118,8 +93,6 @@ class ColumnLineage: expression_sql: str | None upstream_columns: tuple[ColumnLineageSource, ...] nullability: InferredNullability = InferredNullability.UNKNOWN - nodes: tuple[ColumnLineageNode, ...] = field(default_factory=tuple) - edges: tuple[InternalColumnLineageEdge, ...] = field(default_factory=tuple) confidence: ColumnLineageConfidence = ColumnLineageConfidence.UNKNOWN @@ -406,18 +379,6 @@ def edges_targeting(self, model_name: str) -> tuple[ColumnLineageEdge, ...]: for record in self._edge_records_by_target_model.get(model_name, ()) ) - def producing_edge( - self, - *, - model_name: str, - column_name: str, - ) -> ColumnLineageEdge | None: - """Return the first edge that produces `model_name.column_name`.""" - - self._ensure_indexes() - record: object | None = self._edge_record_by_target_column.get((model_name, column_name)) - return self._materialize_edge(record) if record is not None else None - def edges_sourced_from(self, resource_name: str) -> tuple[ColumnLineageEdge, ...]: """Return edges whose source resource is `resource_name`.""" @@ -441,54 +402,6 @@ def column_consumers( if edge.source.column_name == column_name ) - def trace_column( - self, - *, - model_name: str, - column_name: str, - ) -> tuple[ColumnLineageEdge, ...]: - """Trace a model column upstream through project lineage edges.""" - - result: list[ColumnLineageEdge] = [] - stack: list[tuple[str, str]] = [(model_name, column_name)] - visited: set[tuple[str, str]] = set() - - while stack: - current_model, current_column = stack.pop() - if (current_model, current_column) in visited: - continue - visited.add((current_model, current_column)) - for edge in self.edges_targeting(current_model): - if edge.target.column_name == current_column: - result.append(edge) - stack.append((edge.source.resource_name, edge.source.column_name)) - - return tuple(result) - - def trace_column_downstream( - self, - *, - resource_name: str, - column_name: str, - ) -> tuple[ColumnLineageEdge, ...]: - """Trace a resource column downstream through project lineage edges.""" - - result: list[ColumnLineageEdge] = [] - stack: list[tuple[str, str]] = [(resource_name, column_name)] - visited: set[tuple[str, str]] = set() - - while stack: - current_resource, current_column = stack.pop() - if (current_resource, current_column) in visited: - continue - visited.add((current_resource, current_column)) - for edge in self.edges_sourced_from(current_resource): - if edge.source.column_name == current_column: - result.append(edge) - stack.append((edge.target.resource_name, edge.target.column_name)) - - return tuple(result) - @staticmethod def _edge_record_identity(record: object) -> tuple[str, str, str]: if isinstance(record, ColumnLineageEdge): diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py index c3b405e6c4..6da7f7e8f8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/schema.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/schema.py @@ -119,7 +119,9 @@ def detect_schema_changes( kind=SchemaChangeKind.COLUMN_REMOVED, column_name=col_name, source=( - SchemaColumnSource.YML if yml_columns else SchemaColumnSource.SQLGLOT + SchemaColumnSource.YML + if yml_columns + else SchemaColumnSource.SQL_ANALYSIS ), actual_type=col_type, ) @@ -216,7 +218,7 @@ def _compare_inferred_columns( SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name=col.name, - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type=col.type, ) ) @@ -227,7 +229,7 @@ def _compare_inferred_columns( SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name=col.name, - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type=col.type, actual_type=warehouse_map[col.name], ) diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py deleted file mode 100644 index b073ba1f21..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/local_node_planning.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Public local node planning classification entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import LocalNodePlanInput, LocalNodePlanOutcome -from sqlbuild.compiler.planner.types import LocalNodePlanAction, LocalNodePlanReason - - -def classify_local_node_plan(input: LocalNodePlanInput) -> LocalNodePlanOutcome: - """Classify local node state using planner semantics shared by adapters.""" - - if input.full_refresh: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.FULL_REFRESH, - ) - if not input.fingerprint_exists: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.FIRST_RUN, - ) - if not input.relation_exists: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.RELATION_MISSING, - ) - if input.local_hash is not None and input.previous_hash != input.local_hash: - return LocalNodePlanOutcome( - action=LocalNodePlanAction.RUN, - reason=LocalNodePlanReason.LOCAL_CHANGED, - ) - return LocalNodePlanOutcome( - action=LocalNodePlanAction.CURRENT, - reason=LocalNodePlanReason.NO_CHANGE, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py index 6f22ca9682..29ca640fb0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py @@ -29,26 +29,6 @@ def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: ) -def detect_function_backfill( - *, - function: CompiledFunction, - fingerprint_sql: str, - snapshot: WarehouseSnapshot, - query_change_tracking: bool, - full_refresh: bool, -) -> BackfillResult: - """Resolve a function definition change into a cascadeable backfill.""" - - result: tuple[PlanReason, BackfillResult] = detect_function_change( - function=function, - fingerprint_sql=fingerprint_sql, - snapshot=snapshot, - query_change_tracking=query_change_tracking, - full_refresh=full_refresh, - ) - return result[1] - - def detect_function_change( *, function: CompiledFunction, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py b/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py index e79d773568..96a6981e17 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/cursor_type_check.py @@ -190,23 +190,6 @@ def _check_with_heuristic( return None -def _classify_type_heuristic(warehouse_type: str) -> CursorType | None: - """Classify a warehouse type string as timestamp or integer via substrings.""" - - upper: str = warehouse_type.upper() - - substring: str - for substring in _TIMESTAMP_SUBSTRINGS: - if substring in upper: - return CursorType.TIMESTAMP - - for substring in _INTEGER_SUBSTRINGS: - if substring in upper: - return CursorType.INTEGER - - return None - - def classify_cursor_sql_type(sql_type: str) -> CursorType | None: """Classify a supported cross-adapter SQL cursor type.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 84778d33f8..985719ee71 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -851,10 +851,6 @@ def plan_model_from_change( if plan_config.incremental_mode == IncrementalMode.MICROBATCH else get_config_str(values=model.config.values, key="lookback") ), - lookback_is_default=( - plan_config.incremental_mode == IncrementalMode.MICROBATCH - and get_config_str(values=model.config.values, key="lookback") is None - ), cursor_bounds=cursor_bounds, cursor_input_relations=cursor_input_relations, batch_size=plan_config.batch_size, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index 7acdf31041..737f847de7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -2,10 +2,8 @@ from __future__ import annotations -from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( CompiledModel, - CompiledRelationLocation, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -132,47 +130,6 @@ def resolve_schema_actions( return tuple(actions) -def build_logical_ddl( - *, - action: PlanAction, - resolved_sql: str, - target: CompiledRelationLocation, - unique_key: tuple[str, ...], - warehouse_columns: tuple[ColumnInfo, ...], - merge_exclude_columns: tuple[str, ...] = (), -) -> str: - """Generate the logical DDL string for a model plan entry.""" - - qualified_name: str = target.qualified_name or target.name - - if action == PlanAction.CREATE_VIEW: - return f"CREATE OR REPLACE VIEW {qualified_name} AS (\n{resolved_sql}\n)" - - if action == PlanAction.CREATE_TABLE: - return f"CREATE TABLE {qualified_name} AS (\n{resolved_sql}\n)" - - if action == PlanAction.INCREMENTAL_APPEND: - return f"INSERT INTO {qualified_name}\n{resolved_sql}" - - if action == PlanAction.INCREMENTAL_DELETE_INSERT: - return _build_delete_insert_ddl( - qualified_name=qualified_name, - resolved_sql=resolved_sql, - unique_key=unique_key, - ) - - if action == PlanAction.INCREMENTAL_MERGE: - return _build_merge_ddl( - qualified_name=qualified_name, - resolved_sql=resolved_sql, - unique_key=unique_key, - warehouse_columns=warehouse_columns, - merge_exclude_columns=merge_exclude_columns, - ) - - return "" - - def build_model_warnings( *, model_name: str, @@ -211,7 +168,7 @@ def build_model_warnings( ), ) ) - elif finding.source == SchemaColumnSource.SQLGLOT: + elif finding.source == SchemaColumnSource.SQL_ANALYSIS: if finding.kind in ( SchemaChangeKind.COLUMN_ADDED, SchemaChangeKind.COLUMN_TYPE_CHANGED, @@ -393,61 +350,6 @@ def _incremental_action( return action, reason -def _build_delete_insert_ddl( - *, - qualified_name: str, - resolved_sql: str, - unique_key: tuple[str, ...], -) -> str: - """Build logical DDL for delete+insert strategy.""" - - key_list: str = ", ".join(unique_key) - source_key_list: str = ", ".join(unique_key) - delete_stmt: str = ( - f"DELETE FROM {qualified_name}\n" - f"WHERE ({key_list}) IN (SELECT {source_key_list} FROM (\n{resolved_sql}\n))" - ) - insert_stmt: str = f"INSERT INTO {qualified_name}\n{resolved_sql}" - return f"{delete_stmt};\n\n{insert_stmt}" - - -def _build_merge_ddl( - *, - qualified_name: str, - resolved_sql: str, - unique_key: tuple[str, ...], - warehouse_columns: tuple[ColumnInfo, ...], - merge_exclude_columns: tuple[str, ...], -) -> str: - """Build logical DDL for merge/upsert strategy.""" - - on_clause: str = " AND ".join( - f"{_DDL_MERGE_TARGET_ALIAS}.{k} = {_DDL_MERGE_SOURCE_ALIAS}.{k}" for k in unique_key - ) - - immutable_columns: frozenset[str] = frozenset( - column.lower() for column in (*unique_key, *merge_exclude_columns) - ) - non_key_columns: list[str] = [ - col.name for col in warehouse_columns if col.name.lower() not in immutable_columns - ] - all_columns: list[str] = [col.name for col in warehouse_columns] - - update_clause: str = ", ".join( - f"{col} = {_DDL_MERGE_SOURCE_ALIAS}.{col}" for col in non_key_columns - ) - insert_columns: str = ", ".join(all_columns) - insert_values: str = ", ".join(f"{_DDL_MERGE_SOURCE_ALIAS}.{col}" for col in all_columns) - - return ( - f"MERGE INTO {qualified_name} AS {_DDL_MERGE_TARGET_ALIAS}\n" - f"USING (\n{resolved_sql}\n) AS {_DDL_MERGE_SOURCE_ALIAS}\n" - f"ON {on_clause}\n" - f"WHEN MATCHED THEN UPDATE SET {update_clause}\n" - f"WHEN NOT MATCHED THEN INSERT ({insert_columns}) VALUES ({insert_values})" - ) - - def _describe_finding(finding: SchemaFinding) -> str: """Build a human-readable description for one schema finding.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py index ee6d0dbaca..38ddc0e397 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py @@ -335,18 +335,6 @@ def _walk( return requirements -def _dedupe_names(names: tuple[str, ...]) -> tuple[str, ...]: - deduped: list[str] = [] - seen: set[str] = set() - name: str - for name in names: - if name in seen: - continue - seen.add(name) - deduped.append(name) - return tuple(deduped) - - def _fixture_name_for(name: str) -> str: return name.replace(".", "__") diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index b4cdf5513d..8f4f3a586e 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -37,7 +37,6 @@ MODEL_PLACEHOLDERS_CONFIG_KEY: str = "placeholders" MODEL_PRE_HOOKS_CONFIG_KEY: str = "pre_hooks" MODEL_POST_HOOKS_CONFIG_KEY: str = "post_hooks" -MODEL_CONTRACT_CONFIG_KEY: str = "contract" MICROBATCH_START_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.START) MICROBATCH_END_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.END) METADATA_NAME_FILTER_LIMIT: int = 250 @@ -49,7 +48,6 @@ SCENARIO_PLAN_HASH_COLLISION: str = "S502" SCENARIO_PLAN_RELATION_COLLISION: str = "S503" SCENARIO_PLAN_GRAPH_VALIDATION: str = "S504" -SCENARIO_PLAN_SQLGLOT_UNAVAILABLE: str = "S505" SCENARIO_PLAN_SQLGLOT_PARSE: str = "S506" SCENARIO_PLAN_UNKNOWN_SEED: str = "S507" SCENARIO_PLAN_MISSING_FIXTURE_SQL: str = "S508" diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 7121851f5a..b0cb0bba9b 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -45,8 +45,6 @@ CursorWatermarkMode, FixtureKey, GraphResourceKind, - LocalNodePlanAction, - LocalNodePlanReason, MaterializationType, OnSchemaChange, PlanAction, @@ -164,25 +162,6 @@ class SelectorExpansion: downstream: bool = False -@dataclass(frozen=True) -class LocalNodePlanInput: - """Local state used to classify one planner graph node.""" - - fingerprint_exists: bool - relation_exists: bool - full_refresh: bool = False - local_hash: str | None = None - previous_hash: str | None = None - - -@dataclass(frozen=True) -class LocalNodePlanOutcome: - """Local planner action and reason for one graph node.""" - - action: LocalNodePlanAction - reason: LocalNodePlanReason - - @dataclass(frozen=True) class ParsedScenarioArtifactName: """Parsed physical name for one planner-owned scenario artifact.""" @@ -1050,7 +1029,6 @@ class ModelPlanEntry: cursor_start: str | None = None cursor_end: str | None = None lookback: str | None = None - lookback_is_default: bool = False cursor_bounds: CursorBounds | None = None cursor_input_relations: tuple[CursorInputRelation, ...] = field(default_factory=tuple) batch_size: str | None = None diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index 080c37f6ab..5dd189cf1e 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -28,23 +28,6 @@ class CursorInputRole(StrEnum): WATERMARK = "watermark" -class LocalNodePlanAction(StrEnum): - """Action for one locally classified planner graph node.""" - - RUN = "run" - CURRENT = "current" - - -class LocalNodePlanReason(StrEnum): - """Reason for one locally classified planner graph node.""" - - FIRST_RUN = "first_run" - FULL_REFRESH = "full_refresh" - RELATION_MISSING = "relation_missing" - LOCAL_CHANGED = "local_changed" - NO_CHANGE = "no_change" - - class RelationMarkerTargetResolver(Protocol): def __call__(self, *, function_name: str, referenced_name: str) -> str | None: ... @@ -107,14 +90,9 @@ class SchemaChangeKind(StrEnum): COLUMN_TYPE_CHANGED = "column_type_changed" -class SchemaChangeBackfillKey(StrEnum): - ADD_COLUMN = "add_column" - TYPE_CHANGE = "type_change" - - class SchemaColumnSource(StrEnum): YML = "yml" - SQLGLOT = "sql_analysis" + SQL_ANALYSIS = "sql_analysis" class RetentionDirection(StrEnum): diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py index 29ee0a55c6..24edafabb6 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py @@ -79,8 +79,6 @@ def build_python_identity( return PythonNodeIdentity( node_type=node_type, node_name=node_name, - object_module=function.__module__, - object_qualname=_object_qualname(function), source_path=_display_path(source_path=source_path, roots=allowed_roots), source_hash=source_hash, definition_hash=definition_hash, diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py index fbe89b7bd4..69090a97d3 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/inventory.py @@ -123,8 +123,6 @@ def build_python_node_dependency_edges( PythonNodeDependencyEdge( upstream_name=upstream_node.name, downstream_name=node.name, - upstream_function=upstream_node.function, - downstream_function=node.function, ) ) return tuple(edges) @@ -216,6 +214,7 @@ def _build_loader_node( discovered=loader, provider_by_name=provider_by_name, decorator_config={ + "columns": loader.columns, "contract": loader.contract, "cursor_column": loader.cursor_column, "destination": loader.destination, diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/loaders.py b/src/sqlbuild/compiler/python_nodes/_helpers/loaders.py deleted file mode 100644 index b083bd7b94..0000000000 --- a/src/sqlbuild/compiler/python_nodes/_helpers/loaders.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Loader adapters for internal Python-node models.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonLoaderMetadata, - DiscoveredPythonNode, - PythonNodeDependencyEdge, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef - - -def build_python_loader_node(*, loader: DiscoveredLoaderFunction) -> DiscoveredPythonNode: - """Return an internal Python-node view of an existing discovered loader.""" - - return DiscoveredPythonNode( - kind=PythonNodeKind.LOADER, - file_path=loader.file_path, - relative_path=loader.relative_path, - name=loader.name, - function=loader.function, - depends_on=loader.depends_on, - loader=DiscoveredPythonLoaderMetadata( - destination=loader.destination, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - columns=loader.columns, - contract=loader.contract, - connection_mode=loader.connection_mode, - ), - ) - - -def build_python_loader_nodes( - *, loaders: tuple[DiscoveredLoaderFunction, ...] -) -> tuple[DiscoveredPythonNode, ...]: - """Return internal Python-node views of existing discovered loaders.""" - - return tuple(build_python_loader_node(loader=loader) for loader in loaders) - - -def build_python_loader_dependency_edges( - *, loaders: tuple[DiscoveredLoaderFunction, ...] -) -> tuple[PythonNodeDependencyEdge, ...]: - """Return internal dependency edges between existing discovered loaders.""" - - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in loaders - } - edges: list[PythonNodeDependencyEdge] = [] - loader: DiscoveredLoaderFunction - for loader in loaders: - dependency: Callable[..., object] | SqlResourceRef - for dependency in loader.depends_on: - if isinstance(dependency, SqlResourceRef): - continue - upstream_name: str | None = loader_name_by_function.get(dependency) - if upstream_name is None: - continue - edges.append( - PythonNodeDependencyEdge( - upstream_name=upstream_name, - downstream_name=loader.name, - upstream_function=dependency, - downstream_function=loader.function, - ) - ) - return tuple(edges) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py index f541dd0634..1532a3e18d 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/run_lifecycle.py @@ -47,7 +47,6 @@ def build_python_sql_run_lifecycle_plan( return PythonSqlRunLifecyclePlan( ingress_python_node_names=ingress_names, ingress_loader_names=selected_loader_names, - read_side_sql_keys=selection.sql_keys, read_side_python_node_names=read_side_names, ) diff --git a/src/sqlbuild/compiler/python_nodes/models.py b/src/sqlbuild/compiler/python_nodes/models.py index 8bc0a575ef..532d74095f 100644 --- a/src/sqlbuild/compiler/python_nodes/models.py +++ b/src/sqlbuild/compiler/python_nodes/models.py @@ -9,7 +9,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonRunPhase +from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import ColumnLineageRef, RetryPolicy, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.spec.contracts.models import SourceColumnEntry @@ -81,8 +81,6 @@ class PythonNodeDependencyEdge: upstream_name: str downstream_name: str - upstream_function: Callable[..., object] - downstream_function: Callable[..., object] @dataclass(frozen=True) @@ -138,39 +136,8 @@ class PythonSqlRunLifecyclePlan: ingress_python_node_names: frozenset[str] ingress_loader_names: frozenset[str] - read_side_sql_keys: frozenset[CompiledObjectKey] read_side_python_node_names: frozenset[str] - def python_phase(self, node_name: str) -> PythonRunPhase | None: - """Return the assigned lifecycle phase for a Python node name.""" - - if node_name in self.ingress_python_node_names: - return PythonRunPhase.PRE_SQL_INGRESS - if node_name in self.read_side_python_node_names: - return PythonRunPhase.READ_SIDE - return None - - -@dataclass(frozen=True) -class DiscoveredPythonTaskNode: - """Placeholder for future task-specific discovered node metadata.""" - - node: DiscoveredPythonNode - - -@dataclass(frozen=True) -class DiscoveredPythonAssetNode: - """Placeholder for future asset-specific discovered node metadata.""" - - node: DiscoveredPythonNode - - -@dataclass(frozen=True) -class DiscoveredPythonCheckNode: - """Placeholder for future check-specific discovered node metadata.""" - - node: DiscoveredPythonNode - @dataclass(frozen=True) class PythonIdentityDependency: @@ -190,8 +157,6 @@ class PythonNodeIdentity: node_type: str node_name: str - object_module: str - object_qualname: str source_path: str source_hash: str definition_hash: str diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py index 59eccbe38b..f9d2453e32 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ b/src/sqlbuild/compiler/scopes/_helpers/lookup.py @@ -116,16 +116,6 @@ def build_lookup(*, index: ScopeIndex) -> ScopeLookup: key=lambda item: item.resource, sort_key=grant_sort_key, ), - visibility_by_resource=group_records( - records=canonical_index.visibility, - key=lambda item: item.resource, - sort_key=visibility_sort_key, - ), - inaccessible_by_resource=group_records( - records=canonical_index.inaccessible, - key=lambda item: item.resource, - sort_key=inaccessible_sort_key, - ), ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_operations.py b/src/sqlbuild/compiler/scopes/_helpers/report_operations.py index 6cf4c31436..c85de9bc2c 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/report_operations.py +++ b/src/sqlbuild/compiler/scopes/_helpers/report_operations.py @@ -390,12 +390,6 @@ def serialize_result(*, result: ScopeReport | ScopeBrowseResult | ScopeListResul return json.dumps(payload, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + "\n" -def serialize_report(*, report: ScopeReport) -> str: - """Serialize the schema-version-one report payload as deterministic ASCII JSON.""" - - return serialize_result(result=report) - - def _reports( *, lookup: ScopeLookup, diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py index eacc9fa534..0cd2f4b5b5 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py +++ b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Iterable from typing import cast from sqlbuild.compiler.scopes._helpers.lookup import identity_key @@ -378,27 +377,6 @@ def diagnostic_projection(*, diagnostic: ScopeDiagnostic) -> dict[str, object]: } -def records_for_identities( - *, lookup: ScopeLookup, identities: Iterable[DeclarationIdentity] -) -> tuple[DeclarationRecord, ...]: - """Return every known duplicate match in stable identity/location order.""" - - records: list[DeclarationRecord] = [] - for identity in identities: - records.extend(lookup.declarations.get(identity, ())) - return tuple( - sorted( - records, - key=lambda item: ( - identity_key(item.identity), - item.path, - item.line, - item.column, - ), - ) - ) - - def safe_scope_path(*, path: str) -> str: """Return a normalized relative path without leaking invalid absolute input.""" diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index 1f4f6dbd65..0979ca1350 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -38,7 +38,6 @@ SCOPE_ENUM_MEMBER_FIELDS: frozenset[str] = frozenset({"name"}) DEFAULT_ENUM_MEMBER_PREVIEW: int = 20 QUALIFIED_IDENTITY_SEPARATOR: str = ":" -PRIVATE_IDENTITY_SEPARATOR: str = "." PATH_SEPARATOR: str = "/" WINDOWS_PATH_SEPARATOR: str = "\\" CURRENT_PATH_COMPONENT: str = "." diff --git a/src/sqlbuild/compiler/scopes/exceptions.py b/src/sqlbuild/compiler/scopes/exceptions.py index c79d597432..e368b911e8 100644 --- a/src/sqlbuild/compiler/scopes/exceptions.py +++ b/src/sqlbuild/compiler/scopes/exceptions.py @@ -28,12 +28,6 @@ class InvalidQualifiedIdentityError(ScopeError): code: str = "S002" -class DuplicateScopeIdentityError(ScopeError): - """Raised when records duplicate a canonical identity.""" - - code: str = "S003" - - class ScopeCacheDecodeError(ScopeError): """Raised when a persistent scope-index payload is malformed.""" diff --git a/src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py b/src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py deleted file mode 100644 index 256eb057a9..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_normalize_scope_path.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Normalize one project-relative scope path.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.paths import normalize_path - - -def normalize_scope_path(*, path: str | PurePath) -> str: - """Normalize POSIX or Windows input to a project-relative POSIX path.""" - - return normalize_path(path=path) diff --git a/src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py b/src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py deleted file mode 100644 index 302e22be0d..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_parse_qualified_identity.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Parse one qualified scope identity.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes._helpers.identities import parse_identity -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity - - -def parse_qualified_identity(*, value: str) -> ResourceIdentity | DeclarationIdentity: - """Parse only explicit kind-qualified identities; bare names are invalid.""" - - return parse_identity(value=value) diff --git a/src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py b/src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py deleted file mode 100644 index b0f301a499..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_path_is_equal_or_descendant.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Compare normalized project-relative path components.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant - - -def path_is_equal_or_descendant(*, path: str | PurePath, ancestor: str | PurePath) -> bool: - """Compare normalized path components rather than textual prefixes.""" - - return is_equal_or_descendant(path=path, ancestor=ancestor) diff --git a/src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py b/src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py deleted file mode 100644 index a09bdf00c2..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_qualified_identity_text.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Format one qualified scope identity.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes._helpers.identities import format_identity -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity - - -def format_qualified_identity(*, identity: ResourceIdentity | DeclarationIdentity) -> str: - """Format a resource, public declaration, or owner-qualified private declaration.""" - - return format_identity(identity=identity) diff --git a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py b/src/sqlbuild/compiler/scopes/main/_query_scope_target.py deleted file mode 100644 index ba2f9e5702..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_query_scope_target.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Query a scope resource by qualified identity or normalized authored path.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.visibility import query_target -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - ScopeLookup, - ScopeTargetQuery, -) - - -def query_scope_target( - *, - lookup: ScopeLookup, - target: ResourceIdentity | DeclarationIdentity | str | PurePath, -) -> ScopeTargetQuery: - """Return matching resource or declaration records without conflating absence.""" - - return query_target(lookup=lookup, target=target) diff --git a/src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py b/src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py deleted file mode 100644 index 6b6658df2d..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_scope_is_path_visible.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Resolve path-derived lexical scope visibility.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path -from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, PATH_SEPARATOR -from sqlbuild.compiler.scopes.types import ScopeKind - - -def scope_is_path_visible( - *, scope: ScopeKind, owning_path: str | PurePath, resource_path: str | PurePath -) -> bool: - """Resolve path-derived visibility without project discovery or I/O.""" - - if scope is ScopeKind.GLOBAL: - return True - owner: str = normalize_path(path=owning_path) - resource: str = normalize_path(path=resource_path) - resource_parent: str = ( - resource.rsplit(PATH_SEPARATOR, maxsplit=1)[0] - if PATH_SEPARATOR in resource - else CURRENT_PATH_COMPONENT - ) - if scope is ScopeKind.INHERITED: - return is_equal_or_descendant(path=resource_parent, ancestor=owner) - if scope is ScopeKind.LOCAL: - return resource_parent == owner - return False diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index 10ce5c6632..6f46e02a7e 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -219,8 +219,6 @@ class ScopeLookup: usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] - visibility_by_resource: Mapping[ResourceIdentity, tuple[VisibilityRecord, ...]] - inaccessible_by_resource: Mapping[ResourceIdentity, tuple[InaccessibleRecord, ...]] @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py b/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py new file mode 100644 index 0000000000..f42cb54430 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/set_operations.py @@ -0,0 +1,95 @@ +"""Top-level SQL set-operation scanning over the shared code-position iterator.""" + +from __future__ import annotations + +from sqlbuild.compiler.sql_analysis._helpers.scanning import ( + is_identifier_character_impl, + iter_code_positions_impl, + skip_block_comment_impl, + skip_line_comment_impl, +) + +_SET_OPERATOR_KEYWORDS: tuple[str, ...] = ("UNION", "INTERSECT") +_EXCEPT_KEYWORD: str = "EXCEPT" +_ALL_KEYWORD: str = "ALL" +_DISTINCT_KEYWORD: str = "DISTINCT" +_STAR_CHARACTER: str = "*" + + +def split_set_operation_branches_impl(*, sql: str, context: str) -> tuple[str, ...]: + """Split SQL on top-level `UNION`, `INTERSECT` and `EXCEPT` with optional quantifiers.""" + + branches: list[str] = [] + branch_start: int = 0 + resume: int = 0 + previous_code: str | None = None + previous_depth: int = 0 + index: int + depth: int + for index, depth in iter_code_positions_impl(sql=sql, context=context): + if depth != previous_depth: + previous_code = None + previous_depth = depth + operator_end: int | None = ( + None + if index < resume or depth != 0 + else _set_operator_end(sql=sql, start=index, previous_code=previous_code) + ) + if not sql[index].isspace(): + previous_code = sql[index] + if operator_end is None: + continue + branch_sql: str = sql[branch_start:index].strip() + if branch_sql: + branches.append(branch_sql) + resume = _skip_ignorable(sql=sql, start=operator_end, context=context) + quantifier_end: int | None = _keyword_end( + sql=sql, start=resume, keyword=_ALL_KEYWORD + ) or _keyword_end(sql=sql, start=resume, keyword=_DISTINCT_KEYWORD) + if quantifier_end is not None: + resume = _skip_ignorable(sql=sql, start=quantifier_end, context=context) + branch_start = resume + final_branch_sql: str = sql[branch_start:].strip() + if final_branch_sql: + branches.append(final_branch_sql) + return tuple(branches) + + +def _set_operator_end(*, sql: str, start: int, previous_code: str | None) -> int | None: + """Return a set operator's end; `* EXCEPT (...)` star modifiers are not operators.""" + + keyword: str + for keyword in _SET_OPERATOR_KEYWORDS: + keyword_end: int | None = _keyword_end(sql=sql, start=start, keyword=keyword) + if keyword_end is not None: + return keyword_end + if previous_code == _STAR_CHARACTER: + return None + return _keyword_end(sql=sql, start=start, keyword=_EXCEPT_KEYWORD) + + +def _keyword_end(*, sql: str, start: int, keyword: str) -> int | None: + keyword_end: int = start + len(keyword) + if sql[start:keyword_end].upper() != keyword: + return None + if keyword_end < len(sql) and is_identifier_character_impl(sql[keyword_end]): + return None + if start > 0 and is_identifier_character_impl(sql[start - 1]): + return None + return keyword_end + + +def _skip_ignorable(*, sql: str, start: int, context: str) -> int: + index: int = start + while index < len(sql): + if sql[index].isspace(): + index += 1 + continue + if sql.startswith("--", index): + index = skip_line_comment_impl(sql=sql, start=index) + continue + if sql.startswith("/*", index): + index = skip_block_comment_impl(sql=sql, start=index, context=context) + continue + return index + return index diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 74b456c87a..6724ecbc59 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -134,7 +134,6 @@ POLYGLOT_ANALYSIS_COLUMN_USES: str = "columnUses" POLYGLOT_ANALYSIS_CONTEXT: str = "context" POLYGLOT_ANALYSIS_CONTEXT_FILTER: str = "filter" -POLYGLOT_ANALYSIS_CTES: str = "ctes" POLYGLOT_ANALYSIS_CTE_FACTS: str = "cteFacts" POLYGLOT_ANALYSIS_EXPRESSION_SQL: str = "expressionSql" POLYGLOT_ANALYSIS_IS_STAR: str = "isStar" @@ -142,10 +141,8 @@ POLYGLOT_ANALYSIS_NULLABILITY: str = "nullability" POLYGLOT_ANALYSIS_NULLABILITY_NON_NULL: str = "non_null" POLYGLOT_ANALYSIS_NULLABILITY_NULLABLE: str = "nullable" -POLYGLOT_ANALYSIS_NULLABILITY_UNKNOWN: str = "unknown" POLYGLOT_ANALYSIS_PROJECTIONS: str = "projections" POLYGLOT_ANALYSIS_RELATIONS: str = "relations" -POLYGLOT_ANALYSIS_SET_OPERATIONS: str = "setOperations" POLYGLOT_ANALYSIS_SHAPE: str = "shape" POLYGLOT_ANALYSIS_SOURCE_ALIAS: str = "sourceAlias" POLYGLOT_ANALYSIS_SOURCE_CONFIDENCE: str = "confidence" @@ -163,6 +160,5 @@ POLYGLOT_ANALYSIS_TRANSFORM_CAST: str = "cast" POLYGLOT_ANALYSIS_TRANSFORM_CONSTANT: str = "constant" POLYGLOT_ANALYSIS_TRANSFORM_DIRECT: str = "direct" -POLYGLOT_ANALYSIS_TRANSFORM_EXPRESSION: str = "expression" POLYGLOT_ANALYSIS_TRANSFORM_STAR: str = "star" POLYGLOT_ANALYSIS_UNSAFE_TRANSFORMS: frozenset[str] = frozenset() diff --git a/src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py b/src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py new file mode 100644 index 0000000000..b0d1ad4714 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_split_set_operation_branches.py @@ -0,0 +1,11 @@ +"""Top-level SQL set-operation branch splitting entrypoint.""" + +from sqlbuild.compiler.sql_analysis._helpers.set_operations import ( + split_set_operation_branches_impl, +) + + +def split_set_operation_branches(*, sql: str, context: str = "SQL") -> tuple[str, ...]: + """Split SQL on top-level `UNION`, `INTERSECT` and `EXCEPT`, ignoring quotes and comments.""" + + return split_set_operation_branches_impl(sql=sql, context=context) diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py index 5edf101319..c426667edb 100644 --- a/src/sqlbuild/cost/constants.py +++ b/src/sqlbuild/cost/constants.py @@ -15,6 +15,5 @@ NO_WAREHOUSE_COMPUTE_REASON: str = "no_warehouse_compute" MISSING_WAREHOUSE_METADATA_REASON: str = "missing_warehouse_metadata" INCOMPLETE_HISTORY_REASON: str = "incomplete_history" -OUTSIDE_RUN_WINDOW_REASON: str = "outside_run_window" LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX: str = "Statement-ledger persistence failed (" COST_TELEMETRY_HEALTH: CostTelemetryHealth = CostTelemetryHealth() diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index aa59274c65..cd243f2ef6 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -38,16 +38,6 @@ DURATION_MINUTE_UNIT: str = "m" DURATION_SECOND_UNIT: str = "s" -DURATION_UNITS: frozenset[str] = frozenset( - { - DURATION_YEAR_UNIT, - DURATION_MONTH_UNIT, - DURATION_DAY_UNIT, - DURATION_HOUR_UNIT, - DURATION_MINUTE_UNIT, - DURATION_SECOND_UNIT, - } -) MINUTE_TO_DAY_DURATION_UNITS: frozenset[str] = frozenset( {DURATION_MINUTE_UNIT, DURATION_HOUR_UNIT, DURATION_DAY_UNIT} ) diff --git a/src/sqlbuild/cursor_algebra/types.py b/src/sqlbuild/cursor_algebra/types.py index 8f6f680ce7..6d86a5f165 100644 --- a/src/sqlbuild/cursor_algebra/types.py +++ b/src/sqlbuild/cursor_algebra/types.py @@ -2,10 +2,8 @@ from enum import StrEnum -from sqlbuild.compiler.planner.types import CursorGrain from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -type Grain = CursorGrain type CursorScalar = TimestampValue | DateValue | IntegerValue diff --git a/src/sqlbuild/diagnostics/_helpers/logging.py b/src/sqlbuild/diagnostics/_helpers/logging.py index 27407a4341..a6ae2cab03 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging.py +++ b/src/sqlbuild/diagnostics/_helpers/logging.py @@ -34,14 +34,6 @@ def _current_identity_context() -> dict[str, object]: return {f"sqlbuild_{key}": value for key, value in execution_identity_to_dict(identity).items()} -def get_diagnostics_logger(name: str | None = None) -> logging.Logger: - """Return a SQLBuild diagnostics logger.""" - - if name is None: - return logging.getLogger(_LOGGER_ROOT_NAME) - return logging.getLogger(f"{_LOGGER_ROOT_NAME}.{name}") - - def log_sql( *, logger: logging.Logger, diff --git a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py index f155a69aa1..39a986cf75 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py +++ b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py @@ -11,12 +11,6 @@ _SQL_EVENT_FIELD: str = "sqlbuild_sql" -def _format_prefix(*, record: logging.LogRecord, date_text: str) -> str: - return ( - f"{date_text}.{int(record.msecs):03d} {record.levelname} {_short_logger_name(record.name)}" - ) - - def _short_logger_name(name: str) -> str: prefix: str = f"{_LOGGER_ROOT_NAME}." if name.startswith(prefix): diff --git a/src/sqlbuild/execution_history.py b/src/sqlbuild/execution_history.py index aaf1aa1c78..39f5217e07 100644 --- a/src/sqlbuild/execution_history.py +++ b/src/sqlbuild/execution_history.py @@ -1,4 +1,4 @@ -"""Typed public execution history API with a deprecated ``EventLogStorage`` alias.""" +"""Typed public execution history API.""" from collections.abc import Iterable @@ -44,8 +44,6 @@ RunStorage, ) -EventLogStorage: type[LifecycleEventLogStorage] = LifecycleEventLogStorage - __all__ = ( "DEFAULT_PAGE_LIMIT", "MAX_PAGE_LIMIT", @@ -53,7 +51,6 @@ "EventCursor", "EventFamily", "EventFilter", - "EventLogStorage", "LifecycleEventLogStorage", "EventPage", "ExecutionHistoryStorageError", diff --git a/src/sqlbuild/executor/build/_helpers/output.py b/src/sqlbuild/executor/build/_helpers/output.py deleted file mode 100644 index 2ffdfdd146..0000000000 --- a/src/sqlbuild/executor/build/_helpers/output.py +++ /dev/null @@ -1,861 +0,0 @@ -"""Build execution output formatting.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome, AuditRunScope -from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display -from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end -from sqlbuild.compiler.planner.main.execution.model_execution_annotation import ( - model_execution_annotation, -) -from sqlbuild.compiler.planner.main.execution.model_resource_type import model_resource_type -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.aligned_name_value import format_aligned_name_value -from sqlbuild.presentation.main.coded_error_text import format_coded_error -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.resolve_name_column_width import resolve_name_column_width -from sqlbuild.presentation.main.status_cell import format_status_cell -from sqlbuild.presentation.main.summary_footer import format_summary_footer -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.types import CompletionState - - -@dataclass(frozen=True) -class _AuditDisplayEntry: - """Aggregated audit result for display.""" - - label: str - display_name: str - outcome: AuditOutcome - total_row_count: int - batch_pass: int - batch_total: int - executed_sql: str | None = None - - -def format_build_output( - *, - result: BuildExecutionResult, - plan: PlanOutput, - target: str | None = None, - concurrency: int = 1, - elapsed_seconds: float = 0.0, - use_color: bool = False, - verbose: bool = False, -) -> str: - """Format the complete build execution output.""" - - model_entry_map: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} - test_results_by_model: dict[str, SqlTestExecutionResult] = _build_test_results_by_model( - result.test_results - ) - lines: list[str] = [] - lines.append(_format_header(target=target, concurrency=concurrency, use_color=use_color)) - lines.append("") - - counter: int = 0 - total: int = _count_top_level_nodes(result) - top_level_name_width: int = _top_level_name_width(result=result, plan=plan) - sub_name_width: int = _sub_name_width(result) - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - counter += 1 - lines.append( - _format_seed_line( - seed_result=seed_result, - counter=counter, - total=total, - use_color=use_color, - name_width=top_level_name_width, - ) - ) - - model_result: ModelExecutionResult - for model_result in result.model_results: - counter += 1 - plan_entry: ModelPlanEntry | None = model_entry_map.get(model_result.model_name) - resource_type: str = _resolve_resource_type(plan_entry) - annotation: str = _resolve_annotation(plan_entry) - - lines.append( - _format_model_line( - model_result=model_result, - resource_type=resource_type, - annotation=annotation, - counter=counter, - total=total, - use_color=use_color, - name_width=top_level_name_width, - ) - ) - - batch_line: str | None = _format_batch_summary_line( - model_result=model_result, use_color=use_color - ) - if batch_line is not None: - lines.append(batch_line) - - if verbose: - event: LifeCycleEvent - for event in _resolve_verbose_events( - model_result=model_result, - plan_entry=plan_entry, - ): - if event.kind == LifeCycleEventKind.SQL: - lines.extend(_format_sql_block(event.content)) - elif event.kind == LifeCycleEventKind.LOG: - lines.extend(_format_log_block(message=event.content, use_color=use_color)) - - test_result: SqlTestExecutionResult | None = test_results_by_model.get( - model_result.model_name - ) - if test_result is not None: - lines.append( - _format_sub_line( - sub_type="test", - name=test_result.test_name, - status=_test_outcome_to_status(test_result.outcome), - use_color=use_color, - name_width=sub_name_width, - ) - ) - step_result: StepResult - for step_result in test_result.step_results: - lines.append( - _format_test_expectation_sub_line( - step_result=step_result, - use_color=use_color, - name_width=sub_name_width, - ) - ) - - audit_entries: list[_AuditDisplayEntry] = _aggregate_audit_results( - model_result.audit_results - ) - audit_entry: _AuditDisplayEntry - for audit_entry in audit_entries: - lines.append( - _format_audit_sub_line( - entry=audit_entry, - use_color=use_color, - name_width=sub_name_width, - ) - ) - if verbose and audit_entry.executed_sql is not None: - lines.extend(_format_sql_block(audit_entry.executed_sql)) - - lines.append("") - lines.append( - _format_completion_message( - status=result.status, - warning_count=result.warning_count, - use_color=use_color, - summary=_format_summary_counts(result=result, elapsed_seconds=elapsed_seconds), - ) - ) - - failure_lines: list[str] = _format_failure_details(result=result, use_color=use_color) - if failure_lines: - lines.append("") - lines.extend(failure_lines) - - warning_lines: list[str] = _format_warning_details(result) - if warning_lines: - lines.append("") - lines.extend(warning_lines) - - return "\n".join(lines) - - -def _format_header(*, target: str | None, concurrency: int, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - context_parts: list[str] = [] - if target is not None: - context_parts.append(f"target: {target}") - context_parts.append(f"concurrency: {concurrency}") - context: str = f"sqb build ({', '.join(context_parts)})" - return format_surface_header(style=style, title="Execution", context=context) - - -def _format_seed_line( - *, - seed_result: SeedExecutionResult, - counter: int, - total: int, - use_color: bool, - name_width: int, -) -> str: - counter_str: str = f"{counter}/{total}".rjust(len(str(total)) * 2 + 1) - status: str = _execution_status_to_display(seed_result.status) - style: CliStyle = CliStyle(use_color=use_color) - status_cell: str = format_status_cell(style=style, status=status) - duration: str = _format_duration(seed_result.duration_ms) - return format_aligned_name_value( - plain_name=seed_result.seed_name, - styled_name=seed_result.seed_name, - value=f"{status_cell} {style.muted(duration) if duration else ''}", - name_column_width=name_width, - prefix=f" {counter_str} {style.muted('seed ')} ", - ) - - -def _format_model_line( - *, - model_result: ModelExecutionResult, - resource_type: str, - annotation: str, - counter: int, - total: int, - use_color: bool, - name_width: int, -) -> str: - counter_str: str = f"{counter}/{total}".rjust(len(str(total)) * 2 + 1) - name_and_annotation: str = model_result.model_name - if annotation: - name_and_annotation = f"{model_result.model_name} ({annotation})" - - status: str = _execution_status_to_display(model_result.status) - style: CliStyle = CliStyle(use_color=use_color) - status_cell: str = format_status_cell(style=style, status=status) - duration: str = _format_duration(model_result.duration_ms) - detail: str = "" - if model_result.status == ExecutionStatus.FAILED and model_result.failed_phase is not None: - detail = f" {model_result.failed_phase}" - elif model_result.status == ExecutionStatus.SKIPPED: - duration = "" - detail = _model_skip_detail(model_result) - rendered_duration: str = style.muted(duration) if duration else "" - line: str = format_aligned_name_value( - plain_name=name_and_annotation, - styled_name=name_and_annotation, - value=f"{status_cell} {rendered_duration}{detail}", - name_column_width=name_width, - prefix=f" {counter_str} {style.muted(f'{resource_type:<6}')} ", - ) - return line - - -def _format_sql_block(sql: str) -> list[str]: - """Format a SQL block with minimal indent for verbose output.""" - - lines: list[str] = [""] - sql_line: str - for sql_line in _format_display_sql(sql).split("\n"): - lines.append(f" {sql_line}") - lines.append("") - return lines - - -def _format_display_sql(sql: str) -> str: - stripped: str = sql.rstrip() - if not stripped: - return sql - if stripped.endswith(";"): - return stripped - return f"{stripped};" - - -def _resolve_verbose_events( - *, model_result: ModelExecutionResult, plan_entry: ModelPlanEntry | None -) -> tuple[LifeCycleEvent, ...]: - if model_result.lifecycle_events: - return model_result.lifecycle_events - if plan_entry is not None: - return (LifeCycleEvent(kind=LifeCycleEventKind.SQL, content=plan_entry.logical_ddl),) - return () - - -def _format_log_block(*, message: str, use_color: bool) -> list[str]: - """Format a log message with indent for verbose output.""" - - line: str = f" log {message}" - style: CliStyle = CliStyle(use_color=use_color) - line = style.log_label(line) - return ["", line, ""] - - -def _format_batch_summary_line( - *, model_result: ModelExecutionResult, use_color: bool -) -> str | None: - """Return the muted microbatch summary line, if any.""" - - if model_result.batch_count is None: - return None - batch_label: str = "batch" if model_result.batch_count == 1 else "batches" - count_text: str = f"{model_result.batch_count} {batch_label}" - if model_result.batch_size is not None: - count_text = f"{count_text} ({model_result.batch_size})" - parts: list[str] = [count_text] - range_text: str | None = _format_batch_range(model_result=model_result) - if range_text is not None: - parts.append(f"range {range_text}") - if model_result.rows_affected is not None: - parts.append(_format_abbreviated_row_count(count=model_result.rows_affected)) - line: str = f" {' '.join(parts)}" - style: CliStyle = CliStyle(use_color=use_color) - return style.muted(line) - - -def _format_abbreviated_row_count(*, count: int) -> str: - """Format a row count with K/M/B abbreviation above 10,000.""" - - row_label: str = "row" if count == 1 else "rows" - _BILLION: int = 1_000_000_000 - _MILLION: int = 1_000_000 - _THOUSAND: int = 1_000 - _ABBREVIATION_THRESHOLD: int = 10_000 - if count >= _BILLION: - return f"{count / _BILLION:.1f}B {row_label}" - if count >= _MILLION: - return f"{count / _MILLION:.1f}M {row_label}" - if count >= _ABBREVIATION_THRESHOLD: - return f"{count / _THOUSAND:.1f}K {row_label}" - return f"{count:,} {row_label}" - - -def _format_batch_range(*, model_result: ModelExecutionResult) -> str | None: - """Return the inclusive cursor range for a microbatch model, if resolvable.""" - - if model_result.cursor_range_start is None or model_result.cursor_range_end is None: - return None - start: str = cursor_bound_display( - value=model_result.cursor_range_start, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - inclusive_end: str = inclusive_cursor_end( - end=model_result.cursor_range_end, - cursor_type=model_result.cursor_type, - cursor_grain=model_result.cursor_grain, - ) - return f"{start} \u2192 {inclusive_end}" - - -def _format_sub_line( - *, sub_type: str, name: str, status: str, use_color: bool, name_width: int -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - colored_status: str = style.status(status=status) - padding: str = " " * 10 - return format_aligned_name_value( - plain_name=name, - styled_name=name, - value=colored_status, - name_column_width=name_width, - prefix=f"{padding} {style.muted(f'{sub_type:<6}')} ", - ) - - -def _format_audit_sub_line(*, entry: _AuditDisplayEntry, use_color: bool, name_width: int) -> str: - status: str = _audit_outcome_to_display(entry.outcome) - detail: str = "" - if entry.outcome != AuditOutcome.PASS and entry.total_row_count > 0: - row_label: str = "row" if entry.total_row_count == 1 else "rows" - detail = f" {entry.total_row_count} {row_label}" - if entry.batch_total > 1: - detail = f" {entry.batch_pass}/{entry.batch_total}" + detail - return _format_sub_line( - sub_type=entry.label, - name=entry.display_name, - status=f"{status}{detail}", - use_color=use_color, - name_width=name_width, - ) - - -def _format_test_expectation_sub_line( - *, step_result: StepResult, use_color: bool, name_width: int -) -> str: - status: str = _test_outcome_to_status(step_result.outcome) - detail: str = "" - if step_result.outcome != SqlTestOutcome.PASS: - if step_result.model_name.startswith("assertion "): - row_label: str = "row" if step_result.actual_row_count == 1 else "rows" - detail = f" {step_result.actual_row_count} {row_label}" - else: - detail = f" {step_result.mismatched_row_count} mismatched" - style: CliStyle = CliStyle(use_color=use_color) - colored_status: str = style.status(status=status, text=f"{status}{detail}") - name: str = _format_test_expectation_name(step_result.model_name) - return format_aligned_name_value( - plain_name=name, - styled_name=name, - value=colored_status, - name_column_width=name_width, - prefix=f"{'':>14}{style.muted(f'{"expect":<6}')} ", - ) - - -def _format_test_expectation_name(model_name: str) -> str: - if model_name.startswith("assertion "): - return model_name - return f"expected {model_name}" - - -def _format_completion_message( - *, status: BuildStatus, warning_count: int, use_color: bool, summary: str | None = None -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - if status == BuildStatus.FAILED: - return format_completion_line( - style=style, state=CompletionState.FAIL, label="Completed with errors", summary=summary - ) - if warning_count > 0: - return format_completion_line( - style=style, - state=CompletionState.WARN, - label="Completed with warnings", - summary=summary, - ) - return format_completion_line( - style=style, state=CompletionState.OK, label="Completed successfully", summary=summary - ) - - -def _top_level_name_width(*, result: BuildExecutionResult, plan: PlanOutput) -> int: - model_entry_map: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} - names: list[str] = [seed_result.seed_name for seed_result in result.seed_results] - names.extend(function_result.function_name for function_result in result.function_results) - model_result: ModelExecutionResult - for model_result in result.model_results: - plan_entry: ModelPlanEntry | None = model_entry_map.get(model_result.model_name) - annotation: str = _resolve_annotation(plan_entry) - if annotation: - names.append(f"{model_result.model_name} ({annotation})") - else: - names.append(model_result.model_name) - return resolve_name_column_width(names=names) - - -def _sub_name_width(result: BuildExecutionResult) -> int: - names: list[str] = [] - test_result: SqlTestExecutionResult - for test_result in result.test_results: - names.append(test_result.test_name) - step_result: StepResult - for step_result in test_result.step_results: - names.append(_format_test_expectation_name(step_result.model_name)) - model_result: ModelExecutionResult - for model_result in result.model_results: - audit_entry: _AuditDisplayEntry - for audit_entry in _aggregate_audit_results(model_result.audit_results): - names.append(audit_entry.display_name) - return resolve_name_column_width(names=names) - - -def _format_summary_counts( - *, - result: BuildExecutionResult, - elapsed_seconds: float, -) -> str: - pass_count: int = 0 - warn_count: int = 0 - fail_count: int = 0 - skip_count: int = 0 - insufficient_count: int = 0 - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif model_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif model_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - audit_result: AuditExecutionResult - for audit_result in model_result.audit_results: - if audit_result.outcome == AuditOutcome.PASS: - pass_count += 1 - elif audit_result.outcome == AuditOutcome.WARN: - warn_count += 1 - elif audit_result.outcome == AuditOutcome.ERROR: - fail_count += 1 - elif audit_result.outcome == AuditOutcome.INSUFFICIENT: - insufficient_count += 1 - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif seed_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif seed_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status == ExecutionStatus.SUCCESS: - pass_count += 1 - elif function_result.status == ExecutionStatus.FAILED: - fail_count += 1 - elif function_result.status == ExecutionStatus.SKIPPED: - skip_count += 1 - warn_count += len(function_result.warning_messages) - - test_result: SqlTestExecutionResult - for test_result in result.test_results: - if test_result.outcome == SqlTestOutcome.PASS: - pass_count += 1 - else: - fail_count += 1 - - end_audit: AuditExecutionResult - for end_audit in result.end_audit_results: - if end_audit.outcome == AuditOutcome.PASS: - pass_count += 1 - elif end_audit.outcome == AuditOutcome.WARN: - warn_count += 1 - elif end_audit.outcome == AuditOutcome.ERROR: - fail_count += 1 - elif end_audit.outcome == AuditOutcome.INSUFFICIENT: - insufficient_count += 1 - - source_audit: AuditExecutionResult - for source_audit in result.source_audit_results: - if source_audit.outcome == AuditOutcome.PASS: - pass_count += 1 - elif source_audit.outcome == AuditOutcome.WARN: - warn_count += 1 - elif source_audit.outcome == AuditOutcome.ERROR: - fail_count += 1 - elif source_audit.outcome == AuditOutcome.INSUFFICIENT: - insufficient_count += 1 - - total: int = pass_count + warn_count + fail_count + skip_count + insufficient_count - elapsed_str: str = f"{elapsed_seconds:.2f}s" if elapsed_seconds > 0 else "" - return format_summary_footer( - counts=( - ("PASS", pass_count), - ("WARN", warn_count), - ("FAIL", fail_count), - ("INSUFFICIENT", insufficient_count), - ("SKIP", skip_count), - ("TOTAL", total), - ), - use_color=False, - elapsed=elapsed_str or None, - ) - - -def _format_failure_details(*, result: BuildExecutionResult, use_color: bool) -> list[str]: - lines: list[str] = [] - has_failures: bool = False - - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - if seed_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - lines.append(f" {seed_result.seed_name} (seed)") - if seed_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=seed_result.error_code, - error_message=seed_result.error_message, - error_help=seed_result.error_help, - use_color=use_color, - ) - ) - lines.append("") - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if function_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - lines.append(f" {function_result.function_name} ({function_result.function_kind})") - if function_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=function_result.error_code, - error_message=function_result.error_message, - error_help=function_result.error_help, - use_color=use_color, - ) - ) - lines.append("") - - model_result: ModelExecutionResult - for model_result in result.model_results: - if model_result.status != ExecutionStatus.FAILED: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - phase_str: str = f" ({model_result.failed_phase})" if model_result.failed_phase else "" - lines.append(f" {model_result.model_name}{phase_str}") - if model_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=model_result.error_code, - error_message=model_result.error_message, - error_help=model_result.error_help, - use_color=use_color, - ) - ) - if model_result.staging_relation is not None: - lines.append(f" {_inspection_relation_message(model_result.staging_relation)}") - lines.append("") - - test_result: SqlTestExecutionResult - for test_result in result.test_results: - if test_result.outcome == SqlTestOutcome.PASS: - continue - if not has_failures: - lines.append("Failures:") - lines.append("") - has_failures = True - lines.append(f" {test_result.test_name} (test)") - if test_result.error_message is not None: - lines.extend( - _format_result_error_lines( - error_code=test_result.error_code, - error_message=test_result.error_message, - error_help=test_result.error_help, - use_color=use_color, - ) - ) - lines.append("") - - return lines - - -def _format_result_error_lines( - *, error_code: str | None, error_message: str, error_help: str | None, use_color: bool -) -> list[str]: - rendered_error: str = ( - error_message - if error_code is None - else format_coded_error( - code=error_code, - message=error_message, - help=error_help, - use_color=use_color, - ) - ) - return [f" {line}" for line in rendered_error.splitlines()] - - -def _format_warning_details(result: BuildExecutionResult) -> list[str]: - lines: list[str] = [] - has_warnings: bool = False - - model_result: ModelExecutionResult - for model_result in result.model_results: - model_warnings: list[str] = [] - audit_result: AuditExecutionResult - for audit_result in model_result.audit_results: - if audit_result.outcome == AuditOutcome.WARN: - name: str = audit_result.audit_name - if audit_result.attached_column_name: - name = f"{audit_result.audit_name} ({audit_result.attached_column_name})" - if audit_result.evaluation_mode == AuditEvaluationMode.MEASUREMENT: - model_warnings.append( - f" audit {name} measured {audit_result.measured_value}" - ) - else: - row_label: str = "row" if audit_result.row_count == 1 else "rows" - model_warnings.append( - f" audit {name} returned {audit_result.row_count} {row_label}" - ) - warning_msg: str - for warning_msg in model_result.warning_messages: - model_warnings.append(f" {warning_msg}") - if model_warnings: - if not has_warnings: - lines.append("Warnings:") - lines.append("") - has_warnings = True - lines.append(f" {model_result.model_name}") - lines.extend(model_warnings) - lines.append("") - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if not function_result.warning_messages: - continue - if not has_warnings: - lines.append("Warnings:") - lines.append("") - has_warnings = True - lines.append(f" {function_result.function_name} ({function_result.function_kind})") - warning_msg: str - for warning_msg in function_result.warning_messages: - lines.append(f" {warning_msg}") - lines.append("") - - return lines - - -def _inspection_relation_message(relation_name: str) -> str: - if relation_name.endswith("__delta"): - return f"delta table kept for inspection: {relation_name}" - return f"staging table kept for inspection: {relation_name}" - - -def _count_top_level_nodes(result: BuildExecutionResult) -> int: - return len(result.model_results) + len(result.seed_results) - - -def _build_test_results_by_model( - test_results: tuple[SqlTestExecutionResult, ...], -) -> dict[str, SqlTestExecutionResult]: - result_map: dict[str, SqlTestExecutionResult] = {} - test_result: SqlTestExecutionResult - for test_result in test_results: - step: object - for step in test_result.step_results: - if hasattr(step, "model_name"): - result_map[step.model_name] = test_result - return result_map - - -def _resolve_resource_type(plan_entry: ModelPlanEntry | None) -> str: - return model_resource_type(plan_entry) - - -def _resolve_annotation(plan_entry: ModelPlanEntry | None) -> str: - return model_execution_annotation(plan_entry) - - -def _format_duration(duration_ms: int | None) -> str: - if duration_ms is None: - return "" - seconds: float = duration_ms / 1000.0 - seconds_per_minute: int = 60 - if seconds < seconds_per_minute: - return f"{seconds:.2f}s" - minutes: int = int(seconds // 60) - remaining: float = seconds - minutes * 60 - return f"{minutes}m{remaining:.1f}s" - - -def _execution_status_to_display(status: ExecutionStatus) -> str: - if status == ExecutionStatus.SUCCESS: - return "OK" - if status == ExecutionStatus.FAILED: - return "FAIL" - if status == ExecutionStatus.SKIPPED: - return "SKIP" - return str(status) - - -def _model_skip_detail(result: ModelExecutionResult) -> str: - skip_label: str = "skip" - if result.skip_mode is not None: - skip_label = f"{result.skip_mode.value} skip" - if result.skip_reason: - return f" {skip_label}: {result.skip_reason}" - return f" {skip_label}" - - -def _aggregate_audit_results( - audit_results: tuple[AuditExecutionResult, ...], -) -> list[_AuditDisplayEntry]: - """Aggregate per-batch delta audit results into single display entries.""" - - has_delta: bool = any(a.run_scope_phase == AuditRunScope.DELTA_AND_FINAL for a in audit_results) - - groups: dict[tuple[str, str | None, str], list[AuditExecutionResult]] = {} - audit: AuditExecutionResult - for audit in audit_results: - key: tuple[str, str | None, str] = ( - audit.audit_name, - audit.attached_column_name, - audit.run_scope_phase, - ) - groups.setdefault(key, []).append(audit) - - entries: list[_AuditDisplayEntry] = [] - results: list[AuditExecutionResult] - for (_name, _col, _phase), results in groups.items(): - display_name: str = _name - if _col is not None: - display_name = f"{_name} ({_col})" - worst: AuditOutcome = _worst_audit_outcome(results) - total_rows: int = sum(r.row_count for r in results) - pass_count: int = sum(1 for r in results if r.outcome == AuditOutcome.PASS) - label: str = _phase_label(phase=_phase, has_delta_audits=has_delta) - entries.append( - _AuditDisplayEntry( - label=label, - display_name=display_name, - outcome=worst, - total_row_count=total_rows, - batch_pass=pass_count, - batch_total=len(results), - executed_sql=results[0].executed_sql if results else None, - ) - ) - - return entries - - -def _worst_audit_outcome(results: list[AuditExecutionResult]) -> AuditOutcome: - """Return the worst outcome from a list of audit results.""" - - has_error: bool = any(r.outcome == AuditOutcome.ERROR for r in results) - if has_error: - return AuditOutcome.ERROR - has_warn: bool = any(r.outcome == AuditOutcome.WARN for r in results) - if has_warn: - return AuditOutcome.WARN - if any(r.outcome == AuditOutcome.INSUFFICIENT for r in results): - return AuditOutcome.INSUFFICIENT - return AuditOutcome.PASS - - -def _phase_label(*, phase: str, has_delta_audits: bool) -> str: - """Return the audit type label, annotated with phase when delta audits are present.""" - - if not has_delta_audits: - return "audit" - if phase == AuditRunScope.DELTA_AND_FINAL: - return "audit (d)" - return "audit (f)" - - -def _audit_outcome_to_display(outcome: AuditOutcome) -> str: - if outcome == AuditOutcome.PASS: - return "PASS" - if outcome == AuditOutcome.WARN: - return "WARN" - if outcome == AuditOutcome.ERROR: - return "FAIL" - if outcome == AuditOutcome.INSUFFICIENT: - return "INSUFFICIENT" - return str(outcome) - - -def _test_outcome_to_status(outcome: SqlTestOutcome) -> str: - if outcome == SqlTestOutcome.PASS: - return "PASS" - if outcome == SqlTestOutcome.FAIL: - return "FAIL" - if outcome == SqlTestOutcome.ERROR: - return "FAIL" - return str(outcome) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index f566f4adc2..51fd5091d0 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -11,7 +11,6 @@ from collections.abc import Callable, Mapping from concurrent.futures import FIRST_COMPLETED, CancelledError, Future, ThreadPoolExecutor, wait from contextvars import copy_context -from datetime import datetime from functools import partial from pathlib import Path from typing import Any, cast @@ -203,11 +202,6 @@ def __init__( loader_functions=customizations.loader_functions, ).loader_ref_entries ) - self._loader_is_reload: bool = runtime.loader_is_reload - self._start_cursor_ts: datetime | None = runtime.start_cursor_ts - self._end_cursor_ts: datetime | None = runtime.end_cursor_ts - self._start_cursor_int: int | None = runtime.start_cursor_int - self._end_cursor_int: int | None = runtime.end_cursor_int self._target: str = runtime.target self._effective_vars: dict[str, object] = runtime.effective_vars or {} self._warehouse_relations: dict[str, RelationInfo] = initial_state.warehouse_relations or {} diff --git a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py index fb089cd1d4..5276e7a13f 100644 --- a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py +++ b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py @@ -123,24 +123,6 @@ def write_prephase_header(*, stream: TextIO, title: str, use_color: bool) -> Non stream.flush() -def write_prephase_rows( - *, stream: TextIO, rows: tuple[PrephaseProgressRow, ...], use_color: bool -) -> None: - if not rows: - return - total: int = len(rows) - row: PrephaseProgressRow - for index, row in enumerate(rows, start=1): - write_prephase_row( - stream=stream, - row=row, - index=index, - total=total, - use_color=use_color, - ) - stream.flush() - - def write_prephase_row( *, stream: TextIO, row: PrephaseProgressRow, index: int, total: int, use_color: bool ) -> None: diff --git a/src/sqlbuild/executor/clone/types.py b/src/sqlbuild/executor/clone/types.py index 7f498ac274..9f30a97f63 100644 --- a/src/sqlbuild/executor/clone/types.py +++ b/src/sqlbuild/executor/clone/types.py @@ -23,12 +23,6 @@ class CloneFingerprintProgressReporter(CloneFingerprintProgressCallback, Protoco def write_interrupted(self) -> None: ... -class CloneStartCallback(Protocol): - def __call__( - self, *, origin_target_name: str, destination_target_name: str, total: int - ) -> None: ... - - class CloneStatus(StrEnum): SUCCESS = "success" WARNING = "warning" diff --git a/src/sqlbuild/executor/load/_helpers/rows.py b/src/sqlbuild/executor/load/_helpers/rows.py index 86dabf2db5..9df83270e5 100644 --- a/src/sqlbuild/executor/load/_helpers/rows.py +++ b/src/sqlbuild/executor/load/_helpers/rows.py @@ -14,12 +14,6 @@ from sqlbuild.spec.contracts.models import SourceColumnEntry -def normalize_loader_rows(value: object) -> tuple[dict[str, object], ...]: - """Validate and normalize a loader return value to dict rows.""" - - return tuple(iter_normalized_loader_rows(value)) - - def iter_loader_row_batches( *, value: object, batch_size: int ) -> Iterator[tuple[dict[str, object], ...]]: diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 40555505e2..23d14ecfad 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -87,7 +87,6 @@ def execute_ingress_python_loader_nodes( for name in selected_python_names if python_graph.nodes_by_name[name].kind == PythonNodeKind.LOADER ), - read_side_sql_keys=frozenset(), read_side_python_node_names=frozenset(), ), python_graph=python_graph, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py index 64cf1ff845..9a1654c004 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py @@ -21,17 +21,6 @@ def build_ingress_lifecycle_nodes( ) -def build_read_side_python_lifecycle_nodes( - *, plan: PythonSqlRunLifecyclePlan, python_graph: PythonNodeGraph -) -> tuple[LifecycleExecutionNode, ...]: - """Build schedulable SQL-read Python nodes from a lifecycle plan.""" - - return _build_python_lifecycle_nodes( - selected_names=plan.read_side_python_node_names, - python_graph=python_graph, - ) - - def _build_python_lifecycle_nodes( *, selected_names: frozenset[str], python_graph: PythonNodeGraph ) -> tuple[LifecycleExecutionNode, ...]: diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index 85d423612f..825dcedefa 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -69,10 +69,6 @@ def __init__( def results(self) -> tuple[PythonNodeExecutionResult, ...]: return tuple(self._results_by_name.values()) - @property - def completed_python_names(self) -> frozenset[str]: - return frozenset(self._completed_python_names) - def finalize_unrun_python_nodes(self) -> tuple[PythonNodeExecutionResult, ...]: """Record selected Python nodes that could not run after SQL scheduling finished.""" diff --git a/src/sqlbuild/executor/python_nodes/main/_run.py b/src/sqlbuild/executor/python_nodes/main/_run.py deleted file mode 100644 index ffe28e6266..0000000000 --- a/src/sqlbuild/executor/python_nodes/main/_run.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public executor entrypoint for task/asset Python nodes.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.executor.python_nodes._helpers.execution import execute_python_nodes -from sqlbuild.executor.python_nodes.models import PythonNodeExecutorResult, PythonNodeRuntime -from sqlbuild.executor.python_nodes.types import ExecutablePythonNode - - -def run_python_nodes( - *, - nodes: tuple[ExecutablePythonNode, ...], - runtime: PythonNodeRuntime, - statement_recorder: StatementRecorder, -) -> PythonNodeExecutorResult: - """Execute task/asset Python nodes in dependency order.""" - - return execute_python_nodes( - nodes=nodes, - runtime=runtime, - statement_recorder=statement_recorder, - ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index f5e2e88e0d..69e5b0eee1 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -535,7 +535,6 @@ def execute_microbatch_entry( batch_count=batch_outcome.completed_batches, batch_size=batch_plan.effective_batch_size, rows_affected=batch_outcome.rows_affected, - microbatch_applied_intervals=batch_outcome.applied_intervals, **_microbatch_result_fields(history=history_context, succeeded=False), ) final_audit_run: FinalAuditRun = run_final_scope_audits( @@ -618,7 +617,6 @@ def execute_microbatch_entry( batch_count=batch_outcome.completed_batches, batch_size=batch_plan.effective_batch_size, rows_affected=batch_outcome.rows_affected, - microbatch_applied_intervals=batch_outcome.applied_intervals, cursor_range_start=( None if resolved_range is None else sentinel_to_token(sentinel=resolved_range.start) ), @@ -1040,7 +1038,6 @@ def _execute_microbatch_batches( ) schema_checked: bool = False completed_batches: int = 0 - applied_intervals: list[tuple[str, str]] = [] total_rows: int = 0 row_count_known: bool = False batch: BatchWindow @@ -1124,7 +1121,6 @@ def _execute_microbatch_batches( total_rows=total_rows, row_count_known=row_count_known, ), - applied_intervals=tuple(applied_intervals), ) if isinstance(dml_result, int): total_rows += dml_result @@ -1145,7 +1141,6 @@ def _execute_microbatch_batches( total_rows=total_rows, row_count_known=row_count_known, ), - applied_intervals=tuple(applied_intervals), ) completion_failure: ModelExecutionResult | None = ( None @@ -1182,7 +1177,6 @@ def _execute_microbatch_batches( raise lifecycle.completed(affected_rows=dml_result if isinstance(dml_result, int) else None) completed_batches += 1 - applied_intervals.append((render(value=batch.start), render(value=batch.end))) if on_progress is not None: batch_elapsed: float = time.monotonic() - batch_start_time on_progress( @@ -1195,7 +1189,6 @@ def _execute_microbatch_batches( total_rows=total_rows, row_count_known=row_count_known, ), - applied_intervals=tuple(applied_intervals), ) @@ -1361,7 +1354,6 @@ def _execute_microbatch_batches_concurrently( not in history_context.recovery_intervals ) completed_batches: int = 0 - applied_intervals: list[tuple[str, str]] = [] total_rows: int = 0 row_count_known: bool = False for phase_batches in (recovery_batches, ordinary_batches): @@ -1379,7 +1371,6 @@ def _execute_microbatch_batches_concurrently( ) state = phase_outcome.state completed_batches += phase_outcome.completed_batches - applied_intervals.extend(phase_outcome.applied_intervals) if phase_outcome.rows_affected is not None: total_rows += phase_outcome.rows_affected row_count_known = True @@ -1391,7 +1382,6 @@ def _execute_microbatch_batches_concurrently( rows_affected=_reported_rows_affected( total_rows=total_rows, row_count_known=row_count_known ), - applied_intervals=tuple(applied_intervals), ) return MicrobatchPhaseOutcome( state=state, @@ -1399,7 +1389,6 @@ def _execute_microbatch_batches_concurrently( rows_affected=_reported_rows_affected( total_rows=total_rows, row_count_known=row_count_known ), - applied_intervals=tuple(applied_intervals), ) @@ -1468,14 +1457,12 @@ def execute(*, batch: BatchWindow, connection: Any) -> MicrobatchPhaseOutcome: lambda batch, connection: execute(batch=batch, connection=connection), ) completed_batches: int = first_outcome.completed_batches - applied_intervals: list[tuple[str, str]] = list(first_outcome.applied_intervals) total_rows: int = first_outcome.rows_affected or 0 row_count_known: bool = first_outcome.rows_affected is not None failure: ModelExecutionResult | None = None for outcome in outcomes: aggregate_state = _merge_microbatch_states(target=aggregate_state, source=outcome.state) completed_batches += outcome.completed_batches - applied_intervals.extend(outcome.applied_intervals) if outcome.rows_affected is not None: total_rows += outcome.rows_affected row_count_known = True @@ -1488,7 +1475,6 @@ def execute(*, batch: BatchWindow, connection: Any) -> MicrobatchPhaseOutcome: rows_affected=_reported_rows_affected( total_rows=total_rows, row_count_known=row_count_known ), - applied_intervals=tuple(applied_intervals), ) @@ -2748,12 +2734,6 @@ def _cursor_bound(*, values: list[str], cursor_type: str, maximum: bool) -> str: ) -def _cursor_result_string(value: object) -> str: - if isinstance(value, datetime | date): - return value.isoformat() - return str(value) - - def _reported_rows_affected(*, total_rows: int, row_count_known: bool) -> int | None: """Preserve a known zero row count while keeping unavailable counts as None.""" diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index 9cc0e2759b..705525a10d 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -99,68 +99,6 @@ def model_fingerprint_metadata_with_migration_fingerprint( return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) -def same_target_audit_gate_reuse_decision( - *, metadata_json: str | None, model_audits: tuple[AuditPlanEntry, ...] -) -> AuditGateReuseDecision: - """Return whether prior same-target audit gate proof covers planned audits.""" - - if not model_audits: - return AuditGateReuseDecision( - reusable=True, - reason=AuditGateReuseReason.REUSABLE, - ) - if any(audit.always_run for audit in model_audits): - return AuditGateReuseDecision( - reusable=False, - reason=AuditGateReuseReason.ALWAYS_RUN, - ) - audit_gate: AuditGateMetadata | AuditGateMetadataParseFailureDetail = _read_audit_gate( - metadata_json - ) - if isinstance(audit_gate, AuditGateMetadataParseFailureDetail): - _log_parse_failure(audit_gate) - return AuditGateReuseDecision( - reusable=False, reason=_parse_failure_reuse_reason(audit_gate) - ) - if audit_gate.status != AuditGateStatus.PASSED.value: - return AuditGateReuseDecision(reusable=False, reason=AuditGateReuseReason.NON_PASSING) - identity: AuditGateIdentity = build_audit_gate_identity(audits=model_audits) - if audit_gate.binding_set_hash != identity.binding_set_hash: - return AuditGateReuseDecision( - reusable=False, - reason=AuditGateReuseReason.BINDING_SET_CHANGED, - missing_binding_keys=tuple(audit.binding_key for audit in identity.audits), - ) - prior_results: dict[str, AuditGateResultMetadata] = { - result.binding_key: result for result in audit_gate.results - } - - reusable_binding_keys: list[str] = [] - missing_binding_keys: list[str] = [] - audit: AuditIdentity - for audit in identity.audits: - prior_result: AuditGateResultMetadata | None = prior_results.get(audit.binding_key) - if ( - prior_result is None - or prior_result.execution_fingerprint != audit.execution_fingerprint - ): - missing_binding_keys.append(audit.binding_key) - continue - reusable_binding_keys.append(audit.binding_key) - if missing_binding_keys: - return AuditGateReuseDecision( - reusable=False, - reason=AuditGateReuseReason.AUDIT_CHANGED, - reusable_binding_keys=tuple(reusable_binding_keys), - missing_binding_keys=tuple(missing_binding_keys), - ) - return AuditGateReuseDecision( - reusable=True, - reason=AuditGateReuseReason.REUSABLE, - reusable_binding_keys=tuple(reusable_binding_keys), - ) - - def reuse_from_audit_gate_reuse_decision( *, metadata_json: str | None, model_audits: tuple[AuditPlanEntry, ...] ) -> AuditGateReuseDecision: diff --git a/src/sqlbuild/executor/run/main/_promote.py b/src/sqlbuild/executor/run/main/_promote.py deleted file mode 100644 index 21170d79b0..0000000000 --- a/src/sqlbuild/executor/run/main/_promote.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public relation promotion entrypoint for run execution.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.executor.run._helpers.execution.promotion import promote_relation_to_destination - - -def promote_run_relation_to_destination( - *, - adapter: BaseAdapter, - connection: Any, - origin_relation: str, - destination_relation: str, - destination_database: str | None, - destination_schema: str | None, - destination_name: str, - statement_recorder: StatementRecorder, -) -> None: - """Promote an already-created run relation into its final destination.""" - - _ = promote_relation_to_destination( - adapter=adapter, - connection=connection, - origin_relation=origin_relation, - destination_relation=destination_relation, - destination_database=destination_database, - destination_schema=destination_schema, - destination_name=destination_name, - statement_recorder=statement_recorder, - ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 95f83baffa..52c4b70829 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -179,7 +179,6 @@ class ModelExecutionResult: microbatch_accounting_intervals: tuple[MicrobatchAccountingInterval, ...] = field( default_factory=tuple ) - microbatch_applied_intervals: tuple[tuple[str, str], ...] = field(default_factory=tuple) microbatch_limit: int | None = None microbatch_limit_count: int | None = None microbatch_limit_action: MicrobatchLimitAction | None = None @@ -554,7 +553,6 @@ class MicrobatchPhaseOutcome: failure: ModelExecutionResult | None = None completed_batches: int = 0 rows_affected: int | None = None - applied_intervals: tuple[tuple[str, str], ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/run/types.py b/src/sqlbuild/executor/run/types.py index 5cb57e9c45..eee3e57897 100644 --- a/src/sqlbuild/executor/run/types.py +++ b/src/sqlbuild/executor/run/types.py @@ -85,7 +85,6 @@ class AuditGateReuseReason(StrEnum): MISSING = "missing" MALFORMED = "malformed" NON_PASSING = "non_passing" - BINDING_SET_CHANGED = "binding_set_changed" AUDIT_CHANGED = "audit_changed" ALWAYS_RUN = "always_run" diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index f8562e2c62..03b8ad1f02 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -52,7 +52,6 @@ EVENT_OUTPUT_FLAG: str = "--event-output" LIVE_EVENT_COMMANDS: frozenset[str] = ASSET_SELECTION_COMMANDS VIRTUAL_ENV_FLAG: str = "--virtual-env" -VERBOSE_FLAGS: frozenset[str] = frozenset({"--verbose", "-v"}) EXPLICIT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "-s", "--select-file"}) JSON_OUTPUT_FLAGS: frozenset[str] = frozenset({"--json", "--json-output"}) JSON_OUTPUT_FLAG: str = "--json-output" diff --git a/src/sqlbuild/integrations/dagster/types.py b/src/sqlbuild/integrations/dagster/types.py index 0109e5c3c1..e370d93436 100644 --- a/src/sqlbuild/integrations/dagster/types.py +++ b/src/sqlbuild/integrations/dagster/types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -10,4 +10,3 @@ type DagNode = Mapping[str, Any] type DagCheck = Mapping[str, Any] type DagEdge = Mapping[str, Any] -type AssetKeyParts = Sequence[str] diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py index ebcb96f68a..f01974bac9 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py +++ b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py @@ -85,23 +85,3 @@ def validate_compile_dbt_model_reference( package_name=reference.ref_package, name=reference.ref_name, ) - - -def resolve_compile_dbt_ref_references( - *, query_sql: str, dbt_manifest: DbtManifestIndex | None -) -> str: - """Replace model __dbt_ref() calls with dbt manifest relation names.""" - - if dbt_manifest is None: - return query_sql - - def _replace_dbt_ref(match: re.Match[str]) -> str: - first_arg: str = match.group(1) - second_arg: str | None = match.group(2) - return resolve_dbt_manifest_model( - manifest=dbt_manifest, - package_name=first_arg if second_arg is not None else None, - name=second_arg if second_arg is not None else first_arg, - ).relation_name - - return _DBT_REF_PATTERN.sub(_replace_dbt_ref, query_sql) diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py index b96485b59b..c7f0a71beb 100644 --- a/src/sqlbuild/integrations/dbt/constants.py +++ b/src/sqlbuild/integrations/dbt/constants.py @@ -10,24 +10,13 @@ ) ) DBT_EXECUTION_DISPLAY_FLAGS: frozenset[str] = frozenset(("--json", "--verbose", "-v")) -DBT_SUCCESSFUL_RESULT_STATUSES: frozenset[str] = frozenset({"ok", "success", "pass", "passed"}) DBT_MANIFEST_CONFIG_KEY: str = "config" -DBT_MANIFEST_INCREMENTAL_STRATEGY_KEY: str = "incremental_strategy" DBT_MANIFEST_MATERIALIZED_KEY: str = "materialized" -DBT_MANIFEST_META_KEY: str = "meta" -DBT_MANIFEST_RESOURCE_TYPE_KEY: str = "resource_type" -DBT_MANIFEST_SQLBUILD_META_KEY: str = "sqlbuild" DBT_EXECUTABLE_ENV_VAR: str = "DBT_EXECUTABLE" DEFAULT_DBT_EXECUTABLE: str = "dbt" -DBT_MATERIALIZATION_EPHEMERAL: str = "ephemeral" -DBT_MATERIALIZATION_INCREMENTAL: str = "incremental" -DBT_MATERIALIZATION_MICROBATCH: str = "microbatch" -DBT_MATERIALIZATION_SNAPSHOT: str = "snapshot" -DBT_MATERIALIZATION_TABLE: str = "table" -DBT_MATERIALIZATION_VIEW: str = "view" DBT_DEFER_FLAG: str = "--defer" DBT_EXECUTION_FAIL_STATUSES: frozenset[str] = frozenset({"error", "fail", "failed"}) @@ -36,11 +25,7 @@ {"ok", "success", "pass", "passed", "warn", "warning"} ) DBT_EXECUTION_WARN_STATUSES: frozenset[str] = frozenset({"warn", "warning"}) -DBT_EXCLUDE_FLAG: str = "--exclude" DBT_FULL_REFRESH_FLAG: str = "--full-refresh" DBT_PATH_SELECTOR_SEPARATOR: str = "~" -DBT_SELECT_FLAGS: frozenset[str] = frozenset({"--select", "-s"}) DBT_SELECT_FLAG: str = "--select" -DBT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "--exclude"}) DBT_UNIQUE_ID_SEPARATOR: str = "." -DBT_VERBOSE_FLAGS: frozenset[str] = frozenset({"--verbose", "-v"}) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py b/src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py deleted file mode 100644 index 425e86a0c5..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_build_manifest_index.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public dbt manifest indexing entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - build_dbt_manifest_index as _build_dbt_manifest_index, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def build_manifest_index(*, raw_data: object) -> DbtManifestIndex: - """Build a dbt manifest index from decoded manifest JSON.""" - - return _build_dbt_manifest_index(raw_data=raw_data) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py deleted file mode 100644 index 9a69f07c63..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_compile_model_refs.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public dbt compile model ref resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.compile_refs import ( - resolve_compile_dbt_ref_references, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex - - -def resolve_compile_model_refs(*, query_sql: str, dbt_manifest: DbtManifestIndex | None) -> str: - """Replace SQLBuild model __dbt_ref() calls with dbt relation names.""" - - return resolve_compile_dbt_ref_references( - query_sql=query_sql, - dbt_manifest=dbt_manifest, - ) diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py deleted file mode 100644 index 7c2c62e227..0000000000 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_manifest_model.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public dbt manifest model resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.integrations.dbt._helpers.manifest.core import ( - resolve_dbt_manifest_model as _resolve_dbt_manifest_model, -) -from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel - - -def resolve_manifest_model( - *, manifest: DbtManifestIndex, name: str, package_name: str | None = None -) -> DbtManifestModel: - """Resolve a dbt model from a manifest index.""" - - return _resolve_dbt_manifest_model( - manifest=manifest, - name=name, - package_name=package_name, - ) diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 2b80317664..336184a5fe 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -103,15 +103,6 @@ class DbtCliConfigOverrides: target: str | None = None target_path: str | None = None - @property - def has_any(self) -> bool: - """Return whether any dbt CLI config flag was supplied.""" - - return any( - value is not None - for value in (self.project_dir, self.profiles_dir, self.target, self.target_path) - ) - @dataclass(frozen=True) class ResolvedDbtConfig: diff --git a/src/sqlbuild/integrations/dlt/constants.py b/src/sqlbuild/integrations/dlt/constants.py index 8bc0ab8300..bd492e9b06 100644 --- a/src/sqlbuild/integrations/dlt/constants.py +++ b/src/sqlbuild/integrations/dlt/constants.py @@ -2,19 +2,6 @@ from __future__ import annotations -from sqlbuild.adapter.contract.types import BuiltinAdapter - -DLT_DESTINATION_ADAPTERS: frozenset[BuiltinAdapter] = frozenset( - { - BuiltinAdapter.DUCKDB, - BuiltinAdapter.MOTHERDUCK, - BuiltinAdapter.SNOWFLAKE, - BuiltinAdapter.BIGQUERY, - BuiltinAdapter.DATABRICKS, - BuiltinAdapter.POSTGRES, - BuiltinAdapter.SQLSERVER, - } -) DLT_INTEGRATION_KIND: str = "dlt" DLT_SOURCE_TYPE_SQL_DATABASE: str = "sql_database" DLT_SOURCE_TYPE_REST_API: str = "rest_api" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/output.py b/src/sqlbuild/integrations/ingestr/_helpers/output.py index 47f0d2f8d2..a9eed714b8 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/output.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/output.py @@ -3,7 +3,6 @@ from __future__ import annotations from shlex import quote -from typing import TextIO from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.ingestr.models import IngestrCommandResult @@ -15,17 +14,6 @@ def format_ingestr_command(command: tuple[str, ...]) -> str: return " ".join(quote(part) for part in command) -def write_external_output(*, stream: TextIO, label: str, output: str) -> None: - """Write a labeled external command output block.""" - - stream.write(f"{label}\n") - stream.write(output) - if not output.endswith("\n"): - stream.write("\n") - stream.write("\n") - stream.flush() - - def record_ingestr_output(*, ctx: LoaderContext, result: IngestrCommandResult) -> None: """Attach captured ingestr output to the loader lifecycle.""" diff --git a/src/sqlbuild/integrations/rivers/types.py b/src/sqlbuild/integrations/rivers/types.py index 7ea63a5c99..4344d98cb1 100644 --- a/src/sqlbuild/integrations/rivers/types.py +++ b/src/sqlbuild/integrations/rivers/types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -10,4 +10,3 @@ type DagNode = Mapping[str, Any] type DagCheck = Mapping[str, Any] type DagEdge = Mapping[str, Any] -type AssetKeyParts = Sequence[str] diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 72068bc867..3ad24a1da7 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -135,9 +135,9 @@ def prepare_lint_body( project_dir: Path, file_path: Path, contents: str, - body_start: int, - body_end: int, + body_range: tuple[int, int], context: SqlExpansionContext, + dialect: str, external_identifiers: tuple[str, ...] = (), allows_ceremonial_select: bool = False, allows_dynamic_output_star: bool = False, @@ -145,6 +145,9 @@ def prepare_lint_body( ) -> LintBody: """Expand one authored body and neutralize whatever interpolation remains.""" + body_start: int + body_end: int + body_start, body_end = body_range authored_body: str = contents[body_start:body_end] pre_expansion_sites: tuple[InterpolationSite, ...] = () expansion_input: str = authored_body @@ -184,11 +187,16 @@ def prepare_lint_body( sites: tuple[InterpolationSite, ...] native_prepared: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( _native.prepare_lint_sql( - expanded, expansion_input, [site.neutralized_start for site in pre_expansion_sites] + { + "expanded": expanded, + "before_expansion": expansion_input, + "prior_sites": [site.neutralized_start for site in pre_expansion_sites], + "dialect": dialect, + } ) ) if native_prepared is None: - neutralized, sites = neutralize_interpolation(body=expanded) + neutralized, sites = neutralize_interpolation(body=expanded, dialect=dialect) externally_referenced_ctes: tuple[str, ...] = _externally_referenced_ctes( expanded=expanded, interpolation_sites=sites, diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 93eb37b407..caf897b212 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -6,149 +6,6 @@ import tempfile from pathlib import Path -from sqlbuild.compiler.compile.models import SqlExpansionContext -from sqlbuild.lint._helpers.expansion import prepare_lint_body -from sqlbuild.lint._helpers.headers import lint_body_ranges, scan_headers -from sqlbuild.lint._helpers.native import format_native_headers, lint_native_headers -from sqlbuild.lint._helpers.native_sql import run_native_sql_lint -from sqlbuild.lint.constants import FIX_STATUS_APPLIED -from sqlbuild.lint.models import ( - HeaderSpan, - LintBody, - LintConfig, - LintEdit, - LintFixRecord, - LintViolation, -) - - -def apply_header_repairs( - *, files: dict[Path, str], config: LintConfig -) -> tuple[dict[Path, str], list[LintFixRecord]]: - repaired: dict[Path, str] = {} - records: list[LintFixRecord] = [] - for file_path, contents in sorted(files.items()): - before: tuple[LintViolation, ...] = lint_native_headers( - contents=contents, - file_path=file_path, - headers=scan_headers(contents=contents), - config=config, - ) - updated, _faults = format_native_headers( - contents=contents, - file_path=file_path, - config=config, - ) - repaired[file_path] = updated - after_codes: set[str] = { - violation.code - for violation in lint_native_headers( - contents=updated, - file_path=file_path, - headers=scan_headers(contents=updated), - config=config, - ) - } - records.extend( - fix_record(violation=violation, status=FIX_STATUS_APPLIED, reason=None) - for violation in before - if violation.code not in after_codes - ) - return repaired, records - - -def lint_contents( - *, - files: dict[Path, str], - config: LintConfig, - context: SqlExpansionContext | None, - project_dir: Path, -) -> list[LintViolation]: - violations: list[LintViolation] = [] - bodies: list[LintBody] = [] - for file_path, contents in sorted(files.items()): - headers: tuple[HeaderSpan, ...] = scan_headers(contents=contents) - violations.extend( - lint_native_headers( - contents=contents, - file_path=file_path, - headers=headers, - config=config, - ) - ) - if context is None: - continue - for body_start, body_end in lint_body_ranges( - contents=contents, - headers=headers, - file_path=file_path, - project_dir=project_dir, - ): - bodies.append( - prepare_lint_body( - project_dir=project_dir, - file_path=file_path, - contents=contents, - body_start=body_start, - body_end=body_end, - context=context, - ) - ) - if bodies: - native: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( - bodies=tuple(bodies), - contents_by_path=files, - config=config, - ) - for entries in native.values(): - violations.extend(entries) - return violations - - -def planned_edits( - *, violations: list[LintViolation] -) -> tuple[dict[Path, tuple[LintEdit, ...]], list[LintFixRecord]]: - selected: dict[Path, list[LintEdit]] = {} - records: list[LintFixRecord] = [] - for violation in violations: - if violation.fix is None: - continue - existing: list[LintEdit] = selected.setdefault(violation.file_path, []) - if any(overlaps(left=violation.fix, right=edit) for edit in existing): - continue - existing.append(violation.fix) - records.append(fix_record(violation=violation, status=FIX_STATUS_APPLIED, reason=None)) - return ( - { - path: tuple(sorted(edits, key=lambda edit: (edit.start, edit.end, edit.code))) - for path, edits in selected.items() - }, - records, - ) - - -def fix_record(*, violation: LintViolation, status: str, reason: str | None) -> LintFixRecord: - return LintFixRecord( - file_path=violation.file_path, - code=violation.code, - line=violation.line, - column=violation.column, - status=status, - reason=reason, - ) - - -def apply_edits( - *, files: dict[Path, str], edits: dict[Path, tuple[LintEdit, ...]] -) -> dict[Path, str]: - updated: dict[Path, str] = dict(files) - for path, path_edits in edits.items(): - contents: str = files[path] - for edit in reversed(path_edits): - contents = contents[: edit.start] + edit.replacement + contents[edit.end :] - updated[path] = contents - return updated - def write_atomically(*, path: Path, contents: str) -> None: descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) @@ -162,7 +19,3 @@ def write_atomically(*, path: Path, contents: str) -> None: os.replace(temporary_path, path) finally: temporary_path.unlink(missing_ok=True) - - -def overlaps(*, left: LintEdit, right: LintEdit) -> bool: - return left.start < right.end and right.start < left.end diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index 553eb60398..ed30e0bd9d 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -22,6 +22,7 @@ HEADER_KIND_MODEL, IDENTIFIER_SEPARATOR_CHARACTER, LINE_FEED, + LINT_ENGINE_SQLBUILD, RULE_DESCRIPTION_LENGTH, RULE_DESCRIPTION_PRESENT, RULE_HEADER_PARSE, @@ -214,7 +215,7 @@ def _lint_header_values( code=RULE_HEADER_PARSE, message=f"{header.kind}() header could not be parsed: {error}", severity=VIOLATION_SEVERITY_FAULT, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation=f"Correct the {header.kind}() header syntax.", ), ) @@ -317,7 +318,7 @@ def _lint_header_whitespace( code=RULE_HEADER_WHITESPACE, message=f"{header.kind}() header contains trailing whitespace", severity=VIOLATION_SEVERITY_WARNING, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation="Run sqb format to remove trailing header whitespace.", ), ) @@ -568,7 +569,7 @@ def _relocate_leading_comment( code=RULE_LEADING_COMMENT_DESCRIPTION, message=(f"Relocated leading comment exceeds {config.max_description_lines} lines"), severity=VIOLATION_SEVERITY_FAULT, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation=( f"Shorten the relocated description to {config.max_description_lines} lines." ), @@ -675,7 +676,7 @@ def _violation_for_header_start( code=code, message=message, severity=severity, - engine="sqlbuild", + engine=LINT_ENGINE_SQLBUILD, remediation=remediation, ) diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py index 0dc110b5de..00a9d13dda 100644 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ b/src/sqlbuild/lint/_helpers/native_format.py @@ -31,12 +31,6 @@ class _PreparedBody: interpolation_sites: tuple[InterpolationSite, ...] -def newline_style(*, contents: str) -> str: - """Return the newline convention used by the authored contents.""" - - return CARRIAGE_RETURN_LINE_FEED if CARRIAGE_RETURN_LINE_FEED in contents else LINE_FEED - - def with_newline_style(*, contents: str, newline: str) -> str: """Normalize generated contents back to the authored newline convention.""" @@ -66,7 +60,7 @@ def format_native_sql_bodies( body: str = contents[body_start:body_end] trailing: str = body[len(body.rstrip()) :] core: str = body[: len(body) - len(trailing)] if trailing else body - neutralized, sites = neutralize_interpolation(body=core) + neutralized, sites = neutralize_interpolation(body=core, dialect=config.dialect) cache_key: tuple[str, str] = (neutralized, config.dialect) requests_by_key.setdefault( cache_key, diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py index a18b081d4e..ab02e7f638 100644 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ b/src/sqlbuild/lint/_helpers/native_sql.py @@ -132,6 +132,7 @@ def run_native_sql_lint( raw_diagnostic=raw_diagnostic, body=body, contents=contents_by_path[body.file_path], + dialect=config.dialect, ) if violation is not None: violations_by_file.setdefault(body.file_path, []).append(violation) @@ -220,7 +221,7 @@ def _parse_failure_violation( def _authored_violation( - *, raw_diagnostic: object, body: LintBody, contents: str + *, raw_diagnostic: object, body: LintBody, contents: str, dialect: str ) -> LintViolation | None: if not isinstance(raw_diagnostic, dict): raise NativeLintError("native lint diagnostic must be an object") @@ -263,6 +264,7 @@ def _authored_violation( body=body, contents=contents, line_starts=starts, + dialect=dialect, ) fix: LintEdit | None = _authored_fix( raw_fix=raw_fix, @@ -282,6 +284,7 @@ def _authored_violation( mapped=mapped, contents=contents, absolute_offset=absolute_offset, + dialect=dialect, ), severity=VIOLATION_SEVERITY_WARNING, engine=LINT_ENGINE_NATIVE, @@ -341,11 +344,14 @@ def _authored_end_position( body: LintBody, contents: str, line_starts: tuple[int, ...], + dialect: str, ) -> tuple[int, int] | None: """Map a native exclusive end offset only when authored continuity is provable.""" if mapped_start.generated: - token: str | None = interpolation_text_at(body=contents, start=absolute_start) + token: str | None = interpolation_text_at( + body=contents, start=absolute_start, dialect=dialect + ) if token is None: return None return _offset_position(offset=absolute_start + len(token), line_starts=line_starts) @@ -359,11 +365,11 @@ def _authored_end_position( def _violation_message( - *, message: str, mapped: MappedOffset, contents: str, absolute_offset: int + *, message: str, mapped: MappedOffset, contents: str, absolute_offset: int, dialect: str ) -> str: if not mapped.generated: return message - token: str | None = interpolation_text_at(body=contents, start=absolute_offset) + token: str | None = interpolation_text_at(body=contents, start=absolute_offset, dialect=dialect) if token is None: return f"{message} {GENERATED_SQL_MESSAGE_SUFFIX}" return f"{message} {GENERATED_SQL_MESSAGE_TEMPLATE.format(token=token)}" diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py index 3519ed0970..81adcbbc2b 100644 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py @@ -3,7 +3,9 @@ from __future__ import annotations import re +from functools import cache +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import ( MACRO_TOKEN, SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN, @@ -12,6 +14,7 @@ ) from sqlbuild.compiler.compile.models import ExpansionSpan from sqlbuild.lint.constants import ( + BACKTICK_CHARACTER, CLOSING_PAREN_CHARACTER, EXPECTED_SENTINEL_OCCURRENCES, IDENTIFIER_EXTRA_CHARACTER, @@ -43,15 +46,29 @@ _SQLBUILD_FUNCTION_PATTERN: str = "|".join( re.escape(name) for name in sorted(_SQLBUILD_FUNCTION_NAMES) ) +_NON_CODE_SCAN_PATTERN: str = ( + r"--[^\n]*(?:\n|\Z)" + r"|/\*[\s\S]*?(?:\*/|\Z)" + r"|'(?:\\.|''|[^'\\])*(?:'|\Z)" + r'|"(?:\\.|""|[^"\\])*(?:"|\Z)' +) +_BACKTICK_SCAN_PATTERN: str = r"|`(?:``|[^`])*(?:`|\Z)" +_SITE_SCAN_PATTERN: str = rf"|(?P@@|\$\{{|@|(?:{_SQLBUILD_FUNCTION_PATTERN})\s*\()" _INTERPOLATION_SCAN_PATTERN: re.Pattern[str] = re.compile( - rf"--[^\n]*(?:\n|\Z)" - rf"|/\*[\s\S]*?(?:\*/|\Z)" - rf"|'(?:\\.|''|[^'\\])*(?:'|\Z)" - rf'|"(?:\\.|""|[^"\\])*(?:"|\Z)' - rf"|(?P@@|\$\{{|@|(?:{_SQLBUILD_FUNCTION_PATTERN})\s*\()" + _NON_CODE_SCAN_PATTERN + _SITE_SCAN_PATTERN +) +_BACKTICK_INTERPOLATION_SCAN_PATTERN: re.Pattern[str] = re.compile( + _NON_CODE_SCAN_PATTERN + _BACKTICK_SCAN_PATTERN + _SITE_SCAN_PATTERN ) +@cache +def _backtick_identifiers(dialect: str) -> bool: + """Return whether lint treats backticks as identifier quotes, matching the native lexer.""" + + return _native.lint_backtick_identifiers(dialect) + + def neutralize_context_interpolation(*, body: str) -> tuple[str, tuple[InterpolationSite, ...]]: """Replace runtime CTX interpolation with parseable generated sentinels.""" @@ -64,9 +81,7 @@ def neutralize_context_interpolation(*, body: str) -> tuple[str, tuple[Interpola start: int = body.find(token, copied_to) if start < 0: break - end: int | None = _interpolation_site_end(body=body, start=start) - if end is None: - break + end: int = _interpolation_name_end(body=body, start=start + len(SQL_INTERPOLATION_TOKEN)) literal: str = body[copied_to:start] sentinel: str = _CONTEXT_SENTINEL_TEMPLATE.format(index=len(sites)) pieces.extend((literal, sentinel)) @@ -123,21 +138,31 @@ def neutralize_generic_audit_parameters(*, body: str) -> tuple[str, tuple[Interp return "".join(pieces), tuple(sites) -def neutralize_interpolation(*, body: str) -> tuple[str, tuple[InterpolationSite, ...]]: +def neutralize_interpolation( + *, body: str, dialect: str +) -> tuple[str, tuple[InterpolationSite, ...]]: """Replace every interpolation site with a unique sentinel identifier.""" if not _contains_interpolation_candidate(body=body): return body, () + backtick_identifiers: bool = _backtick_identifiers(dialect) + scan_pattern: re.Pattern[str] = ( + _BACKTICK_INTERPOLATION_SCAN_PATTERN + if backtick_identifiers + else _INTERPOLATION_SCAN_PATTERN + ) sites: list[InterpolationSite] = [] pieces: list[str] = [] neutralized_length: int = 0 copied_to: int = 0 match: re.Match[str] - for match in _INTERPOLATION_SCAN_PATTERN.finditer(body): + for match in scan_pattern.finditer(body): if match.group("site") is None or match.start() < copied_to: continue site_start: int = match.start() - site_end: int | None = _interpolation_site_end(body=body, start=site_start) + site_end: int | None = _interpolation_site_end( + body=body, start=site_start, backtick_identifiers=backtick_identifiers + ) if site_end is None: continue literal: str = body[copied_to:site_start] @@ -190,10 +215,12 @@ def restore_interpolation(*, fixed: str, sites: tuple[InterpolationSite, ...]) - return restored -def interpolation_text_at(*, body: str, start: int) -> str | None: +def interpolation_text_at(*, body: str, start: int, dialect: str) -> str | None: """Return the interpolation token beginning at an offset, if there is one.""" - site_end: int | None = _interpolation_site_end(body=body, start=start) + site_end: int | None = _interpolation_site_end( + body=body, start=start, backtick_identifiers=_backtick_identifiers(dialect) + ) if site_end is None: return None return body[start:site_end] @@ -216,40 +243,28 @@ def sentinel_spans(*, sites: tuple[InterpolationSite, ...]) -> tuple[ExpansionSp return tuple(spans) -def map_neutralized_offset(*, offset: int, sites: tuple[InterpolationSite, ...]) -> int: - """Map an offset in neutralized text back onto the authored body.""" - - mapped: int = offset - site: InterpolationSite - for site in sites: - if offset < site.neutralized_start: - break - if offset < site.neutralized_end: - return site.original_start - mapped += (site.original_end - site.original_start) - ( - site.neutralized_end - site.neutralized_start - ) - return mapped - - -def _interpolation_site_end(*, body: str, start: int) -> int | None: +def _interpolation_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: character: str = body[start] if character == MACRO_TOKEN: if body.startswith(SQL_INTERPOLATION_TOKEN, start): return _interpolation_name_end(body=body, start=start + len(SQL_INTERPOLATION_TOKEN)) - return _macro_site_end(body=body, start=start) + return _macro_site_end(body=body, start=start, backtick_identifiers=backtick_identifiers) if character == TEMPLATE_INTERPOLATION_START[0]: return _template_site_end(body=body, start=start) if character == IDENTIFIER_EXTRA_CHARACTER and body.startswith(_SQLBUILD_FUNCTION_NAMES, start): - return _sqlbuild_function_site_end(body=body, start=start) + return _sqlbuild_function_site_end( + body=body, start=start, backtick_identifiers=backtick_identifiers + ) return None -def _sqlbuild_function_site_end(*, body: str, start: int) -> int | None: +def _sqlbuild_function_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: name_end: int = _identifier_end(body=body, start=start) if name_end >= len(body) or body[name_end] != OPENING_PAREN_CHARACTER: return None - call_end: int | None = _matching_paren_end(body=body, opening_index=name_end) + call_end: int | None = _matching_paren_end( + body=body, opening_index=name_end, backtick_identifiers=backtick_identifiers + ) return call_end @@ -262,7 +277,7 @@ def _template_site_end(*, body: str, start: int) -> int | None: return end_index + len(TEMPLATE_INTERPOLATION_END) -def _macro_site_end(*, body: str, start: int) -> int | None: +def _macro_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: name_start: int = start + len(MACRO_TOKEN) if name_start >= len(body): return None @@ -276,10 +291,12 @@ def _macro_site_end(*, body: str, start: int) -> int | None: name_end: int = _identifier_end(body=body, start=name_start) if name_end >= len(body) or body[name_end] != OPENING_PAREN_CHARACTER: return name_end - return _matching_paren_end(body=body, opening_index=name_end) + return _matching_paren_end( + body=body, opening_index=name_end, backtick_identifiers=backtick_identifiers + ) -def _matching_paren_end(*, body: str, opening_index: int) -> int | None: +def _matching_paren_end(*, body: str, opening_index: int, backtick_identifiers: bool) -> int | None: depth: int = 0 index: int = opening_index length: int = len(body) @@ -295,7 +312,11 @@ def _matching_paren_end(*, body: str, opening_index: int) -> int | None: continue character: str = body[index] if quote_character is not None: - if character == SQL_ESCAPE_CHARACTER and index + 1 < length: + if ( + character == SQL_ESCAPE_CHARACTER + and quote_character in SQL_QUOTE_CHARACTERS + and index + 1 < length + ): index += 2 continue if character == quote_character: @@ -305,7 +326,9 @@ def _matching_paren_end(*, body: str, opening_index: int) -> int | None: quote_character = None index += 1 continue - if character in SQL_QUOTE_CHARACTERS: + if character in SQL_QUOTE_CHARACTERS or ( + backtick_identifiers and character == BACKTICK_CHARACTER + ): quote_character = character index += 1 continue diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index e39076ac2e..6d517783bd 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -78,6 +78,7 @@ ADAPTER_CONFIG_KEY: str = "adapter" SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) +BACKTICK_CHARACTER: str = "`" SQL_ESCAPE_CHARACTER: str = "\\" SINGLE_QUOTE_CHARACTER: str = "'" OPENING_PAREN_CHARACTER: str = "(" @@ -95,6 +96,3 @@ LINE_FEED: str = "\n" CARRIAGE_RETURN_LINE_FEED: str = "\r\n" TAB_CHARACTER: str = "\t" -FIX_STATUS_APPLIED: str = "fixed" -FIX_STATUS_SKIPPED: str = "skipped" -MAX_FIX_PASSES: int = 8 diff --git a/src/sqlbuild/lint/exceptions.py b/src/sqlbuild/lint/exceptions.py index 7e8905c218..80e650904b 100644 --- a/src/sqlbuild/lint/exceptions.py +++ b/src/sqlbuild/lint/exceptions.py @@ -31,9 +31,3 @@ class ProjectCompileError(LintError): """The project could not be compiled, so its SQL cannot be linted.""" code: str = "L005" - - -class CustomLintError(LintError): - """A repository-defined statement-local lint rule is invalid or failed.""" - - code: str = "L006" diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py index 522387c993..8c177abae2 100644 --- a/src/sqlbuild/lint/main/run_lint.py +++ b/src/sqlbuild/lint/main/run_lint.py @@ -78,6 +78,7 @@ def run_lint( contents=contents, headers=headers, context=context, + dialect=config.dialect, project_dir=project_dir, allows_dynamic_output_star=file_path.resolve() in dynamic_output_paths, compiled_expansions=compiled_expansions, @@ -124,6 +125,7 @@ def _prepared_bodies( contents: str, headers: tuple[HeaderSpan, ...], context: SqlExpansionContext, + dialect: str, project_dir: Path, allows_dynamic_output_star: bool, compiled_expansions: dict[Path, CompiledSqlExpansion] | None, @@ -151,9 +153,9 @@ def _prepared_bodies( project_dir=project_dir, file_path=file_path, contents=contents, - body_start=body_start, - body_end=body_end, + body_range=(body_start, body_end), context=context, + dialect=dialect, external_identifiers=external_identifiers, allows_ceremonial_select=allows_ceremonial_select, allows_dynamic_output_star=allows_dynamic_output_star, diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index c0079f988a..88563a4875 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -95,18 +95,6 @@ class FormatChange: after: str -@dataclass(frozen=True) -class LintFixRecord: - """One applied or skipped lint repair for human and machine reporting.""" - - file_path: Path - code: str - line: int - column: int - status: str - reason: str | None = None - - @dataclass(frozen=True) class LintConfig: """Resolved lint and format configuration for one run.""" @@ -149,31 +137,3 @@ def warnings(self) -> tuple[LintViolation, ...]: for violation in self.violations if violation.severity == VIOLATION_SEVERITY_WARNING ) - - -@dataclass(frozen=True) -class FixRunResult: - """Aggregated result of planning or applying native lint repairs.""" - - files_checked: int - violations: tuple[LintViolation, ...] - changed_files: tuple[Path, ...] - changes: tuple[FormatChange, ...] - fixes: tuple[LintFixRecord, ...] - source_texts: Mapping[Path, str] = field(default_factory=dict, repr=False, compare=False) - - @property - def faults(self) -> tuple[LintViolation, ...]: - return tuple( - violation - for violation in self.violations - if violation.severity == VIOLATION_SEVERITY_FAULT - ) - - @property - def warnings(self) -> tuple[LintViolation, ...]: - return tuple( - violation - for violation in self.violations - if violation.severity == VIOLATION_SEVERITY_WARNING - ) diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index 1ba51e8557..c79c8b6d41 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -11,5 +11,4 @@ SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" TREE_LAST_GLYPH: str = "\u2514\u2500\u2500" -TREE_PIPE_GLYPH: str = "\u2502" WARNING_STATUS_LABELS: frozenset[str] = frozenset({"WARN", "WARNING"}) diff --git a/src/sqlbuild/rule_engine/_helpers/sql/ast.py b/src/sqlbuild/rule_engine/_helpers/sql/ast.py index af5f37ebbb..a24a0b4ff4 100644 --- a/src/sqlbuild/rule_engine/_helpers/sql/ast.py +++ b/src/sqlbuild/rule_engine/_helpers/sql/ast.py @@ -47,17 +47,6 @@ def name_value(value: object) -> str: return "" -def table_parts(node: Any) -> tuple[str, str, str]: - """Return catalog, schema, and name for one table node.""" - - data: dict[str, object] = payload(node) - return ( - name_value(data.get("catalog")), - name_value(data.get("schema")), - name_value(data.get("name")), - ) - - def cte_name(node: Any) -> str: if isinstance(node, tuple) and len(node) == NAMED_CTE_TUPLE_SIZE: return str(node[0]) diff --git a/src/sqlbuild/rule_engine/_helpers/sql/decisions.py b/src/sqlbuild/rule_engine/_helpers/sql/decisions.py deleted file mode 100644 index d6327c2fd5..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/decisions.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Polyglot decision-site identification.""" - -from __future__ import annotations - -import json -from typing import Any - -_COMPARISON_KINDS: frozenset[str] = frozenset({"eq", "neq", "gt", "gte", "lt", "lte"}) -_DECISION_KEYS: frozenset[str] = frozenset({"having", "ifs", "on", "qualify", "where_clause"}) - - -def decision_comparison_signatures(*, ast: Any) -> frozenset[str]: - """Return stable payload signatures for comparisons under decision clauses.""" - - signatures: set[str] = set() - signatures = _collect_signatures( - value=ast.to_dict(), inside_decision=False, signatures=signatures - ) - return frozenset(signatures) - - -def comparison_signature(*, node: Any) -> str: - """Return the stable payload signature for one Polyglot comparison node.""" - - return json.dumps(node.to_dict(), sort_keys=True, separators=(",", ":")) - - -def _collect_signatures(*, value: object, inside_decision: bool, signatures: set[str]) -> set[str]: - if isinstance(value, list): - for item in value: - signatures = _collect_signatures( - value=item, - inside_decision=inside_decision, - signatures=signatures, - ) - return signatures - if not isinstance(value, dict): - return signatures - for key, child in value.items(): - child_is_decision: bool = inside_decision or key in _DECISION_KEYS - if child_is_decision and key in _COMPARISON_KINDS: - signatures.add(json.dumps({key: child}, sort_keys=True, separators=(",", ":"))) - signatures = _collect_signatures( - value=child, - inside_decision=child_is_decision, - signatures=signatures, - ) - return signatures diff --git a/src/sqlbuild/rule_engine/_helpers/sql/model_name.py b/src/sqlbuild/rule_engine/_helpers/sql/model_name.py deleted file mode 100644 index ebd163f717..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/model_name.py +++ /dev/null @@ -1,52 +0,0 @@ -"""SQLBuild model-name grammar parsing.""" - -from __future__ import annotations - -import re - -from sqlbuild.rule_engine.constants import MAX_ENTITY_PARTS, MODEL_NAME_PART_COUNT -from sqlbuild.rule_engine.models import ModelNameParts - -_LAYERS: tuple[str, ...] = ( - "int_enriched", - "int_clean", - "stg_v", - "mart_v", - "int_v", - "stg", - "mart", -) -_TOKEN_PATTERN: re.Pattern[str] = re.compile(r"^[a-z][a-z0-9_]*$") - - -def parse_model_name(name: str) -> ModelNameParts | None: - """Parse the closed rule model-name grammar.""" - - parts: list[str] = name.split("__") - if len(parts) < MODEL_NAME_PART_COUNT or not _TOKEN_PATTERN.fullmatch(parts[0]): - return None - domain: str = parts[0] - tail: str = "__".join(parts[1:]) - for layer in _LAYERS: - prefix: str = f"{layer}__" - if tail.startswith(prefix): - entity_parts: list[str] = tail[len(prefix) :].split("__") - if len(entity_parts) > MAX_ENTITY_PARTS or any( - not _TOKEN_PATTERN.fullmatch(part) for part in entity_parts - ): - return None - return ModelNameParts( - domain=domain, - layer=layer, - entity=entity_parts[0], - source=entity_parts[1] if len(entity_parts) > 1 else None, - is_view=layer.endswith("_v"), - ) - return None - - -def apparent_layer(name: str) -> str | None: - """Return the raw layer-looking token for invalid-name diagnostics.""" - - parts: list[str] = name.split("__", maxsplit=2) - return parts[1] if len(parts) == MODEL_NAME_PART_COUNT else None diff --git a/src/sqlbuild/rule_engine/_helpers/sql/passthrough.py b/src/sqlbuild/rule_engine/_helpers/sql/passthrough.py deleted file mode 100644 index bc07fb4bce..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/sql/passthrough.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Shared shape-only passthrough classification.""" - -from __future__ import annotations - -import re -from typing import Any, cast - -from sqlbuild.rule_engine._helpers.sql.ast import kind, name_value, payload, top_level_ctes -from sqlbuild.rule_engine.constants import ( - AST_ALIAS_KIND, - AST_COLUMN_KIND, - AST_FUNCTION_KIND, - AST_SELECT_KIND, - AST_STAR_KIND, - AST_TABLE_KIND, - DEPENDENCY_FUNCTIONS, -) - -_REFERENCE_PATTERN: re.Pattern[str] = re.compile(r"__(?:ref|source)\s*\(", re.IGNORECASE) -_FORBIDDEN_KINDS: frozenset[str] = frozenset( - {"aggfunc", "case", "group", "join", "window", "windowfunc"} -) - - -def is_passthrough_ast(*, ast: Any, source: str) -> bool: - """Return whether a model has the narrow import-and-project passthrough shape.""" - - ctes: tuple[Any, ...] = top_level_ctes(ast) - if len(ctes) != 1 or len(_REFERENCE_PATTERN.findall(source)) != 1: - return False - import_body: Any = ctes[0][1] - if not is_dependency_import(node=import_body) or not is_plain_projection( - select=import_body, allow_star=True - ): - return False - terminal_source: dict[str, object] | None = _sole_from_expression(select=ast) - if terminal_source is None or AST_TABLE_KIND not in terminal_source: - return False - table_payload: object = terminal_source.get(AST_TABLE_KIND) - if not isinstance(table_payload, dict): - return False - table_name: str = name_value(cast(dict[str, object], table_payload).get("name")) - return table_name == str(ctes[0][0]) and is_plain_projection(select=ast, allow_star=True) - - -def is_dependency_import(*, node: Any) -> bool: - """Return whether one select is a projection from exactly one dependency macro.""" - - source: dict[str, object] | None = _sole_from_expression(select=node) - if source is None or AST_FUNCTION_KIND not in source: - return False - function_payload: object = source.get(AST_FUNCTION_KIND) - if not isinstance(function_payload, dict): - return False - function_name: object = cast(dict[str, object], function_payload).get("name") - data: dict[str, object] = payload(node) - has_forbidden_shape: bool = any( - ( - data.get("group_by"), - data.get("having"), - data.get("joins"), - data.get("qualify"), - data.get("with"), - ) - ) - return ( - str(function_name).lower() in DEPENDENCY_FUNCTIONS - and not has_forbidden_shape - and _has_plain_expressions(select=node, allow_star=True) - ) - - -def is_plain_projection(*, select: Any, allow_star: bool) -> bool: - """Return whether one select contains only a direct column projection.""" - - if kind(select) != AST_SELECT_KIND: - return False - data: dict[str, object] = payload(select) - if data.get("where_clause") is not None or data.get("having") is not None: - return False - if data.get("qualify") is not None or data.get("group_by") is not None: - return False - if data.get("joins"): - return False - for node in select.walk(): - if kind(node) in _FORBIDDEN_KINDS: - return False - return _has_plain_expressions(select=select, allow_star=allow_star) - - -def _has_plain_expressions(*, select: Any, allow_star: bool) -> bool: - expressions: tuple[Any, ...] = tuple(select.expressions) - if not expressions: - return False - if len(expressions) == 1 and kind(expressions[0]) == AST_STAR_KIND: - return allow_star - for expression in expressions: - expression_kind: str = kind(expression) - if expression_kind == AST_COLUMN_KIND: - continue - if expression_kind == AST_ALIAS_KIND: - children: tuple[Any, ...] = tuple(expression.children()) - if children and kind(children[0]) == AST_COLUMN_KIND: - continue - return False - return True - - -def _sole_from_expression(*, select: Any) -> dict[str, object] | None: - data: dict[str, object] = payload(select) - from_payload: object = data.get("from") - if not isinstance(from_payload, dict): - return None - raw_expressions: object = cast(dict[str, object], from_payload).get("expressions") - if not isinstance(raw_expressions, list) or len(raw_expressions) != 1: - return None - expression: object = raw_expressions[0] - return cast(dict[str, object], expression) if isinstance(expression, dict) else None diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index e435ae04b8..ed7084a29b 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -15,44 +15,21 @@ SKILL_STALE: str = "stale" REPLACEABLE_SKILL_STATES: frozenset[str] = frozenset({SKILL_FRESH, SKILL_MISSING, SKILL_STALE}) -AST_COLUMN_KIND: str = "column" -AST_ALIAS_KIND: str = "alias" -AST_LITERAL_KIND: str = "literal" AST_SELECT_KIND: str = "select" -AST_STAR_KIND: str = "star" -AST_FUNCTION_KIND: str = "function" -AST_TABLE_KIND: str = "table" AST_CTE_KIND: str = "cte" -DEPENDENCY_FUNCTIONS: frozenset[str] = frozenset({"__ref", "__source"}) -CROSS_JOIN_KIND: str = "CROSS" CONTRACT_ENFORCED: str = "enforced" CONTRACT_BOOLEAN_NAME_CODE: str = "SQBRCONTRACT102" CONTRACT_TIMESTAMP_NAME_CODE: str = "SQBRCONTRACT103" CONTRACT_DATE_NAME_CODE: str = "SQBRCONTRACT104" EMPTY_INPUT_TEST_CODE: str = "SQBRTEST203" -MATERIALIZED_VIEW: str = "view" -REFERENCE_KIND_REF: str = "ref" -LAYER_INT_CLEAN: str = "int_clean" -LAYER_INT_ENRICHED: str = "int_enriched" -LAYER_INT_VIEW: str = "int_v" BOOLEAN_TYPE: str = "BOOLEAN" DATE_TYPE: str = "DATE" TIMESTAMP_TYPE: str = "TIMESTAMP" -CANONICAL_NUMERIC_LITERALS: frozenset[str] = frozenset({"-1", "0", "1"}) -FINAL_CTE_NAMES: frozenset[str] = frozenset({"final", "final_cte"}) -SET_OPERATION_BY_NAME: str = " BY NAME" -MODEL_NAME_PART_COUNT: int = 3 NAMED_CTE_TUPLE_SIZE: int = 2 -MAX_ENTITY_PARTS: int = 2 -DECLARATION_DOMAIN_PART_COUNT: int = 3 TARGET_DIRECTORY_NAME: str = "target" -SQL_LINE_COMMENT: str = "--" -SQL_BLOCK_COMMENT: str = "/*" -EVALUATE_RULE_CALL: str = "evaluate_rule" -CANONICAL_ONE: str = "1" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" RULE_CONTEXT_PROJECT_FACT: str = "project" @@ -61,7 +38,6 @@ RULE_CONTEXT_AUDITS_FACT: str = "audits" RULE_CONTEXT_DECLARATIONS_FACT: str = "declarations" PARENT_DIRECTORY_TOKEN: str = ".." -TRACKED_RULES_FILE_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) MIN_AUDITS_PER_MODEL: str = "min_audits_per_model" MIN_TESTS_PER_MODEL: str = "min_tests_per_model" diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index f5f7df34b0..a05beb6bfe 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -351,15 +351,6 @@ class ResolvedRuleset: cacheable: bool -@dataclass(frozen=True) -class ModelNameParts: - domain: str - layer: str - entity: str - source: str | None - is_view: bool - - @dataclass(frozen=True) class RuleFile: path: str diff --git a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py index 28b877e083..034bb04a19 100644 --- a/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py +++ b/src/sqlbuild/runtime/event_exporting/classes/dispatcher.py @@ -155,10 +155,6 @@ def thread(self) -> threading.Thread: def notification_thread(self) -> threading.Thread | None: return self._notification_thread - @property - def health_thread(self) -> threading.Thread | None: - return self._health_thread - def enqueue(self, event: LifecycleEvent) -> None: """Filter and enqueue eligible exporter attempts without waiting.""" diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py index 9c174147ec..7f822ee5a0 100644 --- a/src/sqlbuild/spec/contracts/types.py +++ b/src/sqlbuild/spec/contracts/types.py @@ -4,8 +4,6 @@ from enum import StrEnum -type PathDefaultsMap = dict[str, dict[str, object]] - class EventExportSeverity(StrEnum): """Severity used to filter exported lifecycle events.""" diff --git a/src/sqlbuild/sql_values/types.py b/src/sqlbuild/sql_values/types.py index 834f74fd91..8636af335a 100644 --- a/src/sqlbuild/sql_values/types.py +++ b/src/sqlbuild/sql_values/types.py @@ -41,4 +41,3 @@ class StateSqlValueType(StrEnum): type SqlScalar = str | int | bool | float | Decimal | None type SqlValuePayload = SqlScalar | tuple[object, ...] -type SqlCollectionRendering = CollectionRendering diff --git a/src/sqlbuild/virtual/diff/_helpers/diff.py b/src/sqlbuild/virtual/diff/_helpers/diff.py index d7d5aa7b60..cc5519b35a 100644 --- a/src/sqlbuild/virtual/diff/_helpers/diff.py +++ b/src/sqlbuild/virtual/diff/_helpers/diff.py @@ -23,8 +23,10 @@ from sqlbuild.virtual.planner.main._semantics import build_virtual_plan_semantics from sqlbuild.virtual.planner.main._targets import build_virtual_destination_from_physical_relation from sqlbuild.virtual.planner.models import VirtualPlanSemantics +from sqlbuild.virtual.state.main.environments._ref_model_versions import ( + read_ref_model_versions, +) from sqlbuild.virtual.state.models import ( - ModelVersionRecord, PhysicalRelationRecord, VirtualEnvironmentModelRefRecord, VirtualEnvironmentRecord, @@ -177,7 +179,7 @@ def read_virtual_diff_state( from_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( graph=graph, bound_refs=from_refs, - bound_model_versions=_read_model_versions( + bound_model_versions=read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, @@ -192,7 +194,7 @@ def read_virtual_diff_state( to_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( graph=graph, bound_refs=to_refs, - bound_model_versions=_read_model_versions( + bound_model_versions=read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, @@ -240,24 +242,6 @@ def read_virtual_diff_state( ) -def _read_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, ModelVersionRecord | None]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } - - def execute_virtual_diff_between_relations( *, adapter: BaseAdapter, diff --git a/src/sqlbuild/virtual/executor/_helpers/build.py b/src/sqlbuild/virtual/executor/_helpers/build.py index 3a61334f74..2af08b9849 100644 --- a/src/sqlbuild/virtual/executor/_helpers/build.py +++ b/src/sqlbuild/virtual/executor/_helpers/build.py @@ -375,7 +375,6 @@ def run_virtual_build( return VirtualBuildPipelineResult( project=rewritten.project, - direct_plan_output=plan.plan_output, display_plan_output=plan.plan_output, execution_plan=plan.executor_plan_output, execution_result=result, diff --git a/src/sqlbuild/virtual/executor/_helpers/promote.py b/src/sqlbuild/virtual/executor/_helpers/promote.py index 2bde285f97..ef23e8b9d9 100644 --- a/src/sqlbuild/virtual/executor/_helpers/promote.py +++ b/src/sqlbuild/virtual/executor/_helpers/promote.py @@ -25,6 +25,9 @@ from sqlbuild.virtual.state.main.checkpoints._checkpoints import ( create_finalized_virtual_environment_checkpoint, ) +from sqlbuild.virtual.state.main.environments._ref_model_versions import ( + read_ref_model_versions, +) from sqlbuild.virtual.state.models import ( FunctionVersionRecord, ModelVersionRecord, @@ -152,13 +155,13 @@ def build_promote_semantics( ) -> PromoteSemantics: """Build source and target virtual plan semantics from bound refs.""" - source_versions: dict[str, ModelVersionRecord | None] = _read_model_versions( + source_versions: dict[str, ModelVersionRecord | None] = read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, refs=environment_state.source_refs, ) - target_versions: dict[str, ModelVersionRecord | None] = _read_model_versions( + target_versions: dict[str, ModelVersionRecord | None] = read_ref_model_versions( backend=backend, state_connection=state_connection, schema=schema, @@ -510,24 +513,6 @@ def _stale_models_after_promotion( ) -def _read_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, ModelVersionRecord | None]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } - - def _read_physical_relations( *, backend: Any, diff --git a/src/sqlbuild/virtual/executor/_helpers/rollback.py b/src/sqlbuild/virtual/executor/_helpers/rollback.py index 3c869f7a12..198331aa0c 100644 --- a/src/sqlbuild/virtual/executor/_helpers/rollback.py +++ b/src/sqlbuild/virtual/executor/_helpers/rollback.py @@ -168,24 +168,6 @@ def guard_partial_rollback_scope( return tuple(sorted({*selected_model_names, *stale_upstream_set})), final_version_hashes -def read_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, Any]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } - - def read_physical_relations( *, backend: Any, diff --git a/src/sqlbuild/virtual/executor/models.py b/src/sqlbuild/virtual/executor/models.py index 39dc9240a9..9f88f4e69a 100644 --- a/src/sqlbuild/virtual/executor/models.py +++ b/src/sqlbuild/virtual/executor/models.py @@ -152,7 +152,6 @@ class VirtualBuildPipelineResult: """Result returned by the virtual build pipeline.""" project: CompiledProject - direct_plan_output: PlanOutput display_plan_output: PlanOutput execution_plan: PlanOutput execution_result: BuildExecutionResult diff --git a/src/sqlbuild/virtual/planner/_helpers/state_metadata.py b/src/sqlbuild/virtual/planner/_helpers/state_metadata.py index 42bd023651..c88945b3a8 100644 --- a/src/sqlbuild/virtual/planner/_helpers/state_metadata.py +++ b/src/sqlbuild/virtual/planner/_helpers/state_metadata.py @@ -2,11 +2,8 @@ from __future__ import annotations -from typing import Any - -from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.virtual.state.main.encoding._decode_state_text import decode_state_text -from sqlbuild.virtual.state.models import FunctionVersionRecord, ModelVersionRecord +from sqlbuild.virtual.state.models import ModelVersionRecord def decode_model_version_query_sqls( @@ -39,41 +36,3 @@ def decode_model_version_metadata_jsons( continue result[model_name] = metadata_json return result - - -def read_previous_function_query_sqls( - *, - backend: Any, - state_connection: Any, - schema: str, - graph: ProjectGraph, - virtual_environment_name: str, -) -> dict[str, str]: - """Read persisted virtual function fingerprint SQL by function name.""" - - function_refs: dict[str, str] = { - ref.function_name: ref.version_hash - for ref in backend.get_virtual_environment_function_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - } - function_versions: dict[str, FunctionVersionRecord | None] = { - function.name: backend.get_function_version( - connection=state_connection, - schema=schema, - function_name=function.name, - version_hash=function_refs[function.name], - ) - for function in graph.project.functions - if function.name in function_refs - } - result: dict[str, str] = {} - for function_name, function_version in function_versions.items(): - if function_version is None: - continue - query_sql: str | None = decode_state_text(function_version.definition_text_b64) - if query_sql is not None: - result[function_name] = query_sql - return result diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py b/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py index 994a954e95..4eab0e47fd 100644 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py +++ b/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py @@ -6,7 +6,6 @@ from typing import Any from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.constants import STATE_MIGRATION_LOCK_KEY from sqlbuild.virtual.state.models import StateLockLease @@ -69,28 +68,6 @@ def acquire_model_version_lock( ) -def acquire_state_migration_lock( - *, - backend: StateBackend, - connection: Any, - schema: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire the lock for state schema lifecycle operations.""" - - return acquire_state_lock( - backend=backend, - connection=connection, - schema=schema, - lock_key=STATE_MIGRATION_LOCK_KEY, - owner_id=owner_id, - ttl=ttl, - now=now, - ) - - def acquire_state_lock( *, backend: StateBackend, diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py b/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py index 26040a051d..a9acb526ac 100644 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py +++ b/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py @@ -13,20 +13,6 @@ from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -def append_duckdb_microbatch_event( - *, connection: Any, qualified_table: str, event: MicrobatchEvent -) -> None: - """Append one idempotent logical event to DuckDB state.""" - - placeholders: str = ", ".join("?" for _ in MICROBATCH_COLUMNS) - connection.execute( - f"INSERT INTO {qualified_table} ({', '.join(MICROBATCH_COLUMNS)}) " - f"SELECT {placeholders} WHERE NOT EXISTS " - f"(SELECT 1 FROM {qualified_table} WHERE event_id = ?)", - [*MicrobatchEventCodec.values(event), event.event_id], - ) - - def append_duckdb_microbatch_events( *, connection: Any, qualified_table: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: diff --git a/src/sqlbuild/virtual/state/classes/_sql_state_backend.py b/src/sqlbuild/virtual/state/classes/_sql_state_backend.py index 0606be0831..c31a45cbf2 100644 --- a/src/sqlbuild/virtual/state/classes/_sql_state_backend.py +++ b/src/sqlbuild/virtual/state/classes/_sql_state_backend.py @@ -8,7 +8,6 @@ from datetime import datetime from typing import Any, ClassVar, cast -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.node_results.main.decode_json import decode_node_result_json from sqlbuild.executor.node_results.main.encode_json import encode_node_result_json from sqlbuild.executor.node_results.models import ( @@ -446,66 +445,6 @@ def replace_virtual_environment_model_refs( ), ) - def replace_virtual_environment_function_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - ) -> None: - ref: VirtualEnvironmentFunctionRefRecord - for ref in refs: - if ref.node_type not in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - raise StateBackendConfigError("Function ref node_type must be 'udf' or 'table_fn'") - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] = {} - for node_type in ("udf", "table_fn"): - node_refs: list[VirtualEnvironmentNodeRefRecord] = [] - for ref in refs: - if ref.node_type == node_type: - node_refs.append( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - ) - refs_by_node_type[node_type] = tuple(node_refs) - self.replace_virtual_environment_node_ref_groups( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - refs_by_node_type=refs_by_node_type, - ) - - def replace_virtual_environment_seed_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - ) -> None: - self.replace_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="seed", - refs=tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="seed", - node_name=ref.seed_name, - version_hash=ref.version_hash, - ) - for ref in refs - ), - ) - def upsert_virtual_environment_python_node_ref( self, *, @@ -1421,28 +1360,6 @@ def list_physical_relations_for_artifact( for row in rows ) - def get_physical_relation_ancestry( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationAncestryRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT model_name, version_hash, parent_model_name, parent_version_hash, " - "seed_strategy " - f"FROM {self._qualified_name(schema=schema, table=PHYSICAL_RELATION_ANCESTRY_TABLE)} " - f"WHERE model_name = {p} AND version_hash = {p}", - params=[model_name, version_hash], - ) - if row is None: - return None - return PhysicalRelationAncestryRecord( - model_name=row[0], - version_hash=row[1], - parent_model_name=row[2], - parent_version_hash=row[3], - seed_strategy=row[4], - ) - def get_virtual_environment( self, *, connection: Any, schema: str, virtual_environment_name: str ) -> VirtualEnvironmentRecord | None: diff --git a/src/sqlbuild/virtual/state/classes/duckdb.py b/src/sqlbuild/virtual/state/classes/duckdb.py index d4edb7945f..0f04be4598 100644 --- a/src/sqlbuild/virtual/state/classes/duckdb.py +++ b/src/sqlbuild/virtual/state/classes/duckdb.py @@ -13,7 +13,6 @@ to_naive_utc_wall_clock, ) from sqlbuild.virtual.state._helpers.state_storage.microbatch_events import ( - append_duckdb_microbatch_event, append_duckdb_microbatch_events, read_duckdb_microbatch_model_history, read_duckdb_microbatch_retention_history, @@ -140,15 +139,6 @@ def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidati existing_indexes_by_table=indexes_by_table, ) - def append_microbatch_event( - self, *, connection: Any, schema: str, event: MicrobatchEvent - ) -> None: - append_duckdb_microbatch_event( - connection=connection, - qualified_table=self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE), - event=event, - ) - def append_microbatch_events( self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: diff --git a/src/sqlbuild/virtual/state/classes/postgres.py b/src/sqlbuild/virtual/state/classes/postgres.py index 867e80760e..ac11fb12ec 100644 --- a/src/sqlbuild/virtual/state/classes/postgres.py +++ b/src/sqlbuild/virtual/state/classes/postgres.py @@ -165,20 +165,6 @@ def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidati existing_indexes_by_table=indexes_by_table, ) - def append_microbatch_event( - self, *, connection: Any, schema: str, event: MicrobatchEvent - ) -> None: - placeholders: str = ", ".join("%s" for _ in MICROBATCH_COLUMNS) - with connection.cursor() as cursor: - cursor.execute( - f"INSERT INTO {self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - f"({', '.join(MICROBATCH_COLUMNS)}) SELECT {placeholders} " - "WHERE NOT EXISTS (SELECT 1 FROM " - f"{self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - "WHERE event_id = %s)", - [*MicrobatchEventCodec.values(event), event.event_id], - ) - def append_microbatch_events( self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: @@ -196,7 +182,18 @@ def append_microbatch_events( event for event in events if event.event_id not in existing_ids ) if missing: - row_placeholders: str = "(" + ", ".join("%s" for _ in MICROBATCH_COLUMNS) + ")" + row_placeholders: str = ( + "(" + + ", ".join( + f"%s::{ + self._state_column_sql_type( + STATE_TABLE_COLUMNS[MICROBATCH_EVENT_TABLE][column] + ) + }" + for column in MICROBATCH_COLUMNS + ) + + ")" + ) values_sql: str = ", ".join(row_placeholders for _ in missing) columns: str = ", ".join(MICROBATCH_COLUMNS) params: list[object | None] = [] diff --git a/src/sqlbuild/virtual/state/classes/state_backend.py b/src/sqlbuild/virtual/state/classes/state_backend.py index c8ea885ac3..45bcf63860 100644 --- a/src/sqlbuild/virtual/state/classes/state_backend.py +++ b/src/sqlbuild/virtual/state/classes/state_backend.py @@ -167,12 +167,6 @@ def read_node_results( """Read runtime node result rows for one virtual environment identity.""" ... - def append_microbatch_event( - self, *, connection: Any, schema: str, event: MicrobatchEvent - ) -> None: - """Append one logical microbatch event to virtual state.""" - raise MicrobatchStateError("virtual state backend does not support microbatch events") - def append_microbatch_events( self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] ) -> MicrobatchWriteResult: @@ -259,13 +253,6 @@ def upsert_physical_relation_ancestry( """Insert or replace physical relation ancestry.""" ... - @abstractmethod - def get_physical_relation_ancestry( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationAncestryRecord | None: - """Return physical relation ancestry if it exists.""" - ... - @abstractmethod def upsert_virtual_environment( self, *, connection: Any, schema: str, record: VirtualEnvironmentRecord @@ -390,18 +377,6 @@ def get_virtual_environment_model_refs( """Return refs for a virtual environment.""" ... - @abstractmethod - def replace_virtual_environment_function_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - ) -> None: - """Replace all function refs for a virtual environment.""" - ... - @abstractmethod def get_virtual_environment_function_refs( self, *, connection: Any, schema: str, virtual_environment_name: str @@ -409,18 +384,6 @@ def get_virtual_environment_function_refs( """Return function refs for a virtual environment.""" ... - @abstractmethod - def replace_virtual_environment_seed_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - ) -> None: - """Replace all seed refs for a virtual environment.""" - ... - @abstractmethod def get_virtual_environment_seed_refs( self, *, connection: Any, schema: str, virtual_environment_name: str diff --git a/src/sqlbuild/virtual/state/constants.py b/src/sqlbuild/virtual/state/constants.py index f62be969f3..309451c7fb 100644 --- a/src/sqlbuild/virtual/state/constants.py +++ b/src/sqlbuild/virtual/state/constants.py @@ -33,7 +33,6 @@ ) VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE: str = "virtual_environment_checkpoint_seed_refs" LOCK_TABLE: str = "locks" -STATE_MIGRATION_LOCK_KEY: str = "state_migration" STATE_OPERATION_TABLE: str = "state_operations" PLAN_RUN_TABLE: str = "plan_runs" VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_TABLE: str = "virtual_environment_model_ref_events" @@ -243,7 +242,6 @@ "observed_at": StateColumnType.TIMESTAMP, "updated_at": StateColumnType.TIMESTAMP, } -SOURCE_FRESHNESS_NAIVE_UTC_WALL_CLOCK_COLUMNS: frozenset[str] = frozenset({"observed_at"}) VIRTUAL_ENVIRONMENT_CHECKPOINT_COLUMNS: dict[str, StateColumnType] = { "checkpoint_id": StateColumnType.TEXT, diff --git a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py b/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py deleted file mode 100644 index 295e9dec47..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_physical_relations.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Public helper for physical relations referenced by retained checkpoints.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, -) - - -def list_checkpoint_physical_relations( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, -) -> tuple[PhysicalRelationRecord, ...]: - """List physical relations referenced by retained checkpoints for a VDE.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - backend.list_virtual_environment_checkpoints( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ) - ) - relations: dict[tuple[str | None, str | None, str], PhysicalRelationRecord] = {} - checkpoint: VirtualEnvironmentCheckpointRecord - for checkpoint in checkpoints: - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, - schema=config.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - ref: VirtualEnvironmentCheckpointModelRefRecord - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is None: - continue - relations[ - (relation.database_name, relation.schema_name, relation.relation_name) - ] = relation - return tuple(relations[key] for key in sorted(relations)) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py b/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py new file mode 100644 index 0000000000..5dba51746c --- /dev/null +++ b/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py @@ -0,0 +1,27 @@ +"""Public helper resolving model-version records for environment refs.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.virtual.state.models import ModelVersionRecord, VirtualEnvironmentModelRefRecord + + +def read_ref_model_versions( + *, + backend: Any, + state_connection: Any, + schema: str, + refs: tuple[VirtualEnvironmentModelRefRecord, ...], +) -> dict[str, ModelVersionRecord | None]: + """Return the model-version record bound by each ref, keyed by model name.""" + + return { + ref.model_name: backend.get_model_version( + connection=state_connection, + schema=schema, + model_name=ref.model_name, + version_hash=ref.version_hash, + ) + for ref in refs + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 5ae694aeed..03d490842f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -918,14 +918,6 @@ class SchemaBackfillBuildE2ETestCase: expected_warning_entries: tuple[tuple[str, str], ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DirectReuseFromBuildE2ETestCase: - description: str - project_name: str - expected_prod_build_exit_code: int - expected_dev_build_exit_code: int - - @dataclass(frozen=True) class MixedTimestampGrainBuildE2ETestCase: """Test case for mixed timestamp grain replay behavior.""" @@ -1109,3 +1101,15 @@ class SqlHookIdentityBuildE2ETestCase: added_hook_path: str added_hook_contents: str expected_reason: str + + +@dataclass(frozen=True) +class PythonNodeIdentityBuildE2ETestCase: + """Test case for Python node version identity after one decorator-input edit.""" + + description: str + edited_path: str + original_text: str + edited_text: str + expected_asset_identity_status: str + expected_loader_version_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py new file mode 100644 index 0000000000..f67b193c43 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -0,0 +1,149 @@ +"""E2E tests for Python node version identity across decorator-input edits.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + PythonNodeIdentityBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_LOADERS_PATH: str = "loaders/orders.py" +_ASSETS_PATH: str = "assets/exports.py" +_ORIGINAL_COLUMNS: str = 'LoaderColumnSpec(name="order_id", type="INTEGER")' +_ORIGINAL_RETRY: str = "RetryPolicy(max_attempts=2, retry_on=RuntimeError)" + + +@pytest.mark.parametrize( + "test_case", + [ + PythonNodeIdentityBuildE2ETestCase( + description="loader declared column edit records a new loader version", + edited_path=_LOADERS_PATH, + original_text=_ORIGINAL_COLUMNS, + edited_text=( + 'LoaderColumnSpec(name="order_id", type="INTEGER", ' + 'description="Staged order identifier")' + ), + expected_asset_identity_status="unchanged", + expected_loader_version_count=2, + ), + PythonNodeIdentityBuildE2ETestCase( + description="asset retry edit keeps every version unchanged", + edited_path=_ASSETS_PATH, + original_text=_ORIGINAL_RETRY, + edited_text="RetryPolicy(max_attempts=5, retry_on=RuntimeError)", + expected_asset_identity_status="unchanged", + expected_loader_version_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_built_python_nodes_when_editing_decorator_inputs_then_identity_tracks_columns( + test_case: PythonNodeIdentityBuildE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_node_identity_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "python_node_identity_project" + adapter = "duckdb" + + [connection] + database = "warehouse.duckdb" + """ + ).strip() + + "\n", + _LOADERS_PATH: dedent( + f""" + from sqlbuild.loaders import LoaderColumnSpec, loader + + STAGED_ORDER_COLUMNS = [{_ORIGINAL_COLUMNS}] + + + @loader(write_strategy="table", columns=STAGED_ORDER_COLUMNS) + def staged_orders(ctx): + return [{{"order_id": 1}}] + + + @loader(depends_on=[staged_orders]) + def raw_orders(ctx): + staged = ctx.loader(staged_orders).destination + rows = ctx.query(f"SELECT order_id FROM {{staged}}") + return [{{"order_id": row[0]}} for row in rows] + """ + ).lstrip(), + "sources/raw.yml": dedent( + """ + sources: + - name: raw_orders + managed: true + write_strategy: table + columns: + - name: order_id + type: INTEGER + """ + ).lstrip(), + _ASSETS_PATH: dedent( + f""" + from sqlbuild.assets import asset + from sqlbuild.retries import RetryPolicy + + EXPORT_RETRY = {_ORIGINAL_RETRY} + + + @asset(retry=EXPORT_RETRY) + def export_orders(ctx): + return None + """ + ).lstrip(), + }, + ) + first_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first_build.returncode == 0, first_build.stdout + first_build.stderr + + edited: Path = project_dir / test_case.edited_path + edited.write_text( + edited.read_text(encoding="utf-8").replace(test_case.original_text, test_case.edited_text), + encoding="utf-8", + ) + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("plan", "--json"), project_dir=project_dir + ) + second_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert second_build.returncode == 0, second_build.stdout + second_build.stderr + payload: dict[str, object] = json.loads(plan.stdout) + statuses_by_name: dict[str, str] = { + str(entry["name"]): str(entry["identity_status"]) for entry in payload["python_nodes"] + } + assert statuses_by_name["export_orders"] == test_case.expected_asset_identity_status + assert query_duckdb( + db_path=project_dir / "warehouse.duckdb", + sql=( + "SELECT COUNT(DISTINCT version_hash) FROM main._sqlbuild_fingerprints " + "WHERE node_type = 'loader' AND node_name = 'staged_orders'" + ), + ) == [(test_case.expected_loader_version_count,)] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index faa1412494..60d4fa93bd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -1,88 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class DirectPlanE2ETestCase: - description: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class DirectPlanJsonE2ETestCase: - description: str - expected_selected_count: int - expected_model_count: int - expected_function_count: int - - -@dataclass(frozen=True) -class DirectChangesOnlyJsonE2ETestCase: - description: str - expected_reasons_by_name: dict[str, str] - expected_identity_status_by_name: dict[str, str] - - -@dataclass(frozen=True) -class DirectPlanIdentityJsonE2ETestCase: - description: str - project_name: str - expected_identity_status: str - expected_built_version_present: bool - - -@dataclass(frozen=True) -class DirectChangesOnlyReplayE2ETestCase: - description: str - policy_fragment: str - expected_backfill_action: str - expected_backfill_duration: str | None - - -@dataclass(frozen=True) -class DirectChangesOnlySchemaReplayE2ETestCase: - description: str - expected_reason: str - expected_backfill_action: str - expected_backfill_duration: str | None - - -@dataclass(frozen=True) -class DirectChangesOnlyFunctionReplayE2ETestCase: - description: str - expected_reason: str - expected_backfill_action: str - expected_backfill_duration: str | None - - -@dataclass(frozen=True) -class DirectChangesOnlySelectorE2ETestCase: - description: str - selector: str - expected_selected_count: int - expected_model_names: tuple[str, ...] - unexpected_model_names: tuple[str, ...] - expected_remaining_stale_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class DirectFunctionIdentityE2ETestCase: - description: str - expected_initial_count: int - expected_changed_count: int - expected_model_name: str - - -@dataclass(frozen=True) -class DirectFunctionSelectorE2ETestCase: - description: str - selector: str - expected_plan_fragment: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] - expected_remaining_stale_names: tuple[str, ...] = () - - @dataclass(frozen=True) class DirectModeVirtualFlagGuardE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index de8b43bd2f..10f65d581e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -66,15 +66,6 @@ class PostgresLoaderWaffleShopE2ETestCase: expected_return_code: int = 0 -@dataclass(frozen=True) -class PostgresDiffE2ETestCase: - description: str - mutation_sql: tuple[str, ...] - command: tuple[str, ...] - expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_return_code: int = 0 - - @dataclass(frozen=True) class PostgresSnapshotE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index d75c81bf04..b78f987df6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -176,3 +176,13 @@ class CursorWindowE2ETestCase: expected_exit_code: int expected_output_fragments: tuple[str, ...] expected_compiled_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SetOperationExpectedE2ETestCase: + """SQL tests whose expected CTEs combine branches with set operations.""" + + description: str + test_files: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py new file mode 100644 index 0000000000..32378e33a7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_set_operation_expected.py @@ -0,0 +1,136 @@ +"""E2E coverage for INTERSECT and EXCEPT in SQL-test expected CTEs.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + SetOperationExpectedE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "set_operation_expected"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "set_operation_expected.duckdb"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + "macros/status.py": ( + 'def normalize_status(value: str) -> str:\n return f"LOWER(TRIM({value}))"\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: status\n type: VARCHAR\n" + ), + "models/orders.sql": ( + "MODEL ();\n\n" + 'SELECT order_id, @normalize_status("status") AS status FROM __source("raw_orders")\n' + ), +} +_SOURCE_MOCK: str = ( + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, ' PAID ' AS status\n" + " UNION ALL SELECT 2 AS order_id, 'void' AS status\n" + "),\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SetOperationExpectedE2ETestCase( + description="model and macro tests pass with intersect and except", + test_files={ + "tests/unit/test_orders_except.sql": ( + 'TEST (name "orders_except");\n\nWITH\n' + + _SOURCE_MOCK + + "__expected__orders AS (\n" + " SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n" + " UNION ALL SELECT 3 AS order_id\n" + " EXCEPT SELECT 3 AS order_id\n" + ")\nSELECT 1\n" + ), + "tests/unit/test_orders_intersect.sql": ( + 'TEST (name "orders_intersect");\n\nWITH\n' + + _SOURCE_MOCK + + "__expected__orders AS (\n" + " SELECT order_id FROM (VALUES (1), (2), (3)) AS candidates(order_id)\n" + " INTERSECT ALL\n" + " SELECT order_id FROM (VALUES (1), (2)) AS loaded(order_id)\n" + ")\nSELECT 1\n" + ), + "tests/unit/test_normalize_status.sql": ( + 'TEST (mode macro, name "normalizes_status");\n\nWITH\n' + "input_values AS (SELECT ' PAID ' AS raw_status),\n" + "__macro_actual__ AS (\n" + ' SELECT @normalize_status("raw_status") AS status FROM input_values\n' + "),\n" + "__macro_expected__ AS (\n" + " SELECT 'paid' AS status\n" + " INTERSECT DISTINCT SELECT 'paid' AS status\n" + " EXCEPT SELECT 'void' AS status\n" + ")\nSELECT 1\n" + ), + }, + expected_exit_code=0, + expected_output_fragments=("PASS=3", "FAIL=0"), + ), + SetOperationExpectedE2ETestCase( + description="model test with a mismatched intersect branch fails clearly", + test_files={ + "tests/unit/test_orders_mismatch.sql": ( + 'TEST (name "orders_mismatch");\n\nWITH\n' + + _SOURCE_MOCK + + "__expected__orders AS (\n" + " SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id\n" + ")\nSELECT 1\n" + ), + }, + expected_exit_code=1, + expected_output_fragments=( + "SQL test 'tests/unit/test_orders_mismatch.sql' must use the same " + "__expected__orders projection names and order in every set-operation " + "branch; branch 2 does not match branch 1", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_set_operation_expected_ctes_when_testing_then_every_branch_is_checked( + test_case: SetOperationExpectedE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="set_operation_expected", + repo_files={**_PROJECT_FILES, **test_case.test_files}, + ) + execute_duckdb( + db_path=project_dir / "set_operation_expected.duckdb", + sql="CREATE TABLE raw_orders AS SELECT 1 AS order_id, 'paid' AS status", + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + + output: str = tested.stdout + tested.stderr + assert tested.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index 297f591c16..abae01ff09 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -203,17 +203,6 @@ class DiffRowsErrorTestCase: tolerances: RowDiffTolerances | None = None -@dataclass(frozen=True) -class CountRowsTestCase: - description: str - setup_sql: tuple[str, ...] - relation: str - cursor_column: str | None = None - start_cursor: CursorValue | None = None - end_cursor: CursorValue | None = None - expected_count: int = 0 - - @dataclass(frozen=True) class RecorderWriteTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py b/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py index f0c7249bd9..723bbc4323 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/helpers.py @@ -25,3 +25,12 @@ def _execute_sql(self, *, connection: Any, sql: str) -> Any: def _raise_insert_failure(self, *, connection: Any, sql: str) -> Any: del connection, sql raise RuntimeError("injected snapshot insert failure") + + +def count_relation_rows(*, adapter: DuckDbAdapter, connection: Any, relation: str) -> int: + """Return ``COUNT(*)`` for one relation through the adapter connection.""" + + result: Any = adapter.execute( + connection=connection, sql=f"SELECT COUNT(*) FROM {relation}" + ).fetchone() + return int(result[0]) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index da9977b703..2462ff668c 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -10,7 +10,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, - CursorValue, ExpressionInferenceProfile, QueryResult, RowDiffColumnResult, @@ -21,7 +20,7 @@ RowDiffTolerances, SchemaDiffResult, ) -from sqlbuild.adapter.contract.types import CursorKind, FunctionNullabilityRule +from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SeedCsvSettings @@ -29,7 +28,6 @@ CompositeRowDiffSamplingTestCase, ConnectSettingsTestCase, ConnectTestCase, - CountRowsTestCase, DeleteInsertTestCase, DiffRowsErrorTestCase, DiffRowsTestCase, @@ -50,6 +48,7 @@ SwapTestCase, TransactionalAtomicityTestCase, ) +from tests.integration.src.sqlbuild.adapters.duckdb.helpers import count_relation_rows @pytest.mark.parametrize( @@ -442,7 +441,7 @@ def test_given_sql_when_creating_table_as_then_table_is_writable( statement_recorder=StatementRecorder(), ) connection.execute("INSERT INTO result VALUES (4)") - count: int = adapter.count_rows(connection=connection, relation="result") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="result") assert count == test_case.expected_row_count @@ -561,7 +560,7 @@ def test_given_source_table_when_creating_view_then_view_reflects_source( sql="SELECT id FROM source", statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="result_view") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="result_view") assert count == test_case.expected_row_count @@ -750,7 +749,7 @@ def test_given_source_table_when_cloning_then_target_has_same_rows( destination="cloned_t", statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="cloned_t") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="cloned_t") assert count == test_case.expected_row_count @@ -783,7 +782,7 @@ def test_given_existing_table_when_appending_then_row_count_increases( sql="SELECT * FROM (VALUES (2), (3)) AS t(id)", statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="append_t") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="append_t") assert count == test_case.expected_row_count @@ -831,7 +830,7 @@ def test_given_target_when_delete_inserting_then_matching_rows_replaced( unique_key=test_case.unique_key, statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="di_target") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="di_target") updated_val: Any = connection.execute("SELECT val FROM di_target WHERE id = 1").fetchone() assert count == test_case.expected_row_count @@ -881,7 +880,9 @@ def test_given_target_and_source_when_merging_then_upserts_correctly( unique_key=test_case.unique_key, statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="merge_target") + count: int = count_relation_rows( + adapter=adapter, connection=connection, relation="merge_target" + ) rows: list[tuple[Any, ...]] = connection.execute( "SELECT id, name FROM merge_target ORDER BY id, name" ).fetchall() @@ -956,7 +957,7 @@ def test_given_csv_file_when_loading_seed_twice_then_table_is_replaced( infer_types=test_case.infer_types, statement_recorder=StatementRecorder(), ) - count: int = adapter.count_rows(connection=connection, relation="seed_table") + count: int = count_relation_rows(adapter=adapter, connection=connection, relation="seed_table") first_row: tuple[Any, ...] = connection.execute( "SELECT * FROM seed_table ORDER BY id LIMIT 1" ).fetchone() @@ -1417,53 +1418,6 @@ def test_given_invalid_tolerance_when_diffing_rows_then_raises_clear_error( ) -@pytest.mark.parametrize( - "test_case", - [ - CountRowsTestCase( - description="counts all rows without cursor filter", - setup_sql=( - "CREATE TABLE count_t (id INTEGER)", - "INSERT INTO count_t VALUES (1), (2), (3)", - ), - relation="count_t", - expected_count=3, - ), - CountRowsTestCase( - description="counts rows bounded by integer cursor", - setup_sql=( - "CREATE TABLE count_bounded (id INTEGER)", - "INSERT INTO count_bounded VALUES (1), (2), (3), (4), (5)", - ), - relation="count_bounded", - cursor_column="id", - start_cursor=CursorValue(kind=CursorKind.INTEGER, value=2), - end_cursor=CursorValue(kind=CursorKind.INTEGER, value=4), - expected_count=2, - ), - ], - ids=lambda case: case.description, -) -def test_given_table_when_counting_rows_then_returns_expected_count( - test_case: CountRowsTestCase, - adapter: DuckDbAdapter, - connection: Any, -) -> None: - statement: str - for statement in test_case.setup_sql: - connection.execute(statement) - - count: int = adapter.count_rows( - connection=connection, - relation=test_case.relation, - cursor_column=test_case.cursor_column, - start_cursor=test_case.start_cursor, - end_cursor=test_case.end_cursor, - ) - - assert count == test_case.expected_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py b/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py index b324fd21cd..0eb8197d28 100644 --- a/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/postgres/_test_types.py @@ -68,17 +68,6 @@ class PostgresRowDiffErrorTestCase: tolerances: RowDiffTolerances | None = None -@dataclass(frozen=True) -class PostgresCountRowsTestCase: - description: str - table_name: str - values_sql: str - cursor_column: str | None = None - start_cursor: CursorValue | None = None - end_cursor: CursorValue | None = None - expected_count: int = 0 - - @dataclass(frozen=True) class PostgresRowDiffSampleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py index c266718c59..18a0f9c776 100644 --- a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py @@ -23,7 +23,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from tests.integration.src.sqlbuild.adapters.postgres._test_types import ( PostgresBuildFlowTestCase, - PostgresCountRowsTestCase, PostgresMergeTestCase, PostgresQueryTestCase, PostgresRowDiffErrorTestCase, @@ -404,50 +403,6 @@ def test_given_invalid_diff_when_diffing_rows_then_postgres_raises_clear_error( ) -@pytest.mark.parametrize( - "test_case", - [ - PostgresCountRowsTestCase( - description="counts all rows without cursor filter", - table_name="count_t", - values_sql="(1), (2), (3)", - expected_count=3, - ), - PostgresCountRowsTestCase( - description="counts rows bounded by integer cursor", - table_name="count_bounded", - values_sql="(1), (2), (3), (4), (5)", - cursor_column="id", - start_cursor=CursorValue(kind=CursorKind.INTEGER, value=2), - end_cursor=CursorValue(kind=CursorKind.INTEGER, value=4), - expected_count=2, - ), - ], - ids=lambda case: case.description, -) -def test_given_table_when_counting_rows_then_postgres_returns_expected_count( - test_case: PostgresCountRowsTestCase, - adapter: PostgresAdapter, - connection: Any, - postgres_schema: str, -) -> None: - target: str = qualified_name(schema=postgres_schema, name=test_case.table_name) - adapter.execute(connection=connection, sql=f"CREATE TABLE {target} (id INTEGER)") - adapter.execute( - connection=connection, sql=f"INSERT INTO {target} VALUES {test_case.values_sql}" - ) - - count: int = adapter.count_rows( - connection=connection, - relation=target, - cursor_column=test_case.cursor_column, - start_cursor=test_case.start_cursor, - end_cursor=test_case.end_cursor, - ) - - assert count == test_case.expected_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py index f4250ed9a1..869fc34d1d 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py @@ -116,14 +116,6 @@ class SnowflakeTableTypeConversionTestCase: expected_conversion_statement_count: int -@dataclass(frozen=True) -class SnowflakeE2ETestCase: - description: str - command: tuple[str, ...] - expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_return_code: int = 0 - - @dataclass(frozen=True) class SnowflakeCostTestCase: description: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 33339a443e..26614e5391 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -149,6 +149,15 @@ class FormatCompileIntegrationTestCase: expected_literal: str +@dataclass(frozen=True) +class BacktickDialectFormatIntegrationTestCase: + """One backtick-identifier adapter whose macro call must survive formatting.""" + + description: str + adapter: str + expected_literal: str + + @dataclass(frozen=True) class DescriptionFormatIntegrationTestCase: """One description wrapping expectation through the real CLI and compiler.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py index bad46249d0..961dcb0c07 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format.py @@ -10,6 +10,7 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + BacktickDialectFormatIntegrationTestCase, CanonicalFixtureFormatIntegrationTestCase, DescriptionFormatIntegrationTestCase, FormatCompileIntegrationTestCase, @@ -926,3 +927,50 @@ def test_given_leading_cte_comment_when_formatting_then_comment_rule_still_passe assert test_case.expected_fragment in formatted_sql assert model_path.read_text(encoding="utf-8") == formatted_sql assert "SQBRSQL033" not in output.out + output.err + + +@pytest.mark.parametrize( + "test_case", + [ + BacktickDialectFormatIntegrationTestCase( + description="databricks apostrophe inside a backtick identifier keeps the macro", + adapter="databricks", + expected_literal='@label("order_id") AS order_id', + ), + BacktickDialectFormatIntegrationTestCase( + description="bigquery apostrophe inside a backtick identifier keeps the macro", + adapter="bigquery", + expected_literal='@label("order_id") AS order_id', + ), + ], + ids=lambda case: case.description, +) +def test_given_backtick_identifier_apostrophe_when_formatting_then_macro_call_still_compiles( + test_case: BacktickDialectFormatIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.adapter}"\n', encoding="utf-8" + ) + macros: Path = tmp_path / "models" / "_macros" + macros.mkdir(parents=True) + (macros / "labels.py").write_text( + 'def label(expression: str) -> str:\n """Return one column expression."""\n' + " return expression\n", + encoding="utf-8", + ) + model_path: Path = tmp_path / "models" / "orders.sql" + model_path.write_text( + 'MODEL (description "Orders", database warehouse, schema analytics);\n\n' + 'select `customer\'s id` as customer_id, @label("order_id") as order_id ' + "from raw_orders\n", + encoding="utf-8", + ) + + format_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "format"]) + compile_exit: int = main(["--project-dir", str(tmp_path), "--no-color", "compile"]) + output: CaptureResult[str] = capsys.readouterr() + + assert (format_exit, compile_exit) == (0, 0), output.out + output.err + assert test_case.expected_literal in model_path.read_text(encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py index 4f4b950d50..baef1aa160 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py @@ -95,16 +95,3 @@ class IncrementalFailureTestCase: future_cursor_config: FutureCursorsConfig | None = None invocation_time: datetime | None = None expected_has_future_cursor_safety: bool = False - - -@dataclass(frozen=True) -class IncrementalSeedReuseFailureTestCase: - description: str - origin_sql: str - input_sql: str - model_sql: str - fingerprint_version_hash: str - expected_status: ExecutionStatus - expected_failed_phase: ExecutionPhase - expected_error_fragments: tuple[str, ...] - expected_target_exists: bool diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py index f1ae320a32..27c66108f4 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py @@ -2,6 +2,7 @@ from dataclasses import dataclass +from sqlbuild.microbatches.types import MicrobatchRecordType from sqlbuild.virtual.state.types import VirtualEnvironmentStatus from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( ConditionalPublicationPayload, @@ -272,3 +273,12 @@ class DuckDbStateBackendRefContractTestCase: expected_observation: StateRefContractObservation schema: str sqlbuild_version: str + + +@dataclass(frozen=True) +class MicrobatchBulkAppendTestCase: + """One production-shaped microbatch event batch written through the bulk append.""" + + description: str + record_type: MicrobatchRecordType + expected_write_counts: tuple[int, int] = (1, 1) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py index 34300313bd..1d0ba81788 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py @@ -5,6 +5,13 @@ from pathlib import Path from typing import Any, NamedTuple +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.types import ( + MicrobatchCompletionType, + MicrobatchFingerprintStatus, + MicrobatchRecordType, + MicrobatchRunType, +) from sqlbuild.virtual.state.classes.duckdb import DuckDbStateBackend from sqlbuild.virtual.state.classes.state_backend import StateBackend from sqlbuild.virtual.state.models import ( @@ -348,14 +355,16 @@ def exercise_state_ref_contract( connection=connection, schema=schema, virtual_environment_name="dev", node_type="seed" ) ) - backend.replace_virtual_environment_function_refs( + backend.replace_virtual_environment_node_ref_groups( connection=connection, schema=schema, virtual_environment_name="dev", - refs=( - VirtualEnvironmentFunctionRefRecord("dev", "table_fn", "customer_orders", "fn-v1"), - VirtualEnvironmentFunctionRefRecord("dev", "udf", "normalize_email", "fn-v2"), - ), + refs_by_node_type={ + "table_fn": ( + VirtualEnvironmentNodeRefRecord("dev", "table_fn", "customer_orders", "fn-v1"), + ), + "udf": (VirtualEnvironmentNodeRefRecord("dev", "udf", "normalize_email", "fn-v2"),), + }, ) function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( backend.get_virtual_environment_function_refs( @@ -449,3 +458,54 @@ def exercise_state_ref_contract( deleted_model_refs=(), surviving_model_refs=(VirtualEnvironmentNodeRefRecord("qa", "model", "orders", "orders-v3"),), ) + + +def build_production_shaped_microbatch_event( + *, scope: MicrobatchScope, record_type: MicrobatchRecordType +) -> MicrobatchEvent: + """Build one event with the nullable columns the executor leaves unset for its kind. + + Replay requirements carry no completion or observation facts, and executed partition + completions carry no observation facts, so each single-event batch has timestamp and + integer columns that are NULL in every row. + """ + + run_started_at: datetime = datetime(2026, 1, 1, 9, 0, 0) + common: dict[str, Any] = { + "scope": scope, + "origin_run_id": "run-1", + "execution_run_id": "run-1", + "run_type": MicrobatchRunType.NORMAL, + "run_start": "0", + "run_end": "2", + "batch_size": "1", + "cursor_column": "batch_id", + "cursor_type": "integer", + "model_version_hash": "F2", + "definition_hash": "definition", + "fingerprint_status": MicrobatchFingerprintStatus.KNOWN, + "origin_run_started_at": run_started_at, + "execution_run_started_at": run_started_at, + "created_at": datetime(2026, 1, 1, 9, 0, 1), + } + events: dict[MicrobatchRecordType, MicrobatchEvent] = { + MicrobatchRecordType.REPLAY_REQUIREMENT: MicrobatchEvent( + event_id="requirement-1", + record_type=record_type, + replay_requirement_id="requirement-1", + required_model_version_hash="F2", + replay_policy="full", + **common, + ), + MicrobatchRecordType.PARTITION_COMPLETION: MicrobatchEvent( + event_id="completion-1", + record_type=record_type, + completion_type=MicrobatchCompletionType.INITIAL, + partition_start="0", + partition_end="1", + rows_affected=3, + completed_at=datetime(2026, 1, 1, 9, 0, 2), + **common, + ), + } + return events[record_type] diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py index f993dd7f92..a77e8bf730 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py @@ -2,6 +2,7 @@ from dataclasses import dataclass +from sqlbuild.microbatches.types import MicrobatchRecordType from sqlbuild.virtual.state.types import VirtualEnvironmentStatus from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( ConditionalPublicationPayload, @@ -9,6 +10,15 @@ ) +@dataclass(frozen=True) +class MicrobatchBulkAppendTestCase: + """One production-shaped event batch and its idempotent write counts.""" + + description: str + record_type: MicrobatchRecordType + expected_write_counts: tuple[int, int] = (1, 1) + + @dataclass(frozen=True) class PostgresConditionalVirtualRefPublishTestCase: """Expected conditional virtual-ref publication results.""" diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py index 2c97970d33..492c20ad0c 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py @@ -14,7 +14,7 @@ NodeResultRecord, ) from sqlbuild.executor.node_results.types import NodeResultStatus -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult from sqlbuild.microbatches.types import ( MicrobatchCompletionType, MicrobatchFingerprintStatus, @@ -34,7 +34,6 @@ PhysicalRelationRecord, PythonNodeVersionRecord, ReconcileEventRecord, - SeedVersionRecord, SourceFreshnessRecord, StateLockLease, StateLockRecord, @@ -43,12 +42,10 @@ StateSchemaValidationResult, VirtualEnvironmentCheckpointModelRefRecord, VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, VirtualEnvironmentModelRefRecord, VirtualEnvironmentNodeRefRecord, VirtualEnvironmentPythonNodeRefRecord, VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, ) from sqlbuild.virtual.state.types import ( ModelVersionStatus, @@ -83,10 +80,12 @@ VALID_PAYLOAD, StateReadContractObservation, StateRefContractObservation, + build_production_shaped_microbatch_event, exercise_state_read_contract, exercise_state_ref_contract, ) from tests.integration.src.sqlbuild.virtual.state.classes.postgres._test_types import ( + MicrobatchBulkAppendTestCase, PostgresAtomicFinalizedVirtualPublishTestCase, PostgresConditionalPublicationPayloadContractTestCase, PostgresConditionalVirtualRefPublishTestCase, @@ -104,7 +103,6 @@ PostgresStateBackendOperationEventTestCase, PostgresStateBackendPythonNodeIdentityTestCase, PostgresStateBackendRefContractTestCase, - PostgresStateBackendSeedRefTestCase, PostgresStateBackendSourceFreshnessTestCase, PostgresStateBackendTableCreationTestCase, PostgresStateBackendTransactionRollbackTestCase, @@ -150,10 +148,10 @@ def test_given_retired_postgres_rows_when_reading_history_then_only_active_event ) scope: MicrobatchScope = postgres_microbatch_scope() event: MicrobatchEvent = postgres_microbatch_event(scope=scope, event_id="active-event") - postgres_state_backend.append_microbatch_event( + postgres_state_backend.append_microbatch_events( connection=postgres_state_connection, schema=postgres_state_schema, - event=event, + events=(event,), ) insert_raw_postgres_microbatch_record( connection=postgres_state_connection, @@ -365,6 +363,60 @@ def test_given_invalid_conditional_payload_when_postgres_publishes_then_contract ) +@pytest.mark.parametrize( + "test_case", + [ + MicrobatchBulkAppendTestCase( + description="replay requirement with no completion or observation facts", + record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, + ), + MicrobatchBulkAppendTestCase( + description="executed partition completion with no observation facts", + record_type=MicrobatchRecordType.PARTITION_COMPLETION, + ), + ], + ids=lambda case: case.description, +) +def test_given_all_null_nullable_columns_when_bulk_appending_then_postgres_persists_idempotently( + test_case: MicrobatchBulkAppendTestCase, + postgres_state_backend: PostgresStateBackend, + postgres_state_connection: Any, + postgres_state_schema: str, +) -> None: + postgres_state_backend.initialize( + connection=postgres_state_connection, + schema=postgres_state_schema, + sqlbuild_version="test", + ) + scope: MicrobatchScope = MicrobatchScope( + scope_kind="virtual_physical", + scope_key="postgres:state:orders:F2:analytics.orders__F2", + model_name="orders", + target_database="warehouse", + target_schema="analytics", + target_name="orders__F2", + physical_generation_id="F2:analytics.orders__F2", + virtual_environment_name="dev", + virtual_model_version_hash="F2", + ) + event: MicrobatchEvent = build_production_shaped_microbatch_event( + scope=scope, record_type=test_case.record_type + ) + + first: MicrobatchWriteResult = postgres_state_backend.append_microbatch_events( + connection=postgres_state_connection, schema=postgres_state_schema, events=(event,) + ) + second: MicrobatchWriteResult = postgres_state_backend.append_microbatch_events( + connection=postgres_state_connection, schema=postgres_state_schema, events=(event,) + ) + + history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( + connection=postgres_state_connection, schema=postgres_state_schema, scope=scope + ) + assert (first.inserted, second.already_existing) == test_case.expected_write_counts + assert history == (event,) + + @pytest.mark.parametrize( "test_case", [ @@ -419,15 +471,15 @@ def test_given_postgres_microbatch_event_when_appending_then_scope_history_round created_at=datetime(2026, 1, 1), ) - postgres_state_backend.append_microbatch_event( + postgres_state_backend.append_microbatch_events( connection=postgres_state_connection, schema=postgres_state_schema, - event=event, + events=(event,), ) - postgres_state_backend.append_microbatch_event( + postgres_state_backend.append_microbatch_events( connection=postgres_state_connection, schema=postgres_state_schema, - event=event, + events=(event,), ) history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( @@ -976,15 +1028,6 @@ def test_given_postgres_state_backend_when_upserting_core_records_then_round_tri ) == replaced_relation_record ) - assert ( - postgres_state_backend.get_physical_relation_ancestry( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == ancestry_record - ) assert ( postgres_state_backend.get_virtual_environment( connection=postgres_state_connection, @@ -1143,170 +1186,6 @@ def test_given_postgres_backend_when_replacing_source_freshness_then_round_trips ) -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendSeedRefTestCase( - description="persists seed versions and replaces seed refs", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - seed_name="country_codes", - version_hash="seed123", - identity_metadata_hash="meta123", - identity_metadata_json_b64="e30=", - expected_ref_count_after_replace=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_backend_when_replacing_seed_refs_then_round_trips_records( - test_case: PostgresStateBackendSeedRefTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - seed_version: SeedVersionRecord = SeedVersionRecord( - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - identity_metadata_hash=test_case.identity_metadata_hash, - identity_metadata_json_b64=test_case.identity_metadata_json_b64, - status=ModelVersionStatus.READY, - ) - postgres_state_backend.upsert_seed_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=seed_version, - ) - postgres_state_backend.replace_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - - assert ( - postgres_state_backend.get_seed_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ) - == seed_version - ) - refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert refs == ( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - - postgres_state_backend.replace_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=(), - ) - assert ( - len( - postgres_state_backend.get_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - == test_case.expected_ref_count_after_replace - ) - postgres_state_backend.replace_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - postgres_state_backend.delete_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - postgres_state_backend.get_virtual_environment_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - == () - ) - - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id="chk_seed", - virtual_environment_name=test_case.virtual_environment_name, - ) - postgres_state_backend.create_virtual_environment_checkpoint( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint=checkpoint, - refs=(), - seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_checkpoint_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - assert checkpoint_seed_refs == ( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - postgres_state_backend.delete_virtual_environment_checkpoint( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - assert ( - postgres_state_backend.get_virtual_environment_checkpoint_seed_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - == () - ) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py index 66e92b8089..4c8e46ecbb 100644 --- a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py +++ b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py @@ -22,7 +22,7 @@ NodeResultRecord, ) from sqlbuild.executor.node_results.types import NodeResultStatus -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope +from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult from sqlbuild.microbatches.types import ( MicrobatchCompletionType, MicrobatchFingerprintStatus, @@ -45,7 +45,6 @@ PhysicalRelationRecord, PythonNodeVersionRecord, ReconcileEventRecord, - SeedVersionRecord, SourceFreshnessRecord, StateLockLease, StateLockRecord, @@ -55,7 +54,6 @@ VirtualEnvironmentCheckpointFunctionRefRecord, VirtualEnvironmentCheckpointModelRefRecord, VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, VirtualEnvironmentFunctionRefRecord, VirtualEnvironmentModelRefRecord, VirtualEnvironmentNodeRefRecord, @@ -92,11 +90,11 @@ DuckDbStateBackendPythonNodeIdentityTestCase, DuckDbStateBackendRefContractTestCase, DuckDbStateBackendRollbackTestCase, - DuckDbStateBackendSeedRefTestCase, DuckDbStateBackendSourceFreshnessTestCase, DuckDbStateBackendTableCreationTestCase, DuckDbStateBackendTransactionRollbackTestCase, DuckDbStateBackendValidationTestCase, + MicrobatchBulkAppendTestCase, MicrobatchStateRoundTripTestCase, StateReadContractTestCase, ) @@ -123,6 +121,7 @@ VALID_PAYLOAD, StateReadContractObservation, StateRefContractObservation, + build_production_shaped_microbatch_event, exercise_state_read_contract, exercise_state_ref_contract, fetch_all, @@ -291,6 +290,57 @@ def test_given_invalid_conditional_payload_when_duckdb_publishes_then_contract_r ) +@pytest.mark.parametrize( + "test_case", + [ + MicrobatchBulkAppendTestCase( + description="replay requirement with no completion or observation facts", + record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, + ), + MicrobatchBulkAppendTestCase( + description="executed partition completion with no observation facts", + record_type=MicrobatchRecordType.PARTITION_COMPLETION, + ), + ], + ids=lambda case: case.description, +) +def test_given_all_null_nullable_columns_when_bulk_appending_then_duckdb_persists_idempotently( + test_case: MicrobatchBulkAppendTestCase, tmp_path: Path +) -> None: + backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") + scope: MicrobatchScope = MicrobatchScope( + scope_kind="virtual_physical", + scope_key="duckdb:state:orders:F2:main.orders__F2", + model_name="orders", + target_database=None, + target_schema="main", + target_name="orders__F2", + physical_generation_id="F2:main.orders__F2", + virtual_environment_name="dev", + virtual_model_version_hash="F2", + ) + event: MicrobatchEvent = build_production_shaped_microbatch_event( + scope=scope, record_type=test_case.record_type + ) + try: + backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="test") + + first: MicrobatchWriteResult = backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) + ) + second: MicrobatchWriteResult = backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) + ) + history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_scope_history( + connection=connection, schema="sqlbuild_state", scope=scope + ) + finally: + backend.close(connection) + + assert (first.inserted, second.already_existing) == test_case.expected_write_counts + assert history == (event,) + + @pytest.mark.parametrize( "test_case", [ @@ -339,12 +389,16 @@ def test_given_virtual_microbatch_event_when_appending_then_scope_history_round_ ) try: backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="test") - backend.append_microbatch_event(connection=connection, schema="sqlbuild_state", event=event) + backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) + ) second_event: MicrobatchEvent = replace(event, event_id="event-2") - backend.append_microbatch_event( - connection=connection, schema="sqlbuild_state", event=second_event + backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(second_event,) + ) + backend.append_microbatch_events( + connection=connection, schema="sqlbuild_state", events=(event,) ) - backend.append_microbatch_event(connection=connection, schema="sqlbuild_state", event=event) history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_scope_history( connection=connection, schema="sqlbuild_state", scope=scope @@ -1066,14 +1120,16 @@ def test_given_duckdb_state_backend_when_upserting_core_records_then_round_trips ), ), ) - backend.replace_virtual_environment_seed_refs( + backend.replace_virtual_environment_node_refs( connection=connection, schema=test_case.schema, virtual_environment_name=test_case.expected_virtual_environment_name, + node_type="seed", refs=( - VirtualEnvironmentSeedRefRecord( + VirtualEnvironmentNodeRefRecord( virtual_environment_name=test_case.expected_virtual_environment_name, - seed_name=test_case.expected_model_name, + node_type="seed", + node_name=test_case.expected_model_name, version_hash="seed123", ), ), @@ -1107,15 +1163,6 @@ def test_given_duckdb_state_backend_when_upserting_core_records_then_round_trips ) == replaced_relation_record ) - assert ( - backend.get_physical_relation_ancestry( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == ancestry_record - ) assert ( backend.get_virtual_environment( connection=connection, @@ -1231,14 +1278,16 @@ def test_given_duckdb_state_backend_when_atomic_vde_ref_update_fails_then_rolls_ ), ), ) - backend.replace_virtual_environment_seed_refs( + backend.replace_virtual_environment_node_refs( connection=connection, schema=test_case.schema, virtual_environment_name=test_case.virtual_environment_name, + node_type="seed", refs=( - VirtualEnvironmentSeedRefRecord( + VirtualEnvironmentNodeRefRecord( virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, + node_type="seed", + node_name=test_case.seed_name, version_hash=test_case.expected_original_seed_hash, ), ), @@ -1680,172 +1729,6 @@ def test_given_duckdb_state_backend_when_replacing_source_freshness_then_round_t backend.close(connection) -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendSeedRefTestCase( - description="persists seed versions and replaces seed refs", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - seed_name="country_codes", - version_hash="seed123", - identity_metadata_hash="meta123", - identity_metadata_json_b64="e30=", - replacement_version_hash="seed456", - expected_ref_count_after_replace=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_replacing_seed_refs_then_round_trips_records( - test_case: DuckDbStateBackendSeedRefTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - seed_version: SeedVersionRecord = SeedVersionRecord( - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - identity_metadata_hash=test_case.identity_metadata_hash, - identity_metadata_json_b64=test_case.identity_metadata_json_b64, - status=ModelVersionStatus.READY, - ) - backend.upsert_seed_version( - connection=connection, schema=test_case.schema, record=seed_version - ) - backend.replace_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - - assert ( - backend.get_seed_version( - connection=connection, - schema=test_case.schema, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ) - == seed_version - ) - refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert refs == ( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - - backend.replace_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=(), - ) - assert ( - len( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - == test_case.expected_ref_count_after_replace - ) - backend.replace_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - backend.delete_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - == () - ) - - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id="chk_seed", - virtual_environment_name=test_case.virtual_environment_name, - ) - backend.create_virtual_environment_checkpoint( - connection=connection, - schema=test_case.schema, - checkpoint=checkpoint, - refs=(), - seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ), - ) - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_seed_refs( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - assert checkpoint_seed_refs == ( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - seed_name=test_case.seed_name, - version_hash=test_case.version_hash, - ), - ) - backend.delete_virtual_environment_checkpoint( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - assert ( - backend.get_virtual_environment_checkpoint_seed_refs( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - == () - ) - finally: - backend.close(connection) - - @pytest.mark.parametrize( "test_case", [ @@ -2165,18 +2048,21 @@ def test_given_duckdb_state_backend_when_upserting_function_records_then_round_t ) == function_record ) - backend.replace_virtual_environment_function_refs( + backend.replace_virtual_environment_node_ref_groups( connection=connection, schema=test_case.schema, virtual_environment_name=test_case.expected_virtual_environment_name, - refs=( - VirtualEnvironmentFunctionRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="udf", - function_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, + refs_by_node_type={ + "udf": ( + VirtualEnvironmentNodeRefRecord( + virtual_environment_name=test_case.expected_virtual_environment_name, + node_type="udf", + node_name=test_case.expected_model_name, + version_hash=test_case.expected_version_hash, + ), ), - ), + "table_fn": (), + }, ) function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( backend.get_virtual_environment_function_refs( diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py index 598b423b22..9406da96d3 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py @@ -53,13 +53,3 @@ class BaseAdapterMetadataSqlTestCase: schema: str name: str expected_sql: tuple[str, ...] - - -@dataclass(frozen=True) -class BaseAdapterRelationMaxCursorTestCase: - description: str - rows: tuple[tuple[object, ...], ...] - relation: str - cursor_column: str - expected_value: object | None - expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py index 9a4ce368ba..8b5106839c 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_base_adapter.py @@ -16,7 +16,6 @@ BaseAdapterIdentifierLimitTestCase, BaseAdapterMetadataSqlTestCase, BaseAdapterPythonFunctionSupportTestCase, - BaseAdapterRelationMaxCursorTestCase, BaseAdapterRetentionSupportTestCase, BaseAdapterSqlAnalysisDialectTestCase, ) @@ -199,44 +198,6 @@ def test_given_base_adapter_when_rendering_durable_clone_then_uses_copy_fallback assert result == test_case.expected_statements -@pytest.mark.parametrize( - "test_case", - ( - BaseAdapterRelationMaxCursorTestCase( - description="queries max cursor with quoted identifier", - rows=((42,),), - relation="analytics.events", - cursor_column='event"time', - expected_value=42, - expected_sql='SELECT max("event""time") FROM analytics.events', - ), - BaseAdapterRelationMaxCursorTestCase( - description="returns none when query returns no rows", - rows=(), - relation="analytics.empty_events", - cursor_column="event_time", - expected_value=None, - expected_sql='SELECT max("event_time") FROM analytics.empty_events', - ), - ), - ids=lambda case: case.description, -) -def test_given_relation_when_getting_max_cursor_then_base_adapter_queries_quoted_cursor( - test_case: BaseAdapterRelationMaxCursorTestCase, -) -> None: - adapter: RecordingBaseAdapter = RecordingBaseAdapter() - connection: RecordingConnection = RecordingConnection(rows=test_case.rows) - - result: object | None = adapter.get_relation_max_cursor( - connection=connection, - relation=test_case.relation, - cursor_column=test_case.cursor_column, - ) - - assert result == test_case.expected_value - assert tuple(connection.executed_sql) == (test_case.expected_sql,) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/_test_types.py b/tests/unit/src/sqlbuild/adapters/_test_types.py index 3cc1f0296f..7e89d6ee63 100644 --- a/tests/unit/src/sqlbuild/adapters/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/_test_types.py @@ -25,18 +25,6 @@ class AdapterCloneModeTestCase: expected_statements: tuple[str, ...] -@dataclass(frozen=True) -class AdapterRelationMaxCursorTestCase: - description: str - adapter: StrictAdapter - connection: object - relation: str - cursor_column: str - expected_value: object | None - expected_sql: tuple[str, ...] - expected_closed_cursor_count: int - - @dataclass(frozen=True) class AdapterEligibleMaxCursorSqlTestCase: description: str @@ -47,17 +35,6 @@ class AdapterEligibleMaxCursorSqlTestCase: expected_sql: str -@dataclass(frozen=True) -class AdapterSeedSelectAfterCursorTestCase: - description: str - adapter: StrictAdapter - origin: str - cursor_column: str - cursor_start_exclusive: str - cursor_type: str | None - expected_sql: str - - @dataclass(frozen=True) class AdapterManagedWriteSchemaCapabilityTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py index 581f8a21da..7e4d7d9111 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/_test_types.py @@ -173,13 +173,6 @@ class BigQuerySampleRowsTestCase: expected_side_only_samples: tuple[tuple[tuple[str, object], ...], ...] -@dataclass(frozen=True) -class BigQueryCountRowsTestCase: - description: str - expected_count: int - expected_sql: str - - @dataclass(frozen=True) class BigQueryPruneSqlTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py index 016de83d16..1305c9a277 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/helpers.py @@ -37,24 +37,6 @@ def close(self) -> None: return None -def fake_schema_diff_describe_relation( - connection: object, - relation: str, -) -> tuple[ColumnInfo, ...]: - del connection - relation_columns: dict[str, tuple[ColumnInfo, ...]] = { - "left_relation": ( - ColumnInfo(name="id", type="INT64"), - ColumnInfo(name="status", type="STRING"), - ), - "right_relation": ( - ColumnInfo(name="id", type="STRING"), - ColumnInfo(name="new_col", type="DATE"), - ), - } - return relation_columns[relation] - - def fake_row_diff_describe_relation( connection: object, relation: str, @@ -102,15 +84,6 @@ def fake_execute(connection: object, sql: str) -> StubCursor: return fake_execute -def build_count_rows_execute(executed_sql: list[str]) -> Any: - def fake_execute(connection: object, sql: str) -> StubCursor: - del connection - executed_sql.append(sql) - return StubCursor(row=(2,)) - - return fake_execute - - class FakeBigQuerySchemaField: def __init__(self, name: str) -> None: self.name: str = name diff --git a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py index dd9561133e..285762bc7d 100644 --- a/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/bigquery/test_client.py @@ -7,7 +7,6 @@ from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( ColumnInfo, - CursorValue, ExpressionInferenceProfile, QueryResult, RelationInfo, @@ -37,7 +36,6 @@ ) from tests.unit.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryConnectErrorTestCase, - BigQueryCountRowsTestCase, BigQueryDmlLifecycleTestCase, BigQueryExecutionErrorTestCase, BigQueryExpressionInferenceProfileTestCase, @@ -65,7 +63,6 @@ FakeBigQueryClient, FakeBigQueryFailingClient, FakeBigQueryRows, - build_count_rows_execute, build_fake_bigquery_schema_client, build_row_diff_execute, build_sample_rows_execute, @@ -1072,42 +1069,6 @@ def test_given_bigquery_relations_when_sampling_rows_then_returns_expected_examp assert side_only_samples == test_case.expected_side_only_samples -@pytest.mark.parametrize( - "test_case", - [ - BigQueryCountRowsTestCase( - description="uses typed timestamp cursor filter", - expected_count=2, - expected_sql=( - "SELECT COUNT(*) FROM left_relation WHERE updated_at >= " - "TIMESTAMP '2026-04-01 00:00:00' AND updated_at < " - "TIMESTAMP '2026-04-02 00:00:00'" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_timestamp_cursor_when_counting_rows_then_bigquery_uses_typed_filter( - test_case: BigQueryCountRowsTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - adapter: BigQueryAdapter = BigQueryAdapter() - executed_sql: list[str] = [] - - monkeypatch.setattr(adapter, "execute", build_count_rows_execute(executed_sql)) - - result: int = adapter.count_rows( - connection=object(), - relation="left_relation", - cursor_column="updated_at", - start_cursor=CursorValue(kind=CursorKind.TIMESTAMP, value=datetime(2026, 4, 1, 0, 0, 0)), - end_cursor=CursorValue(kind=CursorKind.TIMESTAMP, value=datetime(2026, 4, 2, 0, 0, 0)), - ) - - assert result == test_case.expected_count - assert executed_sql == [test_case.expected_sql] - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py index b1b57026cd..7317b8a384 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py @@ -47,13 +47,6 @@ class DuckDbMetadataSqlTestCase: expected_sql: tuple[str, ...] -@dataclass(frozen=True) -class DuckDbRelationMaxCursorTestCase: - description: str - expected_populated_value: object - expected_empty_value: object | None - - @dataclass(frozen=True) class DuckDbPruneSqlTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py index 5b205e8c02..ac373d6e7c 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py @@ -18,7 +18,6 @@ DuckDbMergeExclusionTestCase, DuckDbMetadataSqlTestCase, DuckDbPruneSqlTestCase, - DuckDbRelationMaxCursorTestCase, DuckDbRenderCursorBoundLiteralTestCase, DuckDbRenderIdentifierTestCase, DuckDbRenderSourceFreshnessQueryTestCase, @@ -108,46 +107,6 @@ def test_given_cursor_bounds_when_rendering_then_duckdb_returns_expected_literal assert result == test_case.expected_literal -@pytest.mark.parametrize( - "test_case", - [ - DuckDbRelationMaxCursorTestCase( - description="returns max cursor from populated relation and none from empty relation", - expected_populated_value=7, - expected_empty_value=None, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_relation_when_getting_max_cursor_then_returns_relation_max( - test_case: DuckDbRelationMaxCursorTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: object = adapter.connect({"database": ":memory:"}) - try: - adapter.execute(connection=connection, sql='CREATE TABLE events ("event""time" INTEGER)') - adapter.execute(connection=connection, sql="INSERT INTO events VALUES (1), (7), (3)") - adapter.execute( - connection=connection, sql='CREATE TABLE empty_events ("event""time" INTEGER)' - ) - - populated_value: object | None = adapter.get_relation_max_cursor( - connection=connection, - relation="events", - cursor_column='event"time', - ) - empty_value: object | None = adapter.get_relation_max_cursor( - connection=connection, - relation="empty_events", - cursor_column='event"time', - ) - finally: - adapter.close(connection) - - assert populated_value == test_case.expected_populated_value - assert empty_value == test_case.expected_empty_value - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/helpers.py b/tests/unit/src/sqlbuild/adapters/helpers.py deleted file mode 100644 index 1aaea5e9c5..0000000000 --- a/tests/unit/src/sqlbuild/adapters/helpers.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from types import MappingProxyType -from typing import Protocol, cast - - -class AdapterRecordingConnection(Protocol): - def recorded_sql(self) -> tuple[str, ...]: ... - - def closed_cursor_count(self) -> int: ... - - -class AdapterCursorMaxCursor: - def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: - self.rows: tuple[tuple[object, ...], ...] = rows - self.closed: bool = False - self.executed_sql: list[str] = [] - - def execute(self, sql: str) -> None: - self.executed_sql.append(sql) - - def fetchone(self) -> tuple[object, ...] | None: - return next(iter(self.rows), None) - - def close(self) -> None: - self.closed = True - - -class AdapterExecuteRecordingConnection: - def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: - self.cursor: AdapterCursorMaxCursor = AdapterCursorMaxCursor(rows=rows) - self.executed_sql: list[str] = [] - - def execute(self, sql: str) -> AdapterCursorMaxCursor: - self.executed_sql.append(sql) - return self.cursor - - def recorded_sql(self) -> tuple[str, ...]: - return tuple(self.executed_sql) - - def closed_cursor_count(self) -> int: - return 0 - - -class AdapterCursorRecordingConnection: - def __init__(self, rows: tuple[tuple[object, ...], ...]) -> None: - self.cursor_instance: AdapterCursorMaxCursor = AdapterCursorMaxCursor(rows=rows) - - def cursor(self) -> AdapterCursorMaxCursor: - return self.cursor_instance - - def recorded_sql(self) -> tuple[str, ...]: - return tuple(self.cursor_instance.executed_sql) - - def closed_cursor_count(self) -> int: - return int(self.cursor_instance.closed) - - -def adapter_cursor_executed_sql(connection: object) -> tuple[str, ...]: - _CONNECTION_VALIDATORS[ - isinstance( - connection, (AdapterExecuteRecordingConnection, AdapterCursorRecordingConnection) - ) - ](connection) - return cast(AdapterRecordingConnection, connection).recorded_sql() - - -def adapter_closed_cursor_count(connection: object) -> int: - _CONNECTION_VALIDATORS[ - isinstance( - connection, (AdapterExecuteRecordingConnection, AdapterCursorRecordingConnection) - ) - ](connection) - return cast(AdapterRecordingConnection, connection).closed_cursor_count() - - -def _accept_recording_connection(connection: object) -> None: - del connection - - -def _reject_recording_connection(connection: object) -> None: - raise TypeError(f"unsupported connection type: {type(connection).__name__}") - - -_CONNECTION_VALIDATORS: MappingProxyType[bool, Callable[[object], None]] = MappingProxyType( - {False: _reject_recording_connection, True: _accept_recording_connection} -) diff --git a/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py b/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py index 76c8521369..e702ae026b 100644 --- a/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py +++ b/tests/unit/src/sqlbuild/adapters/test_relation_max_cursor.py @@ -11,107 +11,7 @@ from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from tests.unit.src.sqlbuild.adapters._test_types import ( AdapterEligibleMaxCursorSqlTestCase, - AdapterRelationMaxCursorTestCase, - AdapterSeedSelectAfterCursorTestCase, ) -from tests.unit.src.sqlbuild.adapters.helpers import ( - AdapterCursorRecordingConnection, - AdapterExecuteRecordingConnection, - adapter_closed_cursor_count, - adapter_cursor_executed_sql, -) - - -@pytest.mark.parametrize( - "test_case", - ( - AdapterRelationMaxCursorTestCase( - description="duckdb uses shared quoted cursor SQL", - adapter=DuckDbAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((9,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=9, - expected_sql=('SELECT max("event""time") FROM analytics.events',), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="motherduck uses shared quoted cursor SQL", - adapter=MotherDuckAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((10,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=10, - expected_sql=('SELECT max("event""time") FROM analytics.events',), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="postgres uses double quoted cursor SQL", - adapter=PostgresAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((11,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=11, - expected_sql=('SELECT max("event""time") FROM analytics.events',), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="bigquery uses backtick quoted cursor SQL", - adapter=BigQueryAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((12,),)), - relation="analytics.events", - cursor_column="event`time", - expected_value=12, - expected_sql=("SELECT max(`event``time`) FROM analytics.events",), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="sqlserver uses bracket quoted cursor SQL", - adapter=SqlServerAdapter(), - connection=AdapterExecuteRecordingConnection(rows=((13,),)), - relation="analytics.events", - cursor_column="event]time", - expected_value=13, - expected_sql=("SELECT max([event]]time]) FROM analytics.events",), - expected_closed_cursor_count=0, - ), - AdapterRelationMaxCursorTestCase( - description="snowflake uppercases and closes cursor", - adapter=SnowflakeAdapter(), - connection=AdapterCursorRecordingConnection(rows=((14,),)), - relation="analytics.events", - cursor_column='event"time', - expected_value=14, - expected_sql=('SELECT max("EVENT""TIME") FROM analytics.events',), - expected_closed_cursor_count=1, - ), - AdapterRelationMaxCursorTestCase( - description="databricks uses backtick cursor SQL and closes cursor", - adapter=DatabricksAdapter(), - connection=AdapterCursorRecordingConnection(rows=((15,),)), - relation="analytics.events", - cursor_column="event`time", - expected_value=15, - expected_sql=("SELECT max(`event``time`) FROM analytics.events",), - expected_closed_cursor_count=1, - ), - ), - ids=lambda case: case.description, -) -def test_given_first_class_adapter_when_getting_relation_max_cursor_then_uses_adapter_sql( - test_case: AdapterRelationMaxCursorTestCase, -) -> None: - result: object | None = test_case.adapter.get_relation_max_cursor( - connection=test_case.connection, - relation=test_case.relation, - cursor_column=test_case.cursor_column, - ) - - assert result == test_case.expected_value - assert adapter_cursor_executed_sql(test_case.connection) == test_case.expected_sql - assert ( - adapter_closed_cursor_count(test_case.connection) == test_case.expected_closed_cursor_count - ) @pytest.mark.parametrize( @@ -204,95 +104,3 @@ def test_given_cursor_horizon_when_rendering_eligible_max_then_adapter_owns_port ) assert sql == test_case.expected_sql - - -@pytest.mark.parametrize( - "test_case", - ( - AdapterSeedSelectAfterCursorTestCase( - description="duckdb renders seed select after timestamp cursor", - adapter=DuckDbAdapter(), - origin="prod.events", - cursor_column='event"time', - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - 'SELECT * FROM prod.events WHERE "event""time" > TIMESTAMP \'2026-01-01 00:00:00\'' - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="motherduck renders seed select after timestamp cursor", - adapter=MotherDuckAdapter(), - origin="prod.events", - cursor_column='event"time', - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - 'SELECT * FROM prod.events WHERE "event""time" > TIMESTAMP \'2026-01-01 00:00:00\'' - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="postgres renders seed select after integer cursor", - adapter=PostgresAdapter(), - origin="prod.events", - cursor_column='event"index', - cursor_start_exclusive="10", - cursor_type="integer", - expected_sql='SELECT * FROM prod.events WHERE "event""index" > 10', - ), - AdapterSeedSelectAfterCursorTestCase( - description="bigquery renders seed select with backtick cursor", - adapter=BigQueryAdapter(), - origin="prod.events", - cursor_column="event`time", - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - "SELECT * FROM prod.events WHERE `event``time` > TIMESTAMP '2026-01-01 00:00:00'" - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="sqlserver renders seed select with bracket cursor", - adapter=SqlServerAdapter(), - origin="prod.events", - cursor_column="event]index", - cursor_start_exclusive="10", - cursor_type="integer", - expected_sql="SELECT * FROM prod.events WHERE [event]]index] > 10", - ), - AdapterSeedSelectAfterCursorTestCase( - description="snowflake renders seed select with uppercase quoted cursor", - adapter=SnowflakeAdapter(), - origin="prod.events", - cursor_column='event"time', - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - 'SELECT * FROM prod.events WHERE "EVENT""TIME" > \'2026-01-01 00:00:00\'' - ), - ), - AdapterSeedSelectAfterCursorTestCase( - description="databricks renders seed select with backtick cursor", - adapter=DatabricksAdapter(), - origin="prod.events", - cursor_column="event`time", - cursor_start_exclusive="2026-01-01 00:00:00", - cursor_type="timestamp", - expected_sql=( - "SELECT * FROM prod.events WHERE `event``time` > TIMESTAMP '2026-01-01 00:00:00'" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_first_class_adapter_when_rendering_seed_select_after_cursor_then_uses_adapter_sql( - test_case: AdapterSeedSelectAfterCursorTestCase, -) -> None: - sql: str = test_case.adapter.render_seed_select_after_cursor( - origin=test_case.origin, - cursor_column=test_case.cursor_column, - cursor_start_exclusive=test_case.cursor_start_exclusive, - cursor_type=test_case.cursor_type, - ) - - assert sql == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py index 5b825cb43c..a70998a9a2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/_test_types.py @@ -11,10 +11,3 @@ class SourceFreshnessAppendEligibilityTestCase: model_statuses: dict[str, ExecutionStatus] expected_insert_count: int expected_lifecycle_order: tuple[str, ...] - - -@dataclass(frozen=True) -class DbtSqlbuildWorkOutputTestCase: - description: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 9a60750aa0..8c449efd1f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -116,7 +116,6 @@ def test_given_direct_check_sql_refs_when_preflighting_then_refs_are_validated_w lifecycle_plan=PythonSqlRunLifecyclePlan( ingress_python_node_names=frozenset(), ingress_loader_names=frozenset(), - read_side_sql_keys=frozenset(), read_side_python_node_names=frozenset(), ), relation_targets={model_ref: "analytics.orders", source_ref: "raw.orders"}, @@ -198,7 +197,6 @@ def test_given_missing_direct_check_sql_ref_when_preflighting_then_existing_rela lifecycle_plan=PythonSqlRunLifecyclePlan( ingress_python_node_names=frozenset(), ingress_loader_names=frozenset(), - read_side_sql_keys=frozenset(), read_side_python_node_names=frozenset(), ), relation_targets={ref: "analytics.orders"}, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py index 0d0a17d142..efc2c81fc4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -10,7 +10,6 @@ class TargetWriterTestCase: description: str expected_files: dict[str, str] = field(default_factory=dict) initial_files: dict[str, str] = field(default_factory=dict) - expected_summary_line: str = "" model_count: int = 0 expected_max_component_bytes: int = 0 diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index c5579797b0..3c75541eb4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -56,7 +56,6 @@ "SELECT order_id FROM analytics.orders WHERE order_id IS NULL\n" ), }, - expected_summary_line="Compiled 1 model, 1 seed, 2 functions, 1 audit, 1 test", ) ], ids=lambda case: case.description, @@ -74,7 +73,7 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( expected_files: dict[str, str] = dict(test_case.expected_files) expected_files["compiled/tests/_chain_/orders__stg_orders/orders_chain.sql"] = expected_test_sql - written: WrittenTarget = write_compile_target( + write_compile_target( target_dir=tmp_path / "target", adapter=DuckDbAdapter(), plan_output=plan_output, @@ -90,7 +89,6 @@ def test_given_plan_output_when_writing_target_then_expected_files_are_written( manifest=manifest, ) - assert written.summary_line() == test_case.expected_summary_line assert read_target_files(tmp_path / "target", expected_files) == expected_files assert json.loads(manifest_path.read_text()) == manifest assert manifest_path.stat().st_mtime_ns == unchanged_mtime_ns @@ -194,7 +192,6 @@ def test_given_parameterized_test_entries_when_writing_target_then_case_artifact ")\n" ), }, - expected_summary_line="Compiled 1 model, 1 function", ) ], ids=lambda case: case.description, @@ -209,7 +206,7 @@ def test_given_compiled_project_when_writing_static_target_then_expected_files_a stale_path.parent.mkdir(parents=True) stale_path.write_text("SELECT 'stale'\n", encoding="utf-8") - written: WrittenTarget = write_static_compile_target( + write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), project=project, @@ -223,7 +220,6 @@ def test_given_compiled_project_when_writing_static_target_then_expected_files_a project=project, ) - assert written.summary_line() == test_case.expected_summary_line assert read_target_files(target_dir, test_case.expected_files) == test_case.expected_files assert model_path.stat().st_mtime_ns == unchanged_mtime_ns assert not stale_path.exists() diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 339cf22e89..50d4ebc625 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -8,7 +8,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild.cli.commands._helpers.entry.parsing import read_selector_files +from sqlbuild.cli.commands._helpers.entry.parsing import read_selector_file_inputs from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.main.entrypoint.entry import _main_with_dependencies, main from sqlbuild.cli.commands.models import ( @@ -2430,7 +2430,10 @@ def run_plan(request: PlanCommandRequest) -> int: assert exit_code == test_case.expected_exit_code assert received_selects == [test_case.expected_select] - assert read_selector_files([str(tmp_path / "selectors.txt")]) == ("customers", "payments") + assert read_selector_file_inputs([str(tmp_path / "selectors.txt")]).selectors == ( + "customers", + "payments", + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 2981975de0..6968bea9ab 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -18,7 +18,6 @@ ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import UsageRecord -from sqlbuild.compiler.scopes.types import ScopeKind from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.sql_values.models import SqlValue @@ -67,12 +66,6 @@ class AnalysisCacheTestCase: expected_count: int -@dataclass(frozen=True) -class SqlTestCteCacheTestCase: - description: str - expected_scanner_calls: int - - @dataclass(frozen=True) class NativeSqlTestExtractionParityTestCase: description: str @@ -188,23 +181,6 @@ class ExpandSqlMacrosErrorTestCase: macro_overrides: dict[str, str] = field(default_factory=dict) -@dataclass(frozen=True) -class ScopedMacroExpansionTestCase: - description: str - definitions: dict[str, tuple[str, ScopeKind, str | None, str]] - expected_sql: str - expected_dependencies: tuple[str, ...] - expected_usages: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True) -class ScopedMacroExpansionErrorTestCase: - description: str - definitions: dict[str, tuple[str, ScopeKind, str | None, str]] - sql: str - expected_error_fragment: str - - @dataclass(frozen=True) class FindMacroCallNamesTestCase: description: str @@ -453,7 +429,7 @@ class BuildScenarioInputsErrorTestCase: class ExtractSqlAnalysisExpectedBranchesTestCase: description: str sql: str - expected_branch_column_names: tuple[tuple[str, ...], ...] + expected_branch_column_names: tuple[tuple[str, ...], ...] | None @dataclass(frozen=True) @@ -625,13 +601,6 @@ class IncrementalConfigErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class CursorAliasWarningTestCase: - description: str - config_values: tuple[dict[str, object], ...] - expected_warning_count: int - - @dataclass(frozen=True) class ContractConfigValidTestCase: description: str @@ -691,15 +660,6 @@ class ExpandTemplateDataErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class VarMacroCollisionTestCase: - description: str - var_names: tuple[str, ...] - macro_names: tuple[str, ...] - expected_valid: bool - expected_error_fragment: str | None = None - - @dataclass(frozen=True) class NonIncrementalConfigValidTestCase: description: str @@ -910,3 +870,25 @@ class ExpectedProjectionScanTestCase: expected_commas: tuple[str, ...] expected_alias: str | None expected_contains_select_star: bool + + +@dataclass(frozen=True) +class SetOperationExpectedKind: + """One SQL-test kind whose expected CTE accepts set operations.""" + + description: str + mode: SqlTestMode + support_ctes: tuple[tuple[str, str], ...] + expected_cte_name: str + payload_type: str + + +@dataclass(frozen=True) +class SetOperationExpectedTestCase: + """One expected-CTE set operation in one SQL-test kind.""" + + description: str + kind: SetOperationExpectedKind + sql: str + expected_payload_type: str = "" + expected_error_template: str = "" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py deleted file mode 100644 index 32f2e17f8b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_alias_deprecation.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.compile._helpers.config.deprecation import ( - build_cursor_alias_diagnostics, -) -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - CompileModelInput, - CompilerDiagnostic, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - CursorAliasWarningTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CursorAliasWarningTestCase( - description="cursor inputs emit no legacy alias warning", - config_values=( - {"cursor_inputs": {"orders": "event_time"}}, - {"cursor_inputs": {"shipments": "event_time"}}, - ), - expected_warning_count=0, - ), - CursorAliasWarningTestCase( - description="removed cursor role names emit no deprecation warning", - config_values=( - {"cursor_filter_inputs": {"orders": "event_time"}}, - {"cursor_watermark_inputs": {"raw_orders": "loaded_at"}}, - ), - expected_warning_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_project_cursor_roles_when_building_diagnostics_then_warning_cardinality_is_exact( - test_case: CursorAliasWarningTestCase, -) -> None: - model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( - file_path=Path("models/orders.sql"), - relative_path=Path("models/orders.sql"), - contents="SELECT 1", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="SELECT 1", - ) - model_inputs: tuple[CompileModelInput, ...] = tuple( - CompileModelInput(model_file=model_file, config=CompileModelConfig(values=values)) - for values in test_case.config_values - ) - - diagnostics: tuple[CompilerDiagnostic, ...] = build_cursor_alias_diagnostics( - model_inputs=model_inputs - ) - - assert len(diagnostics) == test_case.expected_warning_count - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py index af71f97607..1377c29cfb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py @@ -5,6 +5,9 @@ import pytest from sqlbuild.compiler.compile._helpers.sql_tests import core as sql_test_core +from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( + split_set_operation_branches, +) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedProjectionScanTestCase, ) @@ -58,6 +61,28 @@ expected_alias="a", expected_contains_select_star=False, ), + ExpectedProjectionScanTestCase( + description="intersect and except split branches with quantifiers", + sql="SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a", + expected_branches=("SELECT 1 AS a", "SELECT 2 AS a", "SELECT 3 AS a"), + expected_select_list_end=len( + "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a" + ), + expected_commas=( + "SELECT 1 AS a INTERSECT ALL SELECT 2 AS a EXCEPT DISTINCT SELECT 3 AS a", + ), + expected_alias="a", + expected_contains_select_star=False, + ), + ExpectedProjectionScanTestCase( + description="star except modifier is not a set operation", + sql="SELECT t.* EXCEPT (status), exceptional FROM t", + expected_branches=("SELECT t.* EXCEPT (status), exceptional FROM t",), + expected_select_list_end=len("SELECT t.* EXCEPT (status), exceptional "), + expected_commas=("SELECT t.* EXCEPT (status)", "exceptional FROM t"), + expected_alias=None, + expected_contains_select_star=False, + ), ExpectedProjectionScanTestCase( description="backtick identifiers are quoted", sql="SELECT `a, b` AS `c`, * FROM t", @@ -82,7 +107,10 @@ def test_given_expected_cte_sql_when_scanning_top_level_then_results_are_characterised( test_case: ExpectedProjectionScanTestCase, ) -> None: - assert sql_test_core._split_set_operation_branches(test_case.sql) == test_case.expected_branches + assert ( + split_set_operation_branches(sql=test_case.sql, context="SQL test") + == test_case.expected_branches + ) assert ( sql_test_core._find_select_list_end(sql=test_case.sql, start=0) == test_case.expected_select_list_end diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py new file mode 100644 index 0000000000..123f8a652d --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py @@ -0,0 +1,141 @@ +"""Set-operation expected CTEs across every SQL-test kind validate every branch.""" + +from __future__ import annotations + +from itertools import product + +import pytest + +from sqlbuild.compiler.compile._helpers.sql_tests.core import classify_sql_test_ctes +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileSqlTestCte, CompileSqlTestCtes +from sqlbuild.compiler.compile.types import SqlTestMode +from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + SetOperationExpectedKind, + SetOperationExpectedTestCase, +) + +_FILE: str = "tests/unit/orders.sql" +_MISMATCH: str = ( + "SQL test '{file}' must use the same {cte} projection names and order in every " + "set-operation branch; branch {branch} does not match branch 1" +) +_NOT_SELECT: str = "SQL test '{file}' must define each {cte} set-operation branch as a SELECT query" +_KINDS: tuple[SetOperationExpectedKind, ...] = ( + SetOperationExpectedKind( + description="model", + mode=SqlTestMode.MODEL, + support_ctes=(("__source__raw_orders", "SELECT 1 AS order_id"),), + expected_cte_name="__expected__orders", + payload_type="CompileModelSqlTestCtes", + ), + SetOperationExpectedKind( + description="macro", + mode=SqlTestMode.MACRO, + support_ctes=(("__macro_actual__", "SELECT 1 AS order_id"),), + expected_cte_name="__macro_expected__", + payload_type="CompileDirectLogicSqlTestCtes", + ), + SetOperationExpectedKind( + description="udf", + mode=SqlTestMode.UDF, + support_ctes=(("__udf_actual__", "SELECT 1 AS order_id"),), + expected_cte_name="__udf_expected__", + payload_type="CompileDirectLogicSqlTestCtes", + ), + SetOperationExpectedKind( + description="table_fn", + mode=SqlTestMode.TABLE_FN, + support_ctes=(("__table_fn_actual__", "SELECT 1 AS order_id"),), + expected_cte_name="__table_fn_expected__", + payload_type="CompileDirectLogicSqlTestCtes", + ), +) +_ACCEPTED_SQL: tuple[str, ...] = ( + "SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + "SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + "SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + "SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", +) +_REJECTED_SQL: tuple[tuple[str, str], ...] = ( + ("SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", _MISMATCH.replace("{branch}", "2")), + ( + "SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + _MISMATCH.replace("{branch}", "2"), + ), + ( + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + _MISMATCH.replace("{branch}", "3"), + ), + ("SELECT 1 AS order_id EXCEPT VALUES (1)", _NOT_SELECT), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SetOperationExpectedTestCase( + description=f"{kind.description}: {sql}", + kind=kind, + sql=sql, + expected_payload_type=kind.payload_type, + ) + for kind, sql in product(_KINDS, _ACCEPTED_SQL) + ], + ids=lambda case: case.description, +) +def test_given_set_operation_expected_cte_when_classifying_then_it_is_accepted( + test_case: SetOperationExpectedTestCase, +) -> None: + ctes: tuple[CompileSqlTestCte, ...] = ( + *( + CompileSqlTestCte(name=name, sql_body=body) + for name, body in test_case.kind.support_ctes + ), + CompileSqlTestCte(name=test_case.kind.expected_cte_name, sql_body=test_case.sql), + ) + + classified: CompileSqlTestCtes = classify_sql_test_ctes( + ctes=ctes, file_label=_FILE, mode=test_case.kind.mode + ) + + assert type(classified.payload).__name__ == test_case.expected_payload_type + + +@pytest.mark.parametrize( + "test_case", + [ + SetOperationExpectedTestCase( + description=f"{kind.description}: {rejected[0]}", + kind=kind, + sql=rejected[0], + expected_error_template=rejected[1], + ) + for kind, rejected in product(_KINDS, _REJECTED_SQL) + ], + ids=lambda case: case.description, +) +def test_given_invalid_set_operation_branch_when_classifying_then_it_names_the_expected_cte( + test_case: SetOperationExpectedTestCase, +) -> None: + ctes: tuple[CompileSqlTestCte, ...] = ( + *( + CompileSqlTestCte(name=name, sql_body=body) + for name, body in test_case.kind.support_ctes + ), + CompileSqlTestCte(name=test_case.kind.expected_cte_name, sql_body=test_case.sql), + ) + + with pytest.raises(CompileInputError) as error_info: + _ = classify_sql_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.kind.mode) + + assert str(error_info.value) == test_case.expected_error_template.format( + file=_FILE, cte=test_case.kind.expected_cte_name + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py index ede8156d7c..236430d065 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_tests.py @@ -33,6 +33,52 @@ sql="SELECT order_id, status FROM expected_rows", expected_branch_column_names=(("order_id", "status"),), ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback ignores an apostrophe in a line comment", + sql="SELECT 1 AS order_id; -- don't mix\nUNION ALL SELECT 2 AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback ignores a set operation inside a block comment", + sql="SELECT 1 AS order_id; /* UNION ALL */ UNION ALL SELECT 2 AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback splits union distinct", + sql="SELECT 1 AS order_id; UNION DISTINCT SELECT 2 AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback keeps doubled quotes inside one string", + sql="SELECT 1 AS order_id; UNION SELECT 'it''s UNION' AS order_id", + expected_branch_column_names=(("order_id",), ("order_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="parsed intersect and except return every branch in order", + sql=( + "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT ALL " + "SELECT 3 AS other_id INTERSECT DISTINCT SELECT 4 AS extra_id" + ), + expected_branch_column_names=( + ("order_id",), + ("order_id",), + ("other_id",), + ("extra_id",), + ), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback splits intersect and except with quantifiers", + sql=( + "SELECT 1 AS order_id; INTERSECT ALL SELECT 2 AS order_id " + "EXCEPT DISTINCT SELECT 3 AS other_id" + ), + expected_branch_column_names=(("order_id",), ("order_id",), ("other_id",)), + ), + ExtractSqlAnalysisExpectedBranchesTestCase( + description="fallback defers to textual analysis when a branch cannot parse", + sql='SELECT __udf("net_amount")(1) AS amount UNION ALL SELECT 2 AS amount', + expected_branch_column_names=None, + ), ], ids=lambda case: case.description, ) @@ -62,6 +108,11 @@ def test_given_sql_analysis_available_when_extracting_expected_branches_then_it_ sql="SELECT 1 AS order_id UNION ALL VALUES (2)", expected_error_fragment="set-operation branch as a SELECT query", ), + ExtractSqlAnalysisExpectedBranchesErrorTestCase( + description="fallback rejects a values branch after except", + sql="SELECT 1 AS order_id; EXCEPT VALUES (2)", + expected_error_fragment="set-operation branch as a SELECT query", + ), ExtractSqlAnalysisExpectedBranchesErrorTestCase( description="raises when expected select uses star projection", sql="SELECT * FROM expected_rows", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py deleted file mode 100644 index 54d935531b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_cte_cache.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import sqlite3 -from pathlib import Path -from unittest.mock import Mock - -import pytest - -from sqlbuild.compiler.compile._helpers.sql_tests import cache as cte_cache -from sqlbuild.compiler.compile._helpers.sql_tests.cache import cached_sql_test_cte_extractor -from sqlbuild.compiler.compile.models import CompileSqlTestCtes -from sqlbuild.compiler.compile.types import SqlTestMode -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - SqlTestCteCacheTestCase, -) - -_TEST_SQL: str = ( - "WITH __ref__raw_orders AS (SELECT 1 AS order_id), " - "__expected__orders AS (SELECT 1 AS order_id) SELECT 1" -) - - -@pytest.mark.parametrize( - "test_case", - (SqlTestCteCacheTestCase(description="unchanged SQL reuses CTEs", expected_scanner_calls=0),), - ids=lambda case: case.description, -) -def test_given_cached_test_ctes_when_extracting_again_then_skips_scanner( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: SqlTestCteCacheTestCase, -) -> None: - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - expected: CompileSqlTestCtes = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - scanner: Mock = Mock(wraps=cte_cache.extract_unclassified_sql_test_ctes) - monkeypatch.setattr(cte_cache, "extract_unclassified_sql_test_ctes", scanner) - - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - actual: CompileSqlTestCtes = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - - assert scanner.call_count == test_case.expected_scanner_calls - assert actual == expected - - -@pytest.mark.parametrize( - "test_case", - (SqlTestCteCacheTestCase(description="changed SQL scans CTEs", expected_scanner_calls=1),), - ids=lambda case: case.description, -) -def test_given_changed_test_sql_when_extracting_then_scans_new_ctes( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: SqlTestCteCacheTestCase, -) -> None: - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - scanner: Mock = Mock(wraps=cte_cache.extract_unclassified_sql_test_ctes) - monkeypatch.setattr(cte_cache, "extract_unclassified_sql_test_ctes", scanner) - changed_sql: str = _TEST_SQL.replace("SELECT 1 AS order_id", "SELECT 2 AS order_id") - - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(changed_sql, "test_orders.sql", SqlTestMode.MODEL) - - assert scanner.call_count == test_case.expected_scanner_calls - - -@pytest.mark.parametrize( - "test_case", - (SqlTestCteCacheTestCase(description="corrupt cache rescans", expected_scanner_calls=1),), - ids=lambda case: case.description, -) -def test_given_corrupt_test_cte_cache_when_extracting_then_rescans_safely( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: SqlTestCteCacheTestCase, -) -> None: - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - cache_path: Path = next(tmp_path.rglob("sql-test-ctes.sqlite3")) - with sqlite3.connect(cache_path) as connection: - _ = connection.execute("UPDATE sql_test_cte SET payload = 'broken'") - scanner: Mock = Mock(wraps=cte_cache.extract_unclassified_sql_test_ctes) - monkeypatch.setattr(cte_cache, "extract_unclassified_sql_test_ctes", scanner) - - with cached_sql_test_cte_extractor(root=tmp_path) as extract: - _ = extract(_TEST_SQL, "test_orders.sql", SqlTestMode.MODEL) - - assert scanner.call_count == test_case.expected_scanner_calls diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py index 0476e95425..4f2e2d17ba 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py @@ -648,7 +648,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return __expected__orders AS (SELECT * FROM __source__raw_orders) SELECT 1 """.strip(), - expected_error_fragment=r"must not use SELECT \* in __expected__ CTEs", + expected_error_fragment=r"must not use SELECT \* in __expected__orders CTEs", ), ExtractSqlTestCtesErrorTestCase( description="rejects empty fixture marker for direct logic expected output", @@ -684,7 +684,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return ) SELECT 1 """.strip(), - expected_error_fragment="must alias every non-trivial __expected__ projection", + expected_error_fragment="must alias every non-trivial __expected__orders projection", ), ExtractSqlTestCtesErrorTestCase( description="raises when non union projection alias is missing", @@ -695,7 +695,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return ) SELECT 1 """.strip(), - expected_error_fragment="must alias every non-trivial __expected__ projection", + expected_error_fragment="must alias every non-trivial __expected__orders projection", ), ExtractSqlTestCtesErrorTestCase( description="raises when expected branch is not a select query", diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 55ce12db8c..ffff94995c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -2,13 +2,11 @@ from pathlib import Path from typing import cast -from sqlbuild import _native from sqlbuild.compiler.discovery._helpers.filesystem.core import ( discover_constant_files, discover_enum_files, discover_macro_files, ) -from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers from sqlbuild.compiler.discovery.models import ( DiscoveredConstantFile, DiscoveredEnumFile, @@ -39,149 +37,6 @@ } -def _standard_model_header(index: int, suffix: str) -> str: - return ( - f"name model_{suffix}, enabled true, columns (col_{suffix} " - '(type DECIMAL(10,2), nullable false, description "Order total"))' - ) - - -def _constant_model_header(_index: int, suffix: str) -> str: - return ( - f"constants (_set_{suffix} {{FR, GB, FR}}, " - f"_array_{suffix} constant(value [1, 2], render_as array))" - ) - - -def _hook_model_header(index: int, suffix: str) -> str: - return ( - f'pre_hooks [inline_sql("select {index}"), sql("record_{suffix}", ' - 'table: "orders"), python("notify", attempts: 2, urgent: true)]' - ) - - -def _audit_model_header(_index: int, suffix: str) -> str: - return ( - f"audits [rate_{suffix} (thresholds (warn (outside -1.5 2.5)))], " - f'parent __ref("orders_{suffix}")' - ) - - -def _template_model_header(_index: int, suffix: str) -> str: - return f"schema dev_${{user_{suffix}}}, tags [core, 'daily orders'], value null" - - -def _unicode_column_model_header(_index: int, suffix: str) -> str: - return ( - f"columns (café_{suffix} (type TIMESTAMP_NTZ(9), audits " - "[accepted_values (values [placed, completed])]))" - ) - - -def _numeric_model_header(index: int, suffix: str) -> str: - numeric_headers: tuple[str, ...] = ( - f"unicode_integer_{suffix} +١٢٣", - f"unicode_integer_{suffix} -१२३", - f"unicode_float_{suffix} ١٢.٥", - f"unicode_float_{suffix} -१२.५", - f"numeric_character_{suffix} ²", - f"large_{suffix} {10**40 + index}, ratio_{suffix} +.25", - ) - return numeric_headers[(index // 10) % len(numeric_headers)] - - -def _escaped_model_header(_index: int, suffix: str) -> str: - return f'description "escaped \\"value_{suffix}\\"", config (x [a, b])' - - -def _duplicate_or_control_model_header(index: int, suffix: str) -> str: - headers: tuple[str, str] = ( - 'post_hooks [python("\u001c")]', - f"duplicate_{suffix} one, duplicate_{suffix} two", - ) - return headers[index % 20 == 8] - - -def _invalid_model_header(_index: int, suffix: str) -> str: - return f"columns (col_{suffix} (type DECIMAL(10,2))" - - -_MODEL_HEADER_BUILDERS: tuple[Callable[[int, str], str], ...] = ( - _standard_model_header, - _constant_model_header, - _hook_model_header, - _audit_model_header, - _template_model_header, - _unicode_column_model_header, - _numeric_model_header, - _escaped_model_header, - _duplicate_or_control_model_header, - _invalid_model_header, -) - - -def _validate_successful_native_header( - *, - native_values: dict[str, object] | None, - native_offsets: list[tuple[str, int, int]] | None, - parent_values: dict[str, object] | None, -) -> None: - assert native_values is not None - assert model_file_helpers._project_native_header_map(native_values) == parent_values - assert native_offsets is not None - - -def _validate_failed_native_header( - *, - native_values: dict[str, object] | None, - native_offsets: list[tuple[str, int, int]] | None, - parent_values: dict[str, object] | None, -) -> None: - assert native_values is None - assert parent_values is None - assert native_offsets is None - - -_NATIVE_HEADER_VALIDATORS: tuple[Callable[..., None], ...] = ( - _validate_successful_native_header, - _validate_failed_native_header, -) - - -def assert_generated_model_header_corpus_parity() -> bool: - """Assert native MODEL-header parsing matches the Python reference corpus.""" - - headers: list[str] = [ - "config (columns (nested (type INTEGER))), columns (top (type INTEGER))", - "config (columns (nested (type INTEGER)))", - ] - for index in range(9_998): - suffix: str = str(index) - headers.append(_MODEL_HEADER_BUILDERS[index % len(_MODEL_HEADER_BUILDERS)](index, suffix)) - - native_results: list[ - tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None] - ] = _native.parse_model_headers(headers) - for header, (native_values, native_offsets, native_error) in zip( - headers, native_results, strict=True - ): - try: - parent_values: dict[str, object] | None = model_file_helpers._ModelHeaderParser( - header=header - ).parse() - parent_error: str | None = None - except model_file_helpers.ModelHeaderSyntaxError as error: - parent_values = None - parent_error = str(error) - assert native_error == parent_error - _NATIVE_HEADER_VALIDATORS[native_values is None]( - native_values=native_values, - native_offsets=native_offsets, - parent_values=parent_values, - ) - return True - - def expected_or_actual[T](expected: T | None, actual: T) -> T: return (actual, cast(T, expected))[expected is not None] diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index d228860e25..d700029f9c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_loader_functions +from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_python_node_functions from sqlbuild.compiler.discovery.exceptions import LoaderDiscoveryError from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( @@ -170,7 +170,9 @@ def test_given_project_dir_when_discovering_loaders_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - result: tuple[DiscoveredLoaderFunction, ...] = discover_loader_functions(project_dir=tmp_path) + result: tuple[DiscoveredLoaderFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).loaders + ) assert tuple(loader.name for loader in result) == test_case.expected_names assert tuple(loader.destination for loader in result) == test_case.expected_targets @@ -224,4 +226,4 @@ def test_given_loader_import_error_when_discovering_loaders_then_raises_clear_er file_path.write_text(contents, encoding="utf-8") with pytest.raises(LoaderDiscoveryError, match=test_case.expected_error_fragment): - discover_loader_functions(project_dir=tmp_path) + discover_python_node_functions(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 0f81f83eab..c84d689d24 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -7,9 +7,7 @@ import pytest from sqlbuild.compiler.discovery._helpers.filesystem.core import ( - discover_asset_functions, - discover_check_functions, - discover_task_functions, + discover_python_node_functions, ) from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError from sqlbuild.compiler.discovery.models import ( @@ -93,8 +91,12 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - tasks: tuple[DiscoveredTaskFunction, ...] = discover_task_functions(project_dir=tmp_path) - assets: tuple[DiscoveredAssetFunction, ...] = discover_asset_functions(project_dir=tmp_path) + tasks: tuple[DiscoveredTaskFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).tasks + ) + assets: tuple[DiscoveredAssetFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).assets + ) assert tuple(task.name for task in tasks) == test_case.expected_task_names assert ( @@ -148,7 +150,7 @@ def test_given_python_node_import_error_when_discovering_then_raises_clear_error file_path.write_text(contents, encoding="utf-8") with pytest.raises(PythonNodeDiscoveryError, match=test_case.expected_error_fragment): - discover_task_functions(project_dir=tmp_path) + discover_python_node_functions(project_dir=tmp_path) @pytest.mark.parametrize( @@ -211,7 +213,9 @@ def test_given_project_dir_when_discovering_checks_then_returns_expected( file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - checks: tuple[DiscoveredCheckFunction, ...] = discover_check_functions(project_dir=tmp_path) + checks: tuple[DiscoveredCheckFunction, ...] = tuple( + discover_python_node_functions(project_dir=tmp_path).checks + ) assert tuple(check.name for check in checks) == test_case.expected_check_names assert tuple(len(check.depends_on) for check in checks) == ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 1a9ff291a1..aabfd31b0c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -32,9 +32,6 @@ ParseModelSqlErrorTestCase, ParseModelSqlHeaderTestCase, ) -from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( - assert_generated_model_header_corpus_parity, -) @pytest.mark.parametrize( @@ -134,17 +131,6 @@ def test_given_cached_model_header_when_parsing_twice_then_top_level_dictionarie assert "added" not in second -@pytest.mark.parametrize( - "test_case", - [ExpectedBooleanTestCase(description="generated header corpus matches", expected_result=True)], - ids=lambda case: case.description, -) -def test_given_generated_header_corpus_when_native_parsing_then_parent_behavior_is_exact( - test_case: ExpectedBooleanTestCase, -) -> None: - assert assert_generated_model_header_corpus_parity() is test_case.expected_result - - @pytest.mark.parametrize( "test_case", [ExpectedCountTestCase(description="invalid faults preserve order", expected_count=2)], @@ -204,7 +190,6 @@ def test_given_deferred_output_locations_when_discovering_models_then_projection )[0] assert model_file.output_column_locations == {} - assert model_file.output_column_locations_extracted is False assert ( model_file.extract_implicit_alias_columns is test_case.expected_extract_implicit_alias_columns diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py index dc3feeb51c..599e836f63 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/_test_types.py @@ -20,14 +20,6 @@ class BuildCreateTableSqlTestCase: transient: bool = False -@dataclass(frozen=True) -class BuildReadAllSqlTestCase: - description: str - database: str | None - schema: str - expected_contains: tuple[str, ...] - - @dataclass(frozen=True) class BuildInsertSqlTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index 3484661282..11d8c51746 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -16,15 +16,12 @@ class ColumnLineageAnalyzerTestCase: expected_column: str expected_upstream_columns: tuple[str, ...] expected_transform_kind: ColumnTransformKind - expected_internal_scope_names: tuple[str, ...] = () @dataclass(frozen=True) class ProjectLineageGraphTestCase: description: str - expected_trace: tuple[str, ...] expected_consumers: tuple[str, ...] - expected_downstream_trace: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index e01d20d06d..d628fb85c7 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -186,8 +186,6 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expecte ) assert upstream_columns == tuple(sorted(test_case.expected_upstream_columns)) assert column_lineage.transform_kind == test_case.expected_transform_kind - for expected_scope_name in test_case.expected_internal_scope_names: - assert expected_scope_name in {node.scope_name for node in column_lineage.nodes} @pytest.mark.parametrize( @@ -378,15 +376,13 @@ def test_given_select_star_when_building_column_lineage_then_expands_known_schem "test_case", [ ProjectLineageGraphTestCase( - description="traces upstream and downstream through linear project graph", - expected_trace=("b.id->c.id", "a.id->b.id"), + description="finds column consumers through linear project graph", expected_consumers=("a.id->b.id",), - expected_downstream_trace=("a.id->b.id", "b.id->c.id"), ) ], ids=lambda case: case.description, ) -def test_given_linear_project_when_tracing_column_lineage_then_returns_expected_edges( +def test_given_linear_project_when_listing_column_consumers_then_returns_expected_edges( test_case: ProjectLineageGraphTestCase, ) -> None: project: CompiledProject = make_compiled_project( @@ -404,15 +400,6 @@ def test_given_linear_project_when_tracing_column_lineage_then_returns_expected_ result: ProjectColumnLineage | None = build_project_column_lineage(project=project) assert result is not None - trace: tuple[str, ...] = tuple( - edge_label( - edge.source.resource_name, - edge.source.column_name, - edge.target.resource_name, - edge.target.column_name, - ) - for edge in result.trace_column(model_name="c", column_name="id") - ) consumers: tuple[str, ...] = tuple( edge_label( edge.source.resource_name, @@ -422,18 +409,7 @@ def test_given_linear_project_when_tracing_column_lineage_then_returns_expected_ ) for edge in result.column_consumers(resource_name="a", column_name="id") ) - downstream_trace: tuple[str, ...] = tuple( - edge_label( - edge.source.resource_name, - edge.source.column_name, - edge.target.resource_name, - edge.target.column_name, - ) - for edge in result.trace_column_downstream(resource_name="a", column_name="id") - ) - assert trace == test_case.expected_trace assert consumers == test_case.expected_consumers - assert downstream_trace == test_case.expected_downstream_trace @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 2882afb428..1e8b04cec4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -22,7 +21,6 @@ MissingUpstream, ParsedSelector, PathSelector, - PlannerScope, ScenarioArtifactIdentity, ScenarioGraphPlan, ScenarioRelationMap, @@ -185,34 +183,6 @@ class SeedIdentityCsvConfigTestCase: expected_same_identity: bool -@dataclass(frozen=True) -class MarkVersionIdentityStaleActionsTestCase: - description: str - model_key: CompiledObjectKey - change_kind: ChangeKind - previous_version_hash: str - expected_version_hash: str - expected_cascade_present: bool - - -@dataclass(frozen=True) -class DirectParentRunActionTestCase: - description: str - parent_key: CompiledObjectKey - child_key: CompiledObjectKey - expected_cascade_present: bool - expected_root_cause: str | None = None - expected_root_reason: PlanReason | None = None - - -@dataclass(frozen=True) -class DirectIdentityStaleModelNamesTestCase: - description: str - expected_version_hashes: dict[str, str] - built_version_hashes: dict[str, str | None] - expected_stale_model_names: frozenset[str] - - @dataclass(frozen=True) class SelectionStalenessWarningTestCase: description: str @@ -441,13 +411,6 @@ class FindPathKeysErrorTestCase: expected_error_type: type[Exception] -@dataclass(frozen=True) -class PruneUnchangedScopeTestCase: - description: str - scope: PlannerScope - expected_selected_keys: frozenset[CompiledObjectKey] - - @dataclass(frozen=True) class ParseSelectorTestCase: description: str @@ -523,19 +486,6 @@ class ResolveSchemaActionsTestCase: expected_actions: tuple[SchemaAction, ...] -@dataclass(frozen=True) -class BuildLogicalDdlTestCase: - description: str - action: PlanAction - resolved_sql: str - qualified_name: str | None - unique_key: tuple[str, ...] - warehouse_columns: tuple[ColumnInfo, ...] - expected_ddl_fragment: str - merge_exclude_columns: tuple[str, ...] = () - unexpected_ddl_fragments: tuple[str, ...] = () - - @dataclass(frozen=True) class BuildModelWarningsTestCase: description: str @@ -976,13 +926,6 @@ class RunDespiteUnchangedPlanningTestCase: expected_error_fragment: str | None = None -@dataclass(frozen=True) -class ReuseSatisfiedStalenessTestCase: - description: str - reuse_satisfied_model_names: frozenset[str] - expected_warns: bool - - @dataclass(frozen=True) class PlannerStaleWarningScopeTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index b38cfcd1f9..5031448f40 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -65,11 +65,3 @@ class ResolveBackfillTestCase: description: str raw_value: str | None expected_result: BackfillResult - - -@dataclass(frozen=True) -class ResolveSchemaBackfillTestCase: - description: str - replay_on_change: dict[str, str] - findings: tuple[SchemaFinding, ...] - expected_result: BackfillResult diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index 6ef31435d4..d28471cad1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -95,7 +95,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name="new_col", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, ), ), ), @@ -109,7 +109,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="DECIMAL(10, 2)", actual_type="INTEGER", ), @@ -149,7 +149,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="DECIMAL(10, 2)", actual_type="decimal(10,3)", ), @@ -189,7 +189,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="FLOAT", actual_type="INTEGER", ), @@ -234,7 +234,7 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_REMOVED, column_name="old_col", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, actual_type="BOOLEAN", ), ), @@ -256,14 +256,14 @@ SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="FLOAT", actual_type="INTEGER", ), SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name="loaded_at", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="TIMESTAMP", ), ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py deleted file mode 100644 index ebfdaead50..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_ddl.py +++ /dev/null @@ -1,200 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner._helpers.output.strategy import build_logical_ddl -from sqlbuild.compiler.planner.types import PlanAction -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - BuildLogicalDdlTestCase, -) - -_SIMPLE_SQL: str = "SELECT id, name FROM raw_orders" - -_WAREHOUSE_COLUMNS: tuple[ColumnInfo, ...] = ( - ColumnInfo(name="order_id", type="INTEGER"), - ColumnInfo(name="name", type="VARCHAR"), - ColumnInfo(name="amount", type="DECIMAL"), -) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildLogicalDdlTestCase( - description="create view wraps sql in create or replace view", - action=PlanAction.CREATE_VIEW, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="CREATE OR REPLACE VIEW staging.orders AS", - ), - BuildLogicalDdlTestCase( - description="create table wraps sql in create table as", - action=PlanAction.CREATE_TABLE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="CREATE TABLE staging.orders AS", - ), - BuildLogicalDdlTestCase( - description="incremental append wraps sql in insert into", - action=PlanAction.INCREMENTAL_APPEND, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="INSERT INTO staging.orders", - ), - BuildLogicalDdlTestCase( - description="delete_insert produces delete and insert statements", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=(), - expected_ddl_fragment="DELETE FROM staging.orders", - ), - BuildLogicalDdlTestCase( - description="delete_insert includes unique key in where clause", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=(), - expected_ddl_fragment="WHERE (order_id) IN", - ), - BuildLogicalDdlTestCase( - description="merge produces merge into with on clause", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment="MERGE INTO staging.orders AS target", - ), - BuildLogicalDdlTestCase( - description="merge includes update set for non-key columns", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment=( - "WHEN MATCHED THEN UPDATE SET name = __source.name, amount = __source.amount" - ), - ), - BuildLogicalDdlTestCase( - description="merge includes insert clause with all columns", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment="WHEN NOT MATCHED THEN INSERT (order_id, name, amount)", - ), - BuildLogicalDdlTestCase( - description="merge exclusion only omits matched update column", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - merge_exclude_columns=("NAME",), - expected_ddl_fragment="WHEN MATCHED THEN UPDATE SET amount = __source.amount", - unexpected_ddl_fragments=("UPDATE SET name = __source.name",), - ), - BuildLogicalDdlTestCase( - description="merge on clause uses unique key", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id",), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment="ON target.order_id = __source.order_id", - ), - BuildLogicalDdlTestCase( - description="skip action produces empty ddl", - action=PlanAction.SKIP, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="", - ), - BuildLogicalDdlTestCase( - description="create view includes resolved sql in body", - action=PlanAction.CREATE_VIEW, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment=_SIMPLE_SQL, - ), - BuildLogicalDdlTestCase( - description="delete_insert with composite key uses both columns", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id", "line_id"), - warehouse_columns=(), - expected_ddl_fragment="WHERE (order_id, line_id) IN", - ), - BuildLogicalDdlTestCase( - description="load_seed action produces empty ddl", - action=PlanAction.LOAD_SEED, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="", - ), - BuildLogicalDdlTestCase( - description="merge with composite key joins on both columns", - action=PlanAction.INCREMENTAL_MERGE, - resolved_sql=_SIMPLE_SQL, - qualified_name="staging.orders", - unique_key=("order_id", "line_id"), - warehouse_columns=_WAREHOUSE_COLUMNS, - expected_ddl_fragment=( - "ON target.order_id = __source.order_id AND target.line_id = __source.line_id" - ), - ), - BuildLogicalDdlTestCase( - description=("falls back to target name when qualified name is none"), - action=PlanAction.CREATE_TABLE, - resolved_sql=_SIMPLE_SQL, - qualified_name=None, - unique_key=(), - warehouse_columns=(), - expected_ddl_fragment="CREATE TABLE orders AS", - ), - ], - ids=lambda case: case.description, -) -def test_given_action_and_sql_when_building_ddl_then_contains_expected_fragment( - test_case: BuildLogicalDdlTestCase, -) -> None: - target: CompiledRelationLocation = CompiledRelationLocation( - database=None, - schema="staging", - name="orders", - qualified_name=test_case.qualified_name, - ) - - result: str = build_logical_ddl( - action=test_case.action, - resolved_sql=test_case.resolved_sql, - target=target, - unique_key=test_case.unique_key, - warehouse_columns=test_case.warehouse_columns, - merge_exclude_columns=test_case.merge_exclude_columns, - ) - - assert test_case.expected_ddl_fragment in result - unexpected_fragment: str - for unexpected_fragment in test_case.unexpected_ddl_fragments: - assert unexpected_fragment not in result diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index f1a55b2c00..24d7d390b6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -44,7 +44,7 @@ _TYPE_CHANGED_SQLGLOT_FINDING: SchemaFinding = SchemaFinding( kind=SchemaChangeKind.COLUMN_TYPE_CHANGED, column_name="amount", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="DECIMAL", actual_type="INTEGER", ) @@ -52,7 +52,7 @@ _ADDED_SQLGLOT_FINDING: SchemaFinding = SchemaFinding( kind=SchemaChangeKind.COLUMN_ADDED, column_name="new_col", - source=SchemaColumnSource.SQLGLOT, + source=SchemaColumnSource.SQL_ANALYSIS, expected_type="VARCHAR", ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index cf5dd77382..c439e4c0e4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -87,28 +87,6 @@ class ExternalBlockedPlanOutputTestCase: expected_model_names: tuple[str, ...] -@dataclass(frozen=True) -class DirectInputBaselineTestCase: - description: str - models_by_name: dict[str, str] - origin_model_names: tuple[str, ...] - selected_model_name: str - expected_baseline_names: tuple[str, ...] - unexpected_baseline_names: tuple[str, ...] - - -@dataclass(frozen=True) -class LocalNodePlanningTestCase: - description: str - fingerprint_exists: bool - relation_exists: bool - full_refresh: bool - local_hash: str | None - previous_hash: str | None - expected_action: str - expected_reason: str - - @dataclass(frozen=True) class SelectorExpansionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py deleted file mode 100644 index 8e30470fe1..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_local_node_planning.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.planner._helpers.command_planning.local_node_planning import ( - classify_local_node_plan, -) -from sqlbuild.compiler.planner.models import LocalNodePlanInput, LocalNodePlanOutcome -from sqlbuild.compiler.planner.types import LocalNodePlanAction, LocalNodePlanReason -from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( - LocalNodePlanningTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LocalNodePlanningTestCase( - description="full refresh takes precedence over all local state", - fingerprint_exists=False, - relation_exists=False, - full_refresh=True, - local_hash="current", - previous_hash="previous", - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.FULL_REFRESH, - ), - LocalNodePlanningTestCase( - description="missing fingerprint is first run", - fingerprint_exists=False, - relation_exists=True, - full_refresh=False, - local_hash="current", - previous_hash=None, - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.FIRST_RUN, - ), - LocalNodePlanningTestCase( - description="missing relation with fingerprint reruns relation", - fingerprint_exists=True, - relation_exists=False, - full_refresh=False, - local_hash="current", - previous_hash="current", - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.RELATION_MISSING, - ), - LocalNodePlanningTestCase( - description="changed local hash reruns relation", - fingerprint_exists=True, - relation_exists=True, - full_refresh=False, - local_hash="current", - previous_hash="previous", - expected_action=LocalNodePlanAction.RUN, - expected_reason=LocalNodePlanReason.LOCAL_CHANGED, - ), - LocalNodePlanningTestCase( - description="matching local hash is current", - fingerprint_exists=True, - relation_exists=True, - full_refresh=False, - local_hash="current", - previous_hash="current", - expected_action=LocalNodePlanAction.CURRENT, - expected_reason=LocalNodePlanReason.NO_CHANGE, - ), - LocalNodePlanningTestCase( - description="missing local hash skips hash comparison", - fingerprint_exists=True, - relation_exists=True, - full_refresh=False, - local_hash=None, - previous_hash="previous", - expected_action=LocalNodePlanAction.CURRENT, - expected_reason=LocalNodePlanReason.NO_CHANGE, - ), - ], - ids=lambda case: case.description, -) -def test_given_local_node_state_when_classifying_then_returns_expected_outcome( - test_case: LocalNodePlanningTestCase, -) -> None: - outcome: LocalNodePlanOutcome = classify_local_node_plan( - LocalNodePlanInput( - fingerprint_exists=test_case.fingerprint_exists, - relation_exists=test_case.relation_exists, - full_refresh=test_case.full_refresh, - local_hash=test_case.local_hash, - previous_hash=test_case.previous_hash, - ) - ) - - assert outcome.action == test_case.expected_action - assert outcome.reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py index 968741bc70..384066c5bb 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/_test_types.py @@ -1,32 +1,11 @@ -from collections.abc import Callable from dataclasses import dataclass -from pathlib import Path -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.planner.models import ParsedSelector from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import SqlResourceRef -@dataclass(frozen=True) -class PythonLoaderNodeConversionTestCase: - description: str - expected_kind: PythonNodeKind - expected_file_path: Path - expected_relative_path: Path - expected_name: str - expected_depends_on: tuple[Callable[..., object], ...] - expected_dependency_edges: tuple[tuple[str, str], ...] - expected_target: str | None - expected_write_strategy: str | None - expected_cursor_column: str | None - expected_unique_key: tuple[str, ...] - expected_column_names: tuple[str, ...] - expected_contract: str | None - expected_connection_mode: LoaderConnectionMode - - @dataclass(frozen=True) class PythonNodeGraphInventoryTestCase: description: str @@ -95,7 +74,6 @@ class PythonSqlRunLifecycleTestCase: expected_ingress_python_names: frozenset[str] expected_ingress_loader_names: frozenset[str] expected_read_side_python_names: frozenset[str] - expected_read_side_sql_names: frozenset[str] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index b6eea07a48..5970d9c0ec 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -237,6 +237,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node description="identity decorator config keeps each kind's authored field set", expected_decorator_config_keys_by_selector={ "loader:load_events": ( + "columns", "contract", "cursor_column", "destination", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py deleted file mode 100644 index a9491ea7c6..0000000000 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_loaders.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Tests for loader conversion to internal Python-node models.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes._helpers.loaders import ( - build_python_loader_dependency_edges, - build_python_loader_node, -) -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonLoaderMetadata, - DiscoveredPythonNode, - PythonNodeDependencyEdge, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy -from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( - PythonLoaderNodeConversionTestCase, -) -from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import ( - fetch_events, - load_events, -) - - -@pytest.mark.parametrize( - "test_case", - [ - PythonLoaderNodeConversionTestCase( - description="preserves source loader metadata in internal Python-node view", - expected_kind=PythonNodeKind.LOADER, - expected_file_path=Path("/project/loaders/events.py"), - expected_relative_path=Path("loaders/events.py"), - expected_name="load_events", - expected_depends_on=(fetch_events,), - expected_dependency_edges=(("fetch_events", "load_events"),), - expected_target="staging.events", - expected_write_strategy="merge", - expected_cursor_column="updated_at", - expected_unique_key=("event_id", "updated_at"), - expected_column_names=("event_id", "updated_at"), - expected_contract="enforced", - expected_connection_mode=LoaderConnectionMode.EXTERNAL, - ) - ], - ids=lambda case: case.description, -) -def test_given_discovered_loader_when_building_python_node_then_preserves_loader_metadata( - test_case: PythonLoaderNodeConversionTestCase, -) -> None: - loader: DiscoveredLoaderFunction = DiscoveredLoaderFunction( - file_path=test_case.expected_file_path, - relative_path=test_case.expected_relative_path, - name=test_case.expected_name, - function=load_events, - depends_on=(fetch_events,), - destination=test_case.expected_target, - write_strategy=SourceWriteStrategy(test_case.expected_write_strategy or "table"), - cursor_column=test_case.expected_cursor_column, - unique_key=test_case.expected_unique_key, - columns=( - SourceColumnEntry(name="event_id", type="BIGINT"), - SourceColumnEntry(name="updated_at", type="TIMESTAMP"), - ), - contract=test_case.expected_contract, - connection_mode=test_case.expected_connection_mode, - ) - - node: DiscoveredPythonNode = build_python_loader_node(loader=loader) - dependency_edges: tuple[PythonNodeDependencyEdge, ...] = build_python_loader_dependency_edges( - loaders=( - DiscoveredLoaderFunction( - file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), - name="fetch_events", - function=fetch_events, - ), - loader, - ) - ) - - assert node.kind == test_case.expected_kind - assert node.file_path == test_case.expected_file_path - assert node.relative_path == test_case.expected_relative_path - assert node.name == test_case.expected_name - assert node.function is load_events - assert node.depends_on == test_case.expected_depends_on - assert node.tags == () - assert node.group is None - assert node.description is None - assert node.loader == DiscoveredPythonLoaderMetadata( - destination=test_case.expected_target, - write_strategy=SourceWriteStrategy(test_case.expected_write_strategy or "table"), - cursor_column=test_case.expected_cursor_column, - unique_key=test_case.expected_unique_key, - columns=loader.columns, - contract=test_case.expected_contract, - connection_mode=test_case.expected_connection_mode, - ) - assert node.loader is not None - assert tuple(column.name for column in node.loader.columns) == test_case.expected_column_names - assert tuple((edge.upstream_name, edge.downstream_name) for edge in dependency_edges) == ( - test_case.expected_dependency_edges - ) - assert dependency_edges[0].upstream_function is fetch_events - assert dependency_edges[0].downstream_function is load_events diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 703e755973..93ac2e1b3b 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -41,7 +41,6 @@ expected_ingress_python_names=frozenset({"prepare_orders", "load_events"}), expected_ingress_loader_names=frozenset({"load_events"}), expected_read_side_python_names=frozenset(), - expected_read_side_sql_names=frozenset({"raw_orders"}), ), PythonSqlRunLifecycleTestCase( description="classifies read only asset into read-side Python", @@ -55,7 +54,6 @@ expected_ingress_python_names=frozenset(), expected_ingress_loader_names=frozenset(), expected_read_side_python_names=frozenset({"export_orders"}), - expected_read_side_sql_names=frozenset({"orders"}), ), PythonSqlRunLifecycleTestCase( description="classifies intermediate loader dependency into pre sql ingress", @@ -73,7 +71,6 @@ expected_ingress_python_names=frozenset({"fetch_pages"}), expected_ingress_loader_names=frozenset({"fetch_pages"}), expected_read_side_python_names=frozenset({"export_orders"}), - expected_read_side_sql_names=frozenset({"fetch_pages"}), ), PythonSqlRunLifecycleTestCase( description="excludes external loader from ingress so it runs pre-connection", @@ -85,7 +82,6 @@ expected_ingress_python_names=frozenset(), expected_ingress_loader_names=frozenset(), expected_read_side_python_names=frozenset(), - expected_read_side_sql_names=frozenset(), ), ], ids=lambda case: case.description, @@ -103,6 +99,3 @@ def test_given_run_selection_when_building_lifecycle_plan_then_classifies_phases assert result.ingress_python_node_names == test_case.expected_ingress_python_names assert result.ingress_loader_names == test_case.expected_ingress_loader_names assert result.read_side_python_node_names == test_case.expected_read_side_python_names - assert frozenset(key.name for key in result.read_side_sql_keys) == ( - test_case.expected_read_side_sql_names - ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 3daf38b0f2..904a68c487 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -23,9 +23,9 @@ EnumDeclaration, EnumMember, ) +from sqlbuild.compiler.scopes._helpers.visibility import query_target from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._build_scope_index import build_scope_index -from sqlbuild.compiler.scopes.main._query_scope_target import query_scope_target from sqlbuild.compiler.scopes.main._resolve_scope_visibility import resolve_scope_visibility from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup @@ -293,7 +293,7 @@ def test_given_unknown_qualified_target_when_querying_then_distinguishes_unknown ) -> None: lookup: ScopeLookup = build_scope_lookup(index=ScopeIndex()) - query: ScopeTargetQuery = query_scope_target(lookup=lookup, target="model:missing") + query: ScopeTargetQuery = query_target(lookup=lookup, target="model:missing") resolution: VisibilityResolution = resolve_scope_visibility( lookup=lookup, target="model:missing" ) @@ -319,7 +319,7 @@ def test_given_declaration_qualified_target_when_querying_then_returns_declarati ) lookup: ScopeLookup = build_scope_lookup(index=ScopeIndex(declarations=(declaration,))) - query: ScopeTargetQuery = query_scope_target( + query: ScopeTargetQuery = query_target( lookup=lookup, target="constant:minimum_value", ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index fbb96d762d..f9a33b4eca 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -4,9 +4,8 @@ import pytest +from sqlbuild.compiler.scopes._helpers.identities import format_identity, parse_identity from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError -from sqlbuild.compiler.scopes.main._parse_qualified_identity import parse_qualified_identity -from sqlbuild.compiler.scopes.main._qualified_identity_text import format_qualified_identity from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( @@ -43,10 +42,10 @@ def test_given_qualified_identity_when_parsing_and_formatting_then_round_trips( test_case: QualifiedIdentityCase, ) -> None: - parsed: ResourceIdentity | DeclarationIdentity = parse_qualified_identity(value=test_case.text) + parsed: ResourceIdentity | DeclarationIdentity = parse_identity(value=test_case.text) assert parsed == test_case.expected_identity - assert format_qualified_identity(identity=parsed) == test_case.text + assert format_identity(identity=parsed) == test_case.text @pytest.mark.parametrize( @@ -77,7 +76,7 @@ def test_given_bare_name_when_parsing_then_rejects_unqualified_identity( test_case: InvalidIdentityCase, ) -> None: with pytest.raises(test_case.expected_error, match="kind-qualified"): - parse_qualified_identity(value=test_case.value) + parse_identity(value=test_case.value) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py index a7a8d4cf6b..a16976601d 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py @@ -4,12 +4,7 @@ import pytest -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.main._normalize_scope_path import normalize_scope_path -from sqlbuild.compiler.scopes.main._path_is_equal_or_descendant import ( - path_is_equal_or_descendant, -) -from sqlbuild.compiler.scopes.main._scope_is_path_visible import scope_is_path_visible +from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path from sqlbuild.compiler.scopes.types import ScopeKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( PathNormalizationCacheCase, @@ -42,7 +37,7 @@ def test_given_posix_or_windows_path_when_normalizing_then_returns_canonical_posix_path( test_case: PathNormalizationCase, ) -> None: - assert normalize_scope_path(path=test_case.path) == test_case.expected_path + assert normalize_path(path=test_case.path) == test_case.expected_path @pytest.mark.parametrize( @@ -75,59 +70,6 @@ def test_given_repeated_path_when_normalizing_then_reuses_bounded_cache( normalize_path.cache_clear() -@pytest.mark.parametrize( - "test_case", - [ - PathVisibilityCase("global", ScopeKind.GLOBAL, "constants", "tests/orders.sql", True), - PathVisibilityCase( - "inherited_owner", ScopeKind.INHERITED, "models/staging", "models/staging/a.sql", True - ), - PathVisibilityCase( - "inherited_descendant", - ScopeKind.INHERITED, - "models/staging", - "models/staging/orders/a.sql", - True, - ), - PathVisibilityCase( - "local_owner", ScopeKind.LOCAL, "models/staging", "models/staging/a.sql", True - ), - PathVisibilityCase( - "local_descendant", - ScopeKind.LOCAL, - "models/staging", - "models/staging/orders/a.sql", - False, - ), - PathVisibilityCase( - "textual_prefix", ScopeKind.INHERITED, "models/order", "models/orders/a.sql", False - ), - PathVisibilityCase( - "descendant_hidden", - ScopeKind.INHERITED, - "models/staging/orders", - "models/staging/a.sql", - False, - ), - PathVisibilityCase( - "private", ScopeKind.PRIVATE, "models/staging", "models/staging/a.sql", False - ), - ], - ids=lambda case: case.description, -) -def test_given_scope_and_resource_path_when_resolving_then_enforces_component_visibility( - test_case: PathVisibilityCase, -) -> None: - assert ( - scope_is_path_visible( - scope=test_case.scope, - owning_path=test_case.owner, - resource_path=test_case.resource, - ) - is test_case.expected_visible - ) - - @pytest.mark.parametrize( "test_case", [ @@ -141,7 +83,7 @@ def test_given_component_prefix_only_when_comparing_descendant_then_returns_fals test_case: PathVisibilityCase, ) -> None: assert ( - path_is_equal_or_descendant(path=test_case.resource, ancestor=test_case.owner) + is_equal_or_descendant(path=test_case.resource, ancestor=test_case.owner) is test_case.expected_visible ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index aaedf2c0f3..7507f93334 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -1,5 +1,5 @@ from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import dataclass from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RetentionChangePhase @@ -10,11 +10,9 @@ RetentionDirection, RetentionPlanPhase, ) -from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ModelPlanOverride @dataclass(frozen=True) @@ -30,22 +28,6 @@ class LifecycleProgressTestCase: expected_event_types: tuple[str, ...] -@dataclass(frozen=True) -class BuildOutputTestCase: - """Test case for build output formatting.""" - - description: str - result: BuildExecutionResult - expected_output_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] = field(default_factory=tuple) - model_plan_overrides: tuple[ModelPlanOverride, ...] = field(default_factory=tuple) - target: str | None = None - concurrency: int = 1 - elapsed_seconds: float = 1.5 - verbose: bool = False - use_color: bool = False - - @dataclass(frozen=True) class BuildSchedulerSourceLoadTestCase: description: str @@ -151,28 +133,6 @@ class ExternalBuildSourceLoadTestCase: expected_lifecycle_message: str -@dataclass(frozen=True) -class AbbreviatedRowCountTestCase: - description: str - count: int - expected_output: str - - -@dataclass(frozen=True) -class BatchSummaryTestCase: - description: str - batch_count: int | None - rows_affected: int | None - cursor_range_start: str | None - cursor_range_end: str | None - cursor_type: str | None - cursor_grain: str | None - expected_fragments: tuple[str, ...] - batch_size: str | None = None - expected_absent_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_none: bool = False - - @dataclass(frozen=True) class MaterializationRecreatesRelationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index da4a54d744..fb4e0f42ae 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -2,8 +2,6 @@ from __future__ import annotations -from dataclasses import dataclass -from datetime import UTC, datetime from pathlib import Path from typing import Any from unittest.mock import Mock @@ -32,51 +30,11 @@ PlanAction, PlanReason, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.spec.contracts.models import SeedCsvSettings, SourceColumnEntry, SourceEntry from sqlbuild.spec.contracts.types import SourceWriteStrategy -@dataclass(frozen=True) -class ModelPlanOverride: - """Override for a model plan entry in output tests.""" - - name: str - materialization_type: MaterializationType = MaterializationType.TABLE - action: PlanAction = PlanAction.CREATE_TABLE - snapshot_strategy: str | None = None - observed_at_column: str | None = None - historical_input: str | None = None - - -def build_model_result_fields( - *, - name: str, - status: ExecutionStatus, - duration_ms: int = 100, - failed_phase: ExecutionPhase | None = None, - staging_relation: str | None = None, - promoted_relation: str | None = None, - error_message: str | None = None, - audit_results: tuple[AuditExecutionResult, ...] = (), - warning_messages: tuple[str, ...] = (), -) -> dict[str, object]: - return { - "model_name": name, - "status": status, - "duration_ms": duration_ms, - "failed_phase": failed_phase, - "staging_relation": staging_relation, - "promoted_relation": promoted_relation, - "error_message": error_message, - "audit_results": audit_results, - "warning_messages": warning_messages, - } - - def build_audit_result( *, name: str, @@ -135,26 +93,6 @@ def build_model_plan_entry( ) -def build_source_freshness_record( - *, - source_name: str, - data_hash: str, - data_version: str = "2026-06-30T12:00:00", -) -> SourceFreshnessRecord: - return SourceFreshnessRecord( - source_name=source_name, - target_database=None, - target_schema="main", - target_name=source_name, - run_id="run-1", - strategy="adapter", - value_kind="timestamp", - data_version=data_version, - data_version_hash=data_hash, - observed_at=datetime(2026, 6, 30, 12, 1, tzinfo=UTC), - ) - - def build_seed_plan_entry(*, name: str) -> SeedPlanEntry: return SeedPlanEntry( key=CompiledObjectKey(resource_type=CompiledResourceType.SEED, name=name), diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py deleted file mode 100644 index 9f89537fb1..0000000000 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_batch_summary.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.executor.build._helpers.output import ( - _format_abbreviated_row_count, - _format_batch_summary_line, -) -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( - AbbreviatedRowCountTestCase, - BatchSummaryTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - AbbreviatedRowCountTestCase( - description="exact count below abbreviation threshold", - count=438, - expected_output="438 rows", - ), - AbbreviatedRowCountTestCase( - description="single row uses singular label", - count=1, - expected_output="1 row", - ), - AbbreviatedRowCountTestCase( - description="count at threshold abbreviates to K", - count=10_000, - expected_output="10.0K rows", - ), - AbbreviatedRowCountTestCase( - description="millions abbreviate to M", - count=32_689_379, - expected_output="32.7M rows", - ), - AbbreviatedRowCountTestCase( - description="billions abbreviate to B", - count=1_847_293_612, - expected_output="1.8B rows", - ), - AbbreviatedRowCountTestCase( - description="just below threshold stays exact", - count=9_999, - expected_output="9,999 rows", - ), - ], - ids=lambda case: case.description, -) -def test_given_row_count_when_abbreviating_then_matches_expected_format( - test_case: AbbreviatedRowCountTestCase, -) -> None: - result: str = _format_abbreviated_row_count(count=test_case.count) - - assert result == test_case.expected_output - - -@pytest.mark.parametrize( - "test_case", - [ - BatchSummaryTestCase( - description="microbatch result includes batch count size range and rows", - batch_count=5, - batch_size="1mo", - rows_affected=32_689_379, - cursor_range_start="2014-01-01", - cursor_range_end="2015-01-01", - cursor_type="timestamp", - cursor_grain="day", - expected_fragments=( - "5 batches (1mo)", - "2014-01-01", - "2014-12-31", - "32.7M rows", - ), - ), - BatchSummaryTestCase( - description="microbatch result without rows omits row count", - batch_count=3, - rows_affected=None, - cursor_range_start="2014-01-01", - cursor_range_end="2014-04-01", - cursor_type="timestamp", - cursor_grain="day", - expected_fragments=("3 batches",), - expected_absent_fragments=("rows",), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_result_when_formatting_batch_summary_then_matches_contract( - test_case: BatchSummaryTestCase, -) -> None: - result: ModelExecutionResult = ModelExecutionResult( - model_name="test_model", - status=ExecutionStatus.SUCCESS, - batch_count=test_case.batch_count, - batch_size=test_case.batch_size, - rows_affected=test_case.rows_affected, - cursor_range_start=test_case.cursor_range_start, - cursor_range_end=test_case.cursor_range_end, - cursor_type=test_case.cursor_type, - cursor_grain=test_case.cursor_grain, - ) - - line: str | None = _format_batch_summary_line(model_result=result, use_color=False) - - assert line is not None - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in line - for fragment in test_case.expected_absent_fragments: - assert fragment not in line - - -@pytest.mark.parametrize( - "test_case", - [ - BatchSummaryTestCase( - description="non-microbatch result returns none", - batch_count=None, - rows_affected=None, - cursor_range_start=None, - cursor_range_end=None, - cursor_type=None, - cursor_grain=None, - expected_fragments=(), - expected_none=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_non_microbatch_result_when_formatting_batch_summary_then_returns_none( - test_case: BatchSummaryTestCase, -) -> None: - result: ModelExecutionResult = ModelExecutionResult( - model_name="test_model", - status=ExecutionStatus.SUCCESS, - batch_count=test_case.batch_count, - ) - - line: str | None = _format_batch_summary_line(model_result=result, use_color=False) - - assert (line is None) == test_case.expected_none diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py deleted file mode 100644 index 787354b0b0..0000000000 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_output.py +++ /dev/null @@ -1,805 +0,0 @@ -"""Unit tests for build output formatting.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, -) -from sqlbuild.executor.build._helpers.output import format_build_output -from sqlbuild.executor.build.models import ( - BuildExecutionResult, - FunctionExecutionResult, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult -from sqlbuild.executor.testing.types import SqlTestOutcome -from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( - BuildOutputTestCase, - ModelPlanOverride, -) -from tests.unit.src.sqlbuild.executor.build._helpers.helpers import ( - build_audit_result, - build_model_plan_entry, - build_plan_output, - build_seed_plan_entry, -) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildOutputTestCase( - description="microbatch model shows batch count cursor range and row count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="events", - status=ExecutionStatus.SUCCESS, - duration_ms=1000, - batch_count=5, - rows_affected=32689379, - cursor_range_start="2014-01-01", - cursor_range_end="2015-01-01", - cursor_type="timestamp", - cursor_grain="day", - ), - ), - success_count=1, - ), - expected_output_fragments=( - "events", - "5 batches", - "range 2014-01-01 \u2192 2014-12-31", - "32.7M rows", - ), - ), - BuildOutputTestCase( - description="non-microbatch model shows no batch summary line", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=150, - ), - ), - success_count=1, - ), - expected_output_fragments=("orders", "OK"), - expected_absent_fragments=("batches", "range"), - ), - BuildOutputTestCase( - description="success build shows OK status and completed successfully", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=150, - ), - ), - success_count=1, - ), - expected_output_fragments=( - "1/1", - "table", - "orders", - "OK", - "0.15s", - "\u2713 Completed successfully", - "PASS=1", - "FAIL=0", - ), - ), - BuildOutputTestCase( - description="snapshot model output shows strategy and historical shape annotation", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="customer_snapshot", - status=ExecutionStatus.SUCCESS, - duration_ms=150, - ), - ), - success_count=1, - ), - model_plan_overrides=( - ModelPlanOverride( - name="customer_snapshot", - materialization_type=MaterializationType.SNAPSHOT, - action=PlanAction.SNAPSHOT, - snapshot_strategy="check", - observed_at_column="observed_at", - historical_input="snapshot", - ), - ), - expected_output_fragments=( - "1/1", - "snapshot", - "customer_snapshot (check, historical snapshot)", - "OK", - "0.15s", - ), - expected_absent_fragments=("table",), - ), - BuildOutputTestCase( - description="failed model shows FAIL status with phase and failure detail", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.STAGING, - duration_ms=50, - error_code="R002", - error_message="Table raw_orders does not exist", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "1/1", - "FAIL", - "staging", - "\u2717 Completed with errors", - "FAIL=1", - "Failures:", - "orders (staging)", - "error[R002]: Table raw_orders does not exist", - "Table raw_orders does not exist", - ), - ), - BuildOutputTestCase( - description="colorized failed model shows styled coded error and help", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.PROMOTION, - error_code="R007", - error_help="inspect staging relation", - error_message="promotion failed", - ), - ), - failure_count=1, - ), - use_color=True, - expected_output_fragments=( - "\033[38;5;167m\033[1merror[R007]:\033[0m promotion failed", - "\033[2m= help:\033[0m inspect staging relation", - ), - ), - BuildOutputTestCase( - description="skipped model shows SKIP with no duration", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="upstream", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.STAGING, - duration_ms=10, - error_message="bad SQL", - ), - ModelExecutionResult( - model_name="downstream", - status=ExecutionStatus.SKIPPED, - ), - ), - failure_count=1, - skipped_count=1, - ), - expected_output_fragments=( - "1/2", - "upstream", - "FAIL", - "2/2", - "downstream", - "SKIP", - "SKIP=1", - ), - expected_absent_fragments=("downstream" + "0.00s",), - ), - BuildOutputTestCase( - description="warn audit shows WARN with row count in output and warnings section", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.WARN, - row_count=3, - column_name="email", - ), - ), - ), - ), - success_count=1, - warning_count=1, - ), - expected_output_fragments=( - "WARN", - "3 rows", - "\u2713 Completed with warnings", - "WARN=1", - "Warnings:", - "not_null (email) returned 3 rows", - ), - ), - BuildOutputTestCase( - description="passing audit shows PASS with no row count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - ), - ), - ), - ), - success_count=1, - ), - expected_output_fragments=( - "audit", - "not_null (id)", - "PASS", - ), - expected_absent_fragments=( - "Warnings:", - "Failures:", - ), - ), - BuildOutputTestCase( - description="seed line shows seed type and OK status", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - seed_results=( - SeedExecutionResult( - seed_name="country_codes", - status=ExecutionStatus.SUCCESS, - duration_ms=20, - ), - ), - success_count=1, - ), - expected_output_fragments=( - "1/1", - "seed", - "country_codes", - "OK", - "0.02s", - ), - ), - BuildOutputTestCase( - description="failed seed shows failure detail", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - seed_results=( - SeedExecutionResult( - seed_name="waffle_types", - status=ExecutionStatus.FAILED, - duration_ms=20, - error_message="failed to load seed CSV: invalid input syntax", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "seed", - "waffle_types", - "FAIL", - "Failures:", - "waffle_types (seed)", - "failed to load seed CSV: invalid input syntax", - ), - ), - BuildOutputTestCase( - description="failed function shows failure detail", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - function_results=( - FunctionExecutionResult( - function_name="is_completed_order", - status=ExecutionStatus.FAILED, - function_kind="udf", - error_message="function target could not be qualified", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "FAIL=1", - "Failures:", - "is_completed_order (udf)", - "function target could not be qualified", - ), - ), - BuildOutputTestCase( - description="function warning appears in summary and warnings section", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - function_results=( - FunctionExecutionResult( - function_name="is_completed_order_py", - status=ExecutionStatus.SUCCESS, - function_kind="udf", - warning_messages=("fingerprint write skipped",), - ), - ), - success_count=1, - warning_count=1, - ), - expected_output_fragments=( - "\u2713 Completed with warnings", - "PASS=1", - "WARN=1", - "Warnings:", - "is_completed_order_py (udf)", - "fingerprint write skipped", - ), - ), - BuildOutputTestCase( - description="view model shows view type", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="stg_orders", - status=ExecutionStatus.SUCCESS, - duration_ms=10, - ), - ), - success_count=1, - ), - model_plan_overrides=( - ModelPlanOverride( - name="stg_orders", - materialization_type=MaterializationType.VIEW, - action=PlanAction.CREATE_VIEW, - ), - ), - expected_output_fragments=( - "view", - "stg_orders", - "OK", - ), - ), - BuildOutputTestCase( - description="header includes target and concurrency", - result=BuildExecutionResult(status=BuildStatus.SUCCESS), - target="prod", - concurrency=4, - expected_output_fragments=( - "sqb build", - "target: prod", - "concurrency: 4", - ), - ), - BuildOutputTestCase( - description="counter counts seeds and models together", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - seed_results=( - SeedExecutionResult( - seed_name="seed_a", - status=ExecutionStatus.SUCCESS, - duration_ms=10, - ), - ), - model_results=( - ModelExecutionResult( - model_name="model_a", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ModelExecutionResult( - model_name="model_b", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - success_count=3, - ), - expected_output_fragments=( - "1/3", - "seed_a", - "2/3", - "model_a", - "3/3", - "model_b", - "TOTAL=3", - ), - ), - BuildOutputTestCase( - description="failed test shows in summary counts as FAIL", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SKIPPED, - ), - ), - test_results=( - SqlTestExecutionResult( - test_name="test_orders", - outcome=SqlTestOutcome.FAIL, - step_results=( - StepResult( - model_name="orders", - outcome=SqlTestOutcome.FAIL, - mismatched_row_count=1, - ), - ), - error_message="test 'test_orders' failed for models: orders", - ), - ), - failure_count=1, - skipped_count=1, - ), - expected_output_fragments=( - "SKIP", - "expect expected orders", - "FAIL 1 mismatched", - "\u2717 Completed with errors", - "FAIL=1", - "SKIP=1", - "Failures:", - "test_orders (test)", - ), - ), - BuildOutputTestCase( - description="unit test expectation rows show assertion names", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="fact_orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - test_results=( - SqlTestExecutionResult( - test_name="test_fact_orders", - outcome=SqlTestOutcome.PASS, - step_results=( - StepResult( - model_name="fact_orders", - outcome=SqlTestOutcome.PASS, - ), - StepResult( - model_name="assertion line_totals_are_non_negative", - outcome=SqlTestOutcome.PASS, - ), - ), - ), - ), - success_count=2, - ), - expected_output_fragments=( - "test test_fact_orders", - "expect expected fact_orders", - "expect assertion line_totals_are_non_negative PASS", - ), - ), - BuildOutputTestCase( - description="counter excludes tests from total count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - seed_results=( - SeedExecutionResult( - seed_name="seed_data", - status=ExecutionStatus.SUCCESS, - duration_ms=10, - ), - ), - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - test_results=( - SqlTestExecutionResult( - test_name="test_orders", - outcome=SqlTestOutcome.PASS, - step_results=( - StepResult( - model_name="orders", - outcome=SqlTestOutcome.PASS, - ), - ), - ), - ), - success_count=2, - ), - expected_output_fragments=( - "1/2", - "2/2", - "TOTAL=3", - ), - expected_absent_fragments=( - "3/2", - "1/3", - "2/3", - "3/3", - ), - ), - BuildOutputTestCase( - description="staging retained info shows in failure details", - result=BuildExecutionResult( - status=BuildStatus.FAILED, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.FAILED, - failed_phase=ExecutionPhase.AUDIT, - duration_ms=200, - error_message=( - "final audit for 'orders' failed before replacing target table " - "with severity level: error" - ), - staging_relation="main.orders__staging", - ), - ), - failure_count=1, - ), - expected_output_fragments=( - "Failures:", - "orders (audit)", - "final audit for 'orders' failed before replacing target table", - "staging table kept for inspection: main.orders__staging", - ), - ), - BuildOutputTestCase( - description="elapsed time appears in summary", - result=BuildExecutionResult(status=BuildStatus.SUCCESS), - elapsed_seconds=2.34, - expected_output_fragments=("(2.34s)",), - ), - BuildOutputTestCase( - description="delta_and_final audits show phase labels and batch count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.DELTA_AND_FINAL, - ), - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - run_scope_phase=AuditRunScope.FINAL, - ), - ), - ), - ), - success_count=1, - ), - expected_output_fragments=( - "audit (d)", - "audit (f)", - "not_null (id)", - "2/2", - ), - ), - BuildOutputTestCase( - description="final-only audits show plain audit label without batch count", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - ), - ), - ), - ), - success_count=1, - ), - expected_output_fragments=( - "audit", - "not_null (id)", - "PASS", - ), - expected_absent_fragments=( - "audit (d)", - "audit (f)", - ), - ), - BuildOutputTestCase( - description="verbose mode shows executed lifecycle SQL and audit SQL", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - lifecycle_events=( - LifeCycleEvent( - kind=LifeCycleEventKind.SQL, - content="DROP TABLE IF EXISTS main.orders__staging", - ), - LifeCycleEvent( - kind=LifeCycleEventKind.LOG, - content="building partition 2024-01-01", - ), - LifeCycleEvent( - kind=LifeCycleEventKind.SQL, - content="CREATE OR REPLACE TABLE main.orders__staging AS SELECT 1", - ), - ), - audit_results=( - build_audit_result( - name="not_null", - outcome=AuditOutcome.PASS, - column_name="id", - ), - ), - ), - ), - success_count=1, - ), - verbose=True, - expected_output_fragments=( - "DROP TABLE IF EXISTS main.orders__staging", - "log building partition 2024-01-01", - "CREATE OR REPLACE TABLE main.orders__staging AS SELECT 1;", - "SELECT 1;", - "orders", - "OK", - ), - expected_absent_fragments=("CREATE TABLE main.orders AS SELECT 1",), - ), - BuildOutputTestCase( - description="non-verbose mode omits model DDL", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - ), - ), - success_count=1, - ), - verbose=False, - expected_output_fragments=( - "orders", - "OK", - ), - expected_absent_fragments=("CREATE TABLE",), - ), - BuildOutputTestCase( - description="colorized verbose mode shows lifecycle log messages dimmed", - result=BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=( - ModelExecutionResult( - model_name="orders", - status=ExecutionStatus.SUCCESS, - duration_ms=100, - lifecycle_events=( - LifeCycleEvent( - kind=LifeCycleEventKind.LOG, - content="building partition 2024-01-01", - ), - ), - ), - ), - success_count=1, - ), - verbose=True, - use_color=True, - expected_output_fragments=("\033[2m log building partition 2024-01-01\033[0m",), - ), - ], - ids=lambda case: case.description, -) -def test_given_build_result_when_formatting_output_then_contains_expected_fragments( - test_case: BuildOutputTestCase, -) -> None: - override_map: dict[str, ModelPlanOverride] = {o.name: o for o in test_case.model_plan_overrides} - plan_entries: list[ModelPlanEntry] = [] - model_result: ModelExecutionResult - for model_result in test_case.result.model_results: - override: ModelPlanOverride | None = override_map.get(model_result.model_name) - mat_type: MaterializationType = getattr( - override, "materialization_type", MaterializationType.TABLE - ) - action: PlanAction = getattr(override, "action", PlanAction.CREATE_TABLE) - plan_entries.append( - build_model_plan_entry( - name=model_result.model_name, - materialization_type=mat_type, - action=action, - snapshot_strategy=getattr(override, "snapshot_strategy", None), - observed_at_column=getattr(override, "observed_at_column", None), - historical_input=getattr(override, "historical_input", None), - ) - ) - - seed_entries: tuple[SeedPlanEntry, ...] = tuple( - build_seed_plan_entry(name=sr.seed_name) for sr in test_case.result.seed_results - ) - - plan: object = build_plan_output( - model_entries=tuple(plan_entries), - seed_entries=seed_entries, - ) - - output: str = format_build_output( - result=test_case.result, - plan=plan, - target=test_case.target, - concurrency=test_case.concurrency, - elapsed_seconds=test_case.elapsed_seconds, - use_color=test_case.use_color, - verbose=test_case.verbose, - ) - - expected_fragment: str - for expected_fragment in test_case.expected_output_fragments: - assert expected_fragment in output, ( - f"Expected fragment {expected_fragment!r} not found in output:\n{output}" - ) - - absent_fragment: str - for absent_fragment in test_case.expected_absent_fragments: - assert absent_fragment not in output, ( - f"Fragment {absent_fragment!r} should not be in output:\n{output}" - ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 9b142958f4..aa0eb7cfbf 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -408,31 +408,9 @@ def diff_rows( ) -> RowDiffResult: raise NotImplementedError - def count_rows( - self, - connection: Any, - *, - relation: str, - cursor_column: str | None = None, - start_cursor: CursorValue | None = None, - end_cursor: CursorValue | None = None, - ) -> int: - raise NotImplementedError - def relation_names_match(self, left: str, right: str) -> bool: return left == right - def render_query_with_cursor_bounds( - self, - *, - sql: str, - cursor_column: str, - cursor_start: str, - cursor_end: str | None, - cursor_type: object, - ) -> str: - raise NotImplementedError - def render_seed_select_before_cursor( self, *, diff --git a/tests/unit/src/sqlbuild/executor/load/test_rows.py b/tests/unit/src/sqlbuild/executor/load/test_rows.py index ee6f2c5b22..afcc0b9c14 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_rows.py +++ b/tests/unit/src/sqlbuild/executor/load/test_rows.py @@ -13,7 +13,7 @@ from sqlbuild.executor.load._helpers.rows import ( build_rows_sql, iter_loader_row_batches, - normalize_loader_rows, + iter_normalized_loader_rows, update_loader_rows_schema, ) from sqlbuild.executor.load.models import LoaderRowsSchema @@ -176,7 +176,7 @@ def test_given_reserved_loader_row_columns_when_executing_sql_then_quotes_identi def test_given_loader_return_value_when_normalizing_then_returns_expected_rows( test_case: LoaderRowsNormalizeTestCase, ) -> None: - assert normalize_loader_rows(test_case.value) == test_case.expected_rows + assert tuple(iter_normalized_loader_rows(test_case.value)) == test_case.expected_rows @pytest.mark.parametrize( @@ -209,7 +209,7 @@ def test_given_invalid_loader_return_value_when_normalizing_then_raises( test_case: LoaderRowsNormalizeErrorTestCase, ) -> None: with pytest.raises(ExecutorInputError) as exc_info: - normalize_loader_rows(test_case.value) + tuple(iter_normalized_loader_rows(test_case.value)) assert test_case.expected_error_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py index 38d10e759f..1a1be44d0f 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py @@ -52,8 +52,6 @@ def test_given_python_identity_when_writing_fingerprint_then_uses_fingerprint_st identity=PythonNodeIdentity( node_type="task", node_name="build_orders", - object_module="tasks.orders", - object_qualname="build_orders", source_path="tasks/orders.py", source_hash="source-hash", definition_hash="definition-hash", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py index 58155442f3..98819bdafa 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py @@ -5,10 +5,7 @@ import pytest from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan -from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import ( - build_ingress_lifecycle_nodes, - build_read_side_python_lifecycle_nodes, -) +from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import build_ingress_lifecycle_nodes from sqlbuild.executor.scheduling.models import LifecycleExecutionNode from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( PythonNodeLifecycleNodeBuildTestCase, @@ -64,49 +61,3 @@ def test_given_lifecycle_plan_when_building_ingress_nodes_then_returns_python_sc assert tuple(lifecycle_node_payload_name(node) for node in result) == ( test_case.expected_payload_names ) - - -@pytest.mark.parametrize( - "test_case", - [ - PythonNodeLifecycleNodeBuildTestCase( - description="builds read only asset scheduler node", - python_graph_case="orders", - selected_names=frozenset({"export_orders"}), - expected_names=("export_orders",), - expected_kinds=("asset",), - expected_upstream_names=((),), - expected_payload_names=("export_orders",), - ), - PythonNodeLifecycleNodeBuildTestCase( - description="builds read only task scheduler node", - python_graph_case="orders", - selected_names=frozenset({"prepare_orders"}), - expected_names=("prepare_orders",), - expected_kinds=("task",), - expected_upstream_names=((),), - expected_payload_names=("prepare_orders",), - ), - ], - ids=lambda case: case.description, -) -def test_given_lifecycle_plan_when_building_read_side_nodes_then_returns_python_scheduler_nodes( - test_case: PythonNodeLifecycleNodeBuildTestCase, -) -> None: - graph: PythonNodeGraph = python_graph_for_lifecycle_case(test_case.python_graph_case) - plan: PythonSqlRunLifecyclePlan = build_lifecycle_plan_for_selected_python_names( - graph=graph, - selected_names=test_case.selected_names, - ) - - result: tuple[LifecycleExecutionNode, ...] = build_read_side_python_lifecycle_nodes( - plan=plan, - python_graph=graph, - ) - - assert tuple(node.name for node in result) == test_case.expected_names - assert tuple(node.kind for node in result) == test_case.expected_kinds - assert tuple(node.upstream_names for node in result) == test_case.expected_upstream_names - assert tuple(lifecycle_node_payload_name(node) for node in result) == ( - test_case.expected_payload_names - ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index f7c9f78ce3..a0be8545ff 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -1,5 +1,4 @@ from dataclasses import dataclass, field -from datetime import datetime from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent from sqlbuild.executor.run.types import AuditGateReuseReason, AuditGateStatus, ExecutionPhase @@ -168,26 +167,6 @@ class ReportedRowsAffectedTestCase: expected_rows_affected: int | None -@dataclass(frozen=True) -class MicrobatchCursorDiscoveryTestCase: - description: str - warehouse_column_type: str - cursor_min: object - cursor_max: object - cursor_type: str - expected_start: str - expected_end: str - - -@dataclass(frozen=True) -class MicrobatchCursorDiscoveryFailureTestCase: - description: str - warehouse_column_type: str - cursor_min: object - cursor_max: object - expected_error_fragment: str - - @dataclass(frozen=True) class RuntimeTargetMaxTestCase: description: str @@ -332,43 +311,6 @@ class PythonHookInvocationTestCase: expected_recorded_events: tuple[str, ...] -@dataclass(frozen=True) -class PermanentRequirementTestCase: - description: str - source_created_at: datetime - expected_operation_kind: str - expected_retention_days: int - - -@dataclass(frozen=True) -class PermanentPromotionTestCase: - description: str - initial_state: str - operation_identity: str - expected_timeline: tuple[str, ...] - expected_completion_time: datetime | None - - -@dataclass(frozen=True) -class PermanentArchiveConflictTestCase: - description: str - archive_generation_offset_seconds: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class PermanentPersistedConflictTestCase: - description: str - expected_error_fragment: str - - -@dataclass(frozen=True) -class PermanentIdentifierFitTestCase: - description: str - identifier_limit: int - expected_prefix: str - - @dataclass(frozen=True) class PythonHookSkipTestCase: description: str @@ -450,30 +392,6 @@ class TryWriteFingerprintAuditGateTestCase: expected_result_count: int -@dataclass(frozen=True) -class AuditGateReuseDecisionTestCase: - description: str - metadata_mode: str - status: AuditGateStatus - planned_attached_column_name: str | None - planned_resolved_sql: str - expected_reusable: bool - expected_reason: AuditGateReuseReason - expected_reusable_count: int - expected_missing_count: int - planned_always_run: bool = False - - -@dataclass(frozen=True) -class AuditGatePartialReuseDecisionTestCase: - description: str - changed_resolved_sql: str - expected_reusable: bool - expected_reason: AuditGateReuseReason - expected_reusable_count: int - expected_missing_count: int - - @dataclass(frozen=True) class ReuseFromAuditGateDecisionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py index b5102c1f12..1132488185 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py @@ -20,7 +20,6 @@ parse_audit_gate_metadata, render_audit_gate_metadata, reuse_from_audit_gate_reuse_decision, - same_target_audit_gate_reuse_decision, ) from sqlbuild.executor.run.models import ( AuditGateMetadata, @@ -31,8 +30,6 @@ from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( AuditGateMetadataFailureTestCase, AuditGateMetadataSchemaTestCase, - AuditGatePartialReuseDecisionTestCase, - AuditGateReuseDecisionTestCase, FingerprintAuditGateEdgeTestCase, FingerprintAuditGateMetadataTestCase, FingerprintAuditGateNoAuditsTestCase, @@ -166,7 +163,7 @@ def test_given_invalid_metadata_when_deciding_reuse_then_denies_with_debug_diagn ) -> None: caplog.set_level(logging.DEBUG, logger="sqlbuild.execution") - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( + decision: AuditGateReuseDecision = reuse_from_audit_gate_reuse_decision( metadata_json=test_case.metadata_json, model_audits=(build_fingerprint_audit_plan_entry(),), ) @@ -263,250 +260,6 @@ def test_given_edge_audit_results_when_building_fingerprint_metadata_then_status assert len(audit_gate["results"]) == test_case.expected_result_count # type: ignore[arg-type] -@pytest.mark.parametrize( - "test_case", - [ - AuditGateReuseDecisionTestCase( - description="matching passed proof is reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=True, - expected_reason=AuditGateReuseReason.REUSABLE, - expected_reusable_count=1, - expected_missing_count=0, - ), - AuditGateReuseDecisionTestCase( - description="failed proof is not reusable", - metadata_mode="written", - status=AuditGateStatus.FAILED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.NON_PASSING, - expected_reusable_count=0, - expected_missing_count=0, - ), - AuditGateReuseDecisionTestCase( - description="binding set change is not reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="customer_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.BINDING_SET_CHANGED, - expected_reusable_count=0, - expected_missing_count=1, - ), - AuditGateReuseDecisionTestCase( - description="same binding with changed execution SQL is not reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id < 0", - expected_reusable=False, - expected_reason=AuditGateReuseReason.AUDIT_CHANGED, - expected_reusable_count=0, - expected_missing_count=1, - ), - AuditGateReuseDecisionTestCase( - description="always_run audit is not reusable", - metadata_mode="written", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.ALWAYS_RUN, - expected_reusable_count=0, - expected_missing_count=0, - planned_always_run=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_prior_audit_gate_when_deciding_same_target_reuse_then_returns_decision( - test_case: AuditGateReuseDecisionTestCase, -) -> None: - audit: AuditPlanEntry = build_fingerprint_audit_plan_entry() - audit_result: AuditExecutionResult = build_fingerprint_audit_result(outcome="pass") - metadata_json: str = model_fingerprint_metadata_with_audit_gate( - metadata_json="{}", - model_audits=(audit,), - audit_results=(audit_result,), - run_id="run_1", - ) - metadata: dict[str, object] = json.loads(metadata_json) - audit_gate_value: object = metadata["audit_gate"] - assert isinstance(audit_gate_value, dict) - audit_gate: dict[str, object] = audit_gate_value - audit_gate["status"] = test_case.status.value - metadata_json = json.dumps(metadata, sort_keys=True, separators=(",", ":")) - - planned_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - attached_column_name=test_case.planned_attached_column_name, - resolved_sql=test_case.planned_resolved_sql, - always_run=test_case.planned_always_run, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json=metadata_json, - model_audits=(planned_audit,), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - -@pytest.mark.parametrize( - "test_case", - [ - AuditGateReuseDecisionTestCase( - description="missing proof is not reusable", - metadata_mode="missing", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.MISSING, - expected_reusable_count=0, - expected_missing_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_audit_gate_when_deciding_same_target_reuse_then_returns_missing( - test_case: AuditGateReuseDecisionTestCase, -) -> None: - planned_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - attached_column_name=test_case.planned_attached_column_name, - resolved_sql=test_case.planned_resolved_sql, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json="{}", - model_audits=(planned_audit,), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - -@pytest.mark.parametrize( - "test_case", - [ - AuditGateReuseDecisionTestCase( - description="malformed results are not reusable", - metadata_mode="malformed_results", - status=AuditGateStatus.PASSED, - planned_attached_column_name="order_id", - planned_resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - expected_reusable=False, - expected_reason=AuditGateReuseReason.MALFORMED, - expected_reusable_count=0, - expected_missing_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_malformed_audit_gate_when_deciding_same_target_reuse_then_returns_malformed( - test_case: AuditGateReuseDecisionTestCase, -) -> None: - audit: AuditPlanEntry = build_fingerprint_audit_plan_entry() - audit_result: AuditExecutionResult = build_fingerprint_audit_result(outcome="pass") - metadata_json: str = model_fingerprint_metadata_with_audit_gate( - metadata_json="{}", - model_audits=(audit,), - audit_results=(audit_result,), - run_id="run_1", - ) - metadata: dict[str, object] = json.loads(metadata_json) - audit_gate_value: object = metadata["audit_gate"] - assert isinstance(audit_gate_value, dict) - audit_gate: dict[str, object] = audit_gate_value - audit_gate["results"] = {"not": "a list"} - metadata_json = json.dumps(metadata, sort_keys=True, separators=(",", ":")) - planned_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - attached_column_name=test_case.planned_attached_column_name, - resolved_sql=test_case.planned_resolved_sql, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json=metadata_json, - model_audits=(planned_audit,), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - -@pytest.mark.parametrize( - "test_case", - [ - AuditGatePartialReuseDecisionTestCase( - description="one changed audit reports reusable and missing binding keys", - changed_resolved_sql="SELECT customer_id FROM analytics.orders WHERE customer_id < 0", - expected_reusable=False, - expected_reason=AuditGateReuseReason.AUDIT_CHANGED, - expected_reusable_count=1, - expected_missing_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_one_changed_audit_when_deciding_same_target_reuse_then_returns_partial_keys( - test_case: AuditGatePartialReuseDecisionTestCase, -) -> None: - unchanged_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - name="not_null_orders", - attached_column_name="order_id", - resolved_sql="SELECT order_id FROM analytics.orders WHERE order_id IS NULL", - ) - changed_prior_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - name="not_null_customers", - attached_column_name="customer_id", - resolved_sql="SELECT customer_id FROM analytics.orders WHERE customer_id IS NULL", - ) - metadata_json: str = model_fingerprint_metadata_with_audit_gate( - metadata_json="{}", - model_audits=(unchanged_audit, changed_prior_audit), - audit_results=( - build_fingerprint_audit_result( - outcome="pass", - audit_name="not_null_orders", - attached_column_name="order_id", - ), - build_fingerprint_audit_result( - outcome="pass", - audit_name="not_null_customers", - attached_column_name="customer_id", - ), - ), - run_id="run_1", - ) - changed_current_audit: AuditPlanEntry = build_fingerprint_audit_plan_entry_with_options( - name="not_null_customers", - attached_column_name="customer_id", - resolved_sql=test_case.changed_resolved_sql, - ) - - decision: AuditGateReuseDecision = same_target_audit_gate_reuse_decision( - metadata_json=metadata_json, - model_audits=(unchanged_audit, changed_current_audit), - ) - - assert decision.reusable is test_case.expected_reusable - assert decision.reason == test_case.expected_reason - assert len(decision.reusable_binding_keys) == test_case.expected_reusable_count - assert len(decision.missing_binding_keys) == test_case.expected_missing_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py index 62cc131a12..0bf29e1032 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py @@ -181,7 +181,7 @@ def test_given_existing_destination_when_promoting_then_terminal_records_actual_ ( PromotionProgressTestCase( description="unsupported catalogued strategy fails honestly", - strategy="create_new", + strategy="build_aside", expected_method="relation_exists", expected_event_types=("operation_started", "operation_failed"), ), @@ -196,7 +196,7 @@ def test_given_unsupported_strategy_when_promoting_then_failed_terminal_records_ dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) adapter: Mock = Mock(adapter_name="snowflake") adapter.relation_exists.return_value = True - adapter.default_promotion_strategy.return_value = PromotionStrategy(test_case.strategy) + adapter.default_promotion_strategy.return_value = test_case.strategy with dispatcher_scope(dispatcher), pytest.raises(ExecutorInputError, match="Unsupported"): promote_relation_to_destination( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index 7ed8c637d8..7fa1804900 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -3,12 +3,10 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt.models import ( DbtCliOptions, DbtCommandResult, DbtInteropParsedArgs, - DbtLsNode, ) from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig @@ -100,17 +98,6 @@ class DbtSelectionErrorTestCase: expected_error_fragment: str -@dataclass(frozen=True) -class DbtExecutionSpacingTestCase: - description: str - expected_spacing_fragment: str - expected_no_work_spacing_fragment: str - unexpected_no_blank_fragment: str - unexpected_no_work_no_blank_fragment: str - unexpected_extra_blank_fragment: str - unexpected_no_work_extra_blank_fragment: str - - @dataclass(frozen=True) class DbtCompileFullRefreshPipelineTestCase: description: str @@ -211,22 +198,6 @@ class DbtExecutionSelectionStatusTestCase: expected_completion_fragment: str -@dataclass(frozen=True) -class DbtRunResultsFallbackRenderTestCase: - description: str - unique_id: str - status: str - message: str - expected_output_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class DbtExecutionTotalRenderTestCase: - description: str - expected_output_fragments: tuple[str, ...] - unexpected_output_fragments: tuple[str, ...] - - @dataclass(frozen=True) class DbtExecutionSummaryFooterTestCase: description: str @@ -321,21 +292,6 @@ class DbtPlanTestCase: expected_json_fragments: tuple[str, ...] -@dataclass(frozen=True) -class DbtPlanHumanFormatterTestCase: - description: str - dbt_ls_nodes: tuple[DbtLsNode, ...] - sqlbuild_model_names: tuple[str, ...] - sqlbuild_plan_model_names: tuple[str, ...] - display_limit: int | None - use_color: bool - expected_human_fragments: tuple[str, ...] - expected_human_regex_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] - sqlbuild_command_argvs: tuple[tuple[str, ...], ...] = () - sqlbuild_plan_output: PlanOutput | None = None - - @dataclass(frozen=True) class DbtPlanOrchestrationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py index 51d6bc66ed..ee1d73ea38 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/_test_types.py @@ -4,8 +4,6 @@ from dataclasses import dataclass, field -from sqlbuild.adapter.contract.types import BuiltinAdapter - @dataclass(frozen=True) class DltLoaderDiscoveryTestCase: @@ -38,12 +36,6 @@ class DltDestinationErrorTestCase: destination_config: dict[str, object] | None = None -@dataclass(frozen=True) -class DltDestinationCoverageTestCase: - description: str - expected_adapters: frozenset[BuiltinAdapter] - - @dataclass(frozen=True) class DltProgressCollectorTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py b/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py index 96b4e1a935..37f92ee27f 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/test_destination.py @@ -4,13 +4,10 @@ import pytest -from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.integrations.dlt._helpers.destination import build_dlt_destination -from sqlbuild.integrations.dlt.constants import DLT_DESTINATION_ADAPTERS from sqlbuild.integrations.dlt.exceptions import DltIntegrationError from sqlbuild.integrations.dlt.models import DltDestinationConfig from tests.unit.src.sqlbuild.integrations.dlt._test_types import ( - DltDestinationCoverageTestCase, DltDestinationErrorTestCase, DltDestinationTestCase, ) @@ -172,19 +169,3 @@ def test_given_real_warehouse_without_schema_when_building_dlt_destination_then_ ) assert test_case.expected_error_fragment in str(exc_info.value) - - -@pytest.mark.parametrize( - "test_case", - [ - DltDestinationCoverageTestCase( - description="covers all builtin adapters", - expected_adapters=frozenset(BuiltinAdapter), - ) - ], - ids=lambda case: case.description, -) -def test_given_builtin_adapters_when_checking_dlt_destination_support_then_all_are_explicit( - test_case: DltDestinationCoverageTestCase, -) -> None: - assert DLT_DESTINATION_ADAPTERS == test_case.expected_adapters diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index b22f40f168..c55bb96eac 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -147,13 +147,14 @@ class NeutralizeInterpolationTestCase: @dataclass(frozen=True) -class MapOffsetTestCase: - """Test case for neutralized-to-original offset mapping.""" +class DialectNeutralizeInterpolationTestCase: + """Test case for interpolation neutralization under one lint dialect.""" description: str + dialect: str body: str - neutralized_offset: int - expected_original_offset: int + expected_neutralized: str + expected_original_texts: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index f3a00e4407..7d4ed6267d 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -6,14 +6,13 @@ import sqlbuild._native as _native from sqlbuild.lint._helpers.sqlbuild_tokens import ( - map_neutralized_offset, neutralize_interpolation, restore_interpolation, ) from sqlbuild.lint.exceptions import InterpolationRestorationError from sqlbuild.lint.models import InterpolationSite from tests.unit.src.sqlbuild.lint._helpers._test_types import ( - MapOffsetTestCase, + DialectNeutralizeInterpolationTestCase, NeutralizeInterpolationTestCase, RestoreFailureTestCase, RestoreInterpolationTestCase, @@ -147,11 +146,18 @@ def test_given_body_when_neutralizing_then_sentinels_replace_interpolation( ) -> None: neutralized: str sites: tuple[InterpolationSite, ...] - neutralized, sites = neutralize_interpolation(body=test_case.body) + neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") assert neutralized == test_case.expected_neutralized assert tuple(site.original_text for site in sites) == test_case.expected_original_texts native: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( - _native.prepare_lint_sql(test_case.body, test_case.body, []) + _native.prepare_lint_sql( + { + "expanded": test_case.body, + "before_expansion": test_case.body, + "prior_sites": [], + "dialect": "generic", + } + ) ) assert native is not None assert native[0] == test_case.expected_neutralized @@ -161,71 +167,83 @@ def test_given_body_when_neutralizing_then_sentinels_replace_interpolation( @pytest.mark.parametrize( "test_case", [ - NeutralizeInterpolationTestCase( - description="sentinels are unique across sites", - body="SELECT @a, @b, @@c, ${d} FROM t", - expected_neutralized=( - "SELECT __sqb_lint_0__, __sqb_lint_1__, __sqb_lint_2__, __sqb_lint_3__ FROM t" - ), - expected_original_texts=("@a", "@b", "@@c", "${d}"), + DialectNeutralizeInterpolationTestCase( + description="databricks backtick keeps a close paren inside a macro call", + dialect="databricks", + body="SELECT @m(`a)b` x) AS y", + expected_neutralized="SELECT __sqb_lint_0__ AS y", + expected_original_texts=("@m(`a)b` x)",), + ), + DialectNeutralizeInterpolationTestCase( + description="bigquery backtick apostrophe does not hide a later macro", + dialect="bigquery", + body='SELECT `customer\'s id` AS customer_id, @label("order_id") AS order_id', + expected_neutralized="SELECT `customer's id` AS customer_id, __sqb_lint_0__ AS order_id", + expected_original_texts=('@label("order_id")',), + ), + DialectNeutralizeInterpolationTestCase( + description="postgres backtick remains ordinary code", + dialect="postgres", + body="SELECT @m(`a)b` x) AS y", + expected_neutralized="SELECT __sqb_lint_0__b` x) AS y", + expected_original_texts=("@m(`a)",), + ), + DialectNeutralizeInterpolationTestCase( + description="sqlserver backtick remains ordinary code", + dialect="tsql", + body="SELECT @m(`a)b` x) AS y", + expected_neutralized="SELECT __sqb_lint_0__b` x) AS y", + expected_original_texts=("@m(`a)",), ), ], ids=lambda case: case.description, ) -def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( - test_case: NeutralizeInterpolationTestCase, +def test_given_dialect_quoting_when_neutralizing_then_python_and_native_sites_agree( + test_case: DialectNeutralizeInterpolationTestCase, ) -> None: neutralized: str sites: tuple[InterpolationSite, ...] - neutralized, sites = neutralize_interpolation(body=test_case.body) + neutralized, sites = neutralize_interpolation(body=test_case.body, dialect=test_case.dialect) assert neutralized == test_case.expected_neutralized - sentinels: tuple[str, ...] = tuple(site.sentinel for site in sites) - assert len(set(sentinels)) == len(test_case.expected_original_texts) + assert tuple(site.original_text for site in sites) == test_case.expected_original_texts + native: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( + _native.prepare_lint_sql( + { + "expanded": test_case.body, + "before_expansion": test_case.body, + "prior_sites": [], + "dialect": test_case.dialect, + } + ) + ) + assert native is not None + assert native[0] == test_case.expected_neutralized + assert tuple(InterpolationSite(*site) for site in native[1]) == sites @pytest.mark.parametrize( "test_case", [ - MapOffsetTestCase( - description="offset before any site is unchanged", - body="SELECT @a FROM t", - neutralized_offset=0, - expected_original_offset=0, - ), - MapOffsetTestCase( - description="offset at a sentinel start maps to the interpolation start", - body="SELECT @a FROM t", - neutralized_offset=7, - expected_original_offset=7, - ), - MapOffsetTestCase( - description="offset inside a sentinel clamps to the interpolation start", - body="SELECT @a FROM t", - neutralized_offset=12, - expected_original_offset=7, - ), - MapOffsetTestCase( - description="offset after a sentinel is shifted back by the length delta", - body="SELECT @a FROM t", - neutralized_offset=22, - expected_original_offset=10, - ), - MapOffsetTestCase( - description="offset after two sentinels accumulates both deltas", - body="SELECT @a, @b FROM t", - neutralized_offset=39, - expected_original_offset=15, + NeutralizeInterpolationTestCase( + description="sentinels are unique across sites", + body="SELECT @a, @b, @@c, ${d} FROM t", + expected_neutralized=( + "SELECT __sqb_lint_0__, __sqb_lint_1__, __sqb_lint_2__, __sqb_lint_3__ FROM t" + ), + expected_original_texts=("@a", "@b", "@@c", "${d}"), ), ], ids=lambda case: case.description, ) -def test_given_neutralized_offset_when_mapping_then_authored_offset_matches( - test_case: MapOffsetTestCase, +def test_given_many_sites_when_neutralizing_then_every_sentinel_is_distinct( + test_case: NeutralizeInterpolationTestCase, ) -> None: + neutralized: str sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body) - mapped: int = map_neutralized_offset(offset=test_case.neutralized_offset, sites=sites) - assert mapped == test_case.expected_original_offset + neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") + assert neutralized == test_case.expected_neutralized + sentinels: tuple[str, ...] = tuple(site.sentinel for site in sites) + assert len(set(sentinels)) == len(test_case.expected_original_texts) @pytest.mark.parametrize( @@ -283,7 +301,7 @@ def test_given_fixed_text_when_restoring_then_original_interpolation_returns( test_case: RestoreInterpolationTestCase, ) -> None: sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body) + _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") restored: str = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) assert restored == test_case.expected_restored @@ -310,7 +328,7 @@ def test_given_mangled_sentinels_when_restoring_then_error_is_raised( test_case: RestoreFailureTestCase, ) -> None: sites: tuple[InterpolationSite, ...] - _neutralized, sites = neutralize_interpolation(body=test_case.body) + _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") with pytest.raises(InterpolationRestorationError) as error: _ = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) assert test_case.expected_message_fragment in str(error.value) diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 6f9b2ffb29..d4c8801470 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -271,9 +271,9 @@ def test_given_generic_audit_argument_when_mapping_then_generated_region_is_not_ project_dir=tmp_path, file_path=audit, contents=contents, - body_start=10, - body_end=len(contents), + body_range=(10, len(contents)), context=context, + dialect="duckdb", ) sentinel_offset: int = body.lint_text.index("__sqlbuild_audit_parameter_0__") mapped: MappedOffset = map_expanded_offset(offset=sentinel_offset, passes=body.passes) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py index 08dde06482..3210501fc8 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/_test_types.py @@ -1,34 +1,12 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class EventExporterDecoratorTestCase: - description: str - expected_name: str - - -@dataclass(frozen=True) -class InvalidEventExporterNameTestCase: - description: str - name: str - expected_error: str - - @dataclass(frozen=True) class SinkApiTestCase: description: str expected_name: str | None = None -@dataclass(frozen=True) -class EventExporterFilterTestCase: - description: str - event_kinds: object - min_severity: str - expected_error: str | None = None - expected_kinds: frozenset[str] | None = None - - @dataclass(frozen=True) class EventExportPolicyTestCase: description: str diff --git a/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py b/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py index 25fd91b2d5..5c4c885051 100644 --- a/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py +++ b/tests/unit/src/sqlbuild/runtime/execution_history/test_import_boundary.py @@ -30,7 +30,7 @@ def test_given_clean_interpreter_when_importing_execution_history_then_optional_ module_names: str = ",".join(repr(name) for name in test_case.expected_forbidden_imports) script = ( "import sys; import sqlbuild.execution_history as history; " - "assert history.EventLogStorage is history.LifecycleEventLogStorage; " + "assert not hasattr(history, 'EventLogStorage'); " f"forbidden=({module_names},); " "loaded=tuple(name for name in forbidden if name in sys.modules); print(repr(loaded))" ) diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py index 14626d3566..d501162b99 100644 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py @@ -1,7 +1,5 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction - @dataclass(frozen=True) class HydrationLifecycleTestCase: @@ -44,35 +42,6 @@ class RewriteProjectTargetsTestCase: expected_rewritten_name: str -@dataclass(frozen=True) -class SeededPlanAdaptationTestCase: - description: str - incremental_strategy: str - expected_action: PlanAction - expected_sql_fragment: str - - -@dataclass(frozen=True) -class SeededPlanNoAdaptationTestCase: - description: str - incremental_strategy: str - bound_version_hash: str | None - expected_version_hash: str - materialization_type: MaterializationType - action: PlanAction - cursor_bounds_enabled: bool - expected_action: PlanAction - unexpected_sql_fragment: str - - -@dataclass(frozen=True) -class SeededPlanBoundsPrecedenceTestCase: - description: str - entry_bounds_enabled: bool - expected_sql_fragment: str - unexpected_sql_fragment: str - - @dataclass(frozen=True) class SeedingStrategyTestCase: description: str diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py index 4cdc1f1a89..68dcd3a0f0 100644 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py @@ -45,7 +45,6 @@ class StateLockServiceTestCase: ttl: timedelta expected_virtual_environment_lock_key: str expected_model_version_lock_key: str - expected_state_migration_lock_key: str @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py index 705e7093be..304f30ac97 100644 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py @@ -303,11 +303,6 @@ def upsert_physical_relation_ancestry( ) -> None: return None - def get_physical_relation_ancestry( - self, connection: Any, *, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationAncestryRecord | None: - return None - def upsert_virtual_environment( self, connection: Any, *, schema: str, record: VirtualEnvironmentRecord ) -> None: @@ -431,31 +426,11 @@ def get_virtual_environment_source_freshness( ) -> tuple[SourceFreshnessRecord, ...]: return () - def replace_virtual_environment_function_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - ) -> None: - return None - def get_virtual_environment_function_refs( self, connection: Any, *, schema: str, virtual_environment_name: str ) -> tuple[VirtualEnvironmentFunctionRefRecord, ...]: return () - def replace_virtual_environment_seed_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - ) -> None: - return None - def get_virtual_environment_seed_refs( self, connection: Any, *, schema: str, virtual_environment_name: str ) -> tuple[VirtualEnvironmentSeedRefRecord, ...]: diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py index df8660c332..b4ecf1ed0f 100644 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py +++ b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py @@ -6,7 +6,6 @@ from sqlbuild.virtual.state._helpers.state_storage.locks import ( acquire_model_version_lock, - acquire_state_migration_lock, acquire_virtual_environment_lock, release_state_lock, ) @@ -28,7 +27,6 @@ ttl=timedelta(minutes=15), expected_virtual_environment_lock_key="virtual_env:dev", expected_model_version_lock_key="model_version:fact_orders:abc123", - expected_state_migration_lock_key="state_migration", ) ], ids=lambda case: case.description, @@ -58,26 +56,15 @@ def test_given_state_lock_service_when_acquiring_scoped_locks_then_uses_expected ttl=test_case.ttl, now=test_case.now, ) - state_migration_lease: StateLockLease | None = acquire_state_migration_lock( - backend=backend, - connection=connection, - schema=test_case.schema, - owner_id=test_case.owner_id, - ttl=test_case.ttl, - now=test_case.now, - ) assert virtual_environment_lease is not None assert model_version_lease is not None - assert state_migration_lease is not None assert virtual_environment_lease.lock_key == test_case.expected_virtual_environment_lock_key assert model_version_lease.lock_key == test_case.expected_model_version_lock_key - assert state_migration_lease.lock_key == test_case.expected_state_migration_lock_key assert virtual_environment_lease.expires_at == test_case.now + test_case.ttl assert tuple(call[0] for call in backend.acquire_calls) == ( test_case.expected_virtual_environment_lock_key, test_case.expected_model_version_lock_key, - test_case.expected_state_migration_lock_key, ) assert release_state_lock( backend=backend, @@ -101,7 +88,6 @@ def test_given_state_lock_service_when_acquiring_scoped_locks_then_uses_expected ttl=timedelta(minutes=15), expected_virtual_environment_lock_key="virtual_env:dev", expected_model_version_lock_key="model_version:fact_orders:abc123", - expected_state_migration_lock_key="state_migration", ) ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py index edfd3b1ba1..f70a8f48a3 100644 --- a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py +++ b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py @@ -40,8 +40,6 @@ def test_given_virtual_python_identity_when_recording_then_reads_previous_finger identity: PythonNodeIdentity = PythonNodeIdentity( node_type=test_case.expected_node_type, node_name=test_case.expected_node_name, - object_module="tasks.orders", - object_qualname="prepare_orders", source_path="tasks/orders.py", source_hash="source-hash", definition_hash="definition-hash",